def test_get_jobs_params_happy(self): try: validation = Validation(load_schemas=True) # schema_path = os.path.dirname(__file__) + '/../../lib/JobBrowserBFF/schemas/' # schema_file_name = 'get_jobs_params.json' # schema = validation.load_schema(schema_file_name, path=schema_path) # self.assertIsInstance(schema, dict) sample_params = [{ 'job_ids': [] }, { 'job_ids': ['hi'] }, { 'job_ids': ['a', 'b', 'c'] }] for sample_param in sample_params: try: validation.validate_params('get_jobs', sample_param) except InvalidParamsError as ve: self.assertTrue( False, 'Unexpected validation failure: ' + ve.message) except Exception as ex: # traceback.print_tb(ex) self.assertTrue(False, 'Did not expect an exception: ' + str(ex)) return
def test_get_jobs_params_sad(self): try: validation = Validation(load_schemas=True) sample_params = [{}, { 'job_ids': None }, { 'job_ids': 1 }, { 'job_ids': 'hello' }, { 'job_ids': [None] }, { 'job_ids': [1] }, { 'job_ids': [['a']] }, { 'job_ids': [{ 'a': 'b' }] }] for sample_param in sample_params: with self.assertRaises(InvalidParamsError): validation.validate_params('get_jobs', sample_param) except Exception as ex: # traceback.print_tb(ex) self.assertTrue(False, 'Did not expect an exception: ' + str(ex)) return
def test_query_jobs_validate_params_happy(self): try: validation = Validation(load_schemas=True) method_names = [ 'query_jobs', 'get_jobs', 'get_job_log', 'cancel_job' ] for method_name in method_names: sample_files = self.get_data_files(method_name, 'happy', 'params') for sample_file in sample_files: data = self.load_data_file(method_name, 'happy', 'params', sample_file) validation.validate_params(method_name, data) except Exception as ex: self.assert_no_exception(ex) return
def __init__(self, config): # BEGIN_CONSTRUCTOR self.validation = Validation(schema_dir="impl", load_schemas=True) # fix up the config because, as an INI file, everything is a string... config['default-timeout'] = int(config['default-timeout']) config['cache-refresh-interval'] = int( config['cache-refresh-interval']) config['cache-refresh-initial-delay'] = int( config['cache-refresh-initial-delay']) self.validation.validate_config(config) self.config = config self.shared_folder = config['scratch'] logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) self.definitions = Definitions(load=True) def setwal(db): db.cursor().execute("pragma journal_mode=wal") # custom auto checkpoint interval (use zero to disable) db.wal_autocheckpoint(0) apsw.connection_hooks.append(setwal) # Set up cache directory Path(config['cache-directory']).mkdir(parents=True, exist_ok=True) # The app cache can be populated upon load. # TODO: need a process to refresh the cache periodically app_cache_path = config['cache-directory'] + '/app.db' app_cache = AppCache(path=app_cache_path, narrative_method_store_url=config['nms-url'], upstream_timeout=60) app_cache.initialize() user_profile_cache_path = config['cache-directory'] + '/user_profile.db' user_profile_cache = UserProfileCache( path=user_profile_cache_path, user_profile_url=config['user-profile-url'], upstream_timeout=60) user_profile_cache.initialize() # END_CONSTRUCTOR pass
def test_get_jobs_validate_result_happy(self): try: validation = Validation(load_schemas=True) method_names = [ 'get_jobs', 'query_jobs', 'get_job_log', 'cancel_job', 'get_client_groups', 'get_job_states', 'get_job_types', 'get_log_levels', 'get_searchable_job_fields' ] for method_name in method_names: sample_files = self.get_data_files(method_name, 'happy', 'result') for sample_file in sample_files: data = self.load_data_file(method_name, 'happy', 'result', sample_file) validation.validate_result(method_name, data) except Exception as ex: self.assert_no_exception(ex) return
def test_get_jobs_result_happy(self): try: validation = Validation(load_schemas=True) # validation = Validation() # schema_path = os.path.dirname(__file__) + '/../../lib/JobBrowserBFF/schemas/' # schema_file_name = 'get_jobs_result.json' # schema = validation.load_schema(schema_file_name, path=schema_path) data_path = os.path.dirname( __file__) + '/../data/methods/get_jobs/happy/result' data_file_name = 'sample1.json' with open(os.path.join(data_path, data_file_name)) as fin: sample_result = json.load(fin) # self.assertIsInstance(schema, dict) try: validation.validate_result('get_jobs', sample_result) except ValidationError as ve: self.assertTrue(False, 'Unexpected validation failure: ' + ve.message) except Exception as ex: # traceback.print_tb(ex) self.assertTrue(False, 'Did not expect an exception: ' + str(ex)) return
import os import json from JobBrowserBFF.Validation import Validation from JobBrowserBFF.model.EE2Model import EE2Model from JobBrowserBFF.model.KBaseServices import KBaseServices validation = Validation(schema_dir='impl', load_schemas=True) schema = { 'type': 'array', 'items': { '$ref': 'base.json#definitions/job_info' } } token = os.environ['KBASE_TOKEN'] env = 'prod' config = { 'ee2-url': 'https://kbase.us/services/ee2', 'workspace-url': 'https://kbase.us/services/ws', 'auth-url': 'https://kbase.us/services/auth', 'catalog-url': 'https://kbase.us/services/catalog', 'nms-url': 'https://kbase.us/services/narrative_method_store/rpc' } model = EE2Model( config=config, token=token, timeout=60,
class JobBrowserBFF: ''' Module Name: JobBrowserBFF Module Description: A KBase module: JobBrowserBFF ''' ######## WARNING FOR GEVENT USERS ####### noqa # Since asynchronous IO can lead to methods - even the same method - # interrupting each other, you must be *very* careful when using global # state. A method could easily clobber the state set by another while # the latter method is running. ######################################### noqa VERSION = "0.0.1" GIT_URL = "" GIT_COMMIT_HASH = "0de05d2b9029adbdcdb546279cb82c09e16daa7f" # BEGIN_CLASS_HEADER # END_CLASS_HEADER # config contains contents of config file in a hash or None if it couldn't # be found def __init__(self, config): # BEGIN_CONSTRUCTOR self.validation = Validation(schema_dir="impl", load_schemas=True) # fix up the config because, as an INI file, everything is a string... config['default-timeout'] = int(config['default-timeout']) config['cache-refresh-interval'] = int( config['cache-refresh-interval']) config['cache-refresh-initial-delay'] = int( config['cache-refresh-initial-delay']) self.validation.validate_config(config) self.config = config self.shared_folder = config['scratch'] logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) self.definitions = Definitions(load=True) def setwal(db): db.cursor().execute("pragma journal_mode=wal") # custom auto checkpoint interval (use zero to disable) db.wal_autocheckpoint(0) apsw.connection_hooks.append(setwal) # Set up cache directory Path(config['cache-directory']).mkdir(parents=True, exist_ok=True) # The app cache can be populated upon load. # TODO: need a process to refresh the cache periodically app_cache_path = config['cache-directory'] + '/app.db' app_cache = AppCache(path=app_cache_path, narrative_method_store_url=config['nms-url'], upstream_timeout=60) app_cache.initialize() user_profile_cache_path = config['cache-directory'] + '/user_profile.db' user_profile_cache = UserProfileCache( path=user_profile_cache_path, user_profile_url=config['user-profile-url'], upstream_timeout=60) user_profile_cache.initialize() # END_CONSTRUCTOR pass def get_jobs(self, ctx, params): """ :param params: instance of type "GetJobsParams" (get_jobs Given a set of job ids, returns the job information for each job, in the same order as the ids were provided. As with other methods, this one takes an "admin" parameter which indicates whether the call is intended for administrator usage or not. If for administrator usage, the token provided in the call must be associated with an account with admin privileges for the upstream service. An error with code 50 is returned otherwise. Params: - job_ids: a list of job ids to look up and provide information about - admin: a boolean indicating whether the request is for a admin usage or not Returns: - jobs - list of JobStatus Throws: - 10 - Job not found: If the any of the given job ids are not found) -> structure: parameter "job_ids" of list of type "JobID" (A job id is a uuid), parameter "admin" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) :returns: instance of type "GetJobsResult" -> structure: parameter "jobs" of list of type "JobInfo" -> structure: parameter "job_id" of type "JobID" (A job id is a uuid), parameter "owner" of type "User" -> structure: parameter "username" of type "username" (A KBase username), parameter "realname" of String, parameter "state" of type "JobState" (Superset of all fields used to represent job state See the TS typing and json-schema) -> structure: parameter "status" of type "JobStatus" (create | queue | run | complete | error | terminate), parameter "create_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "queue_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "run_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "finish_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "client_group" of type "ClientGroup" (njs, bigmem, bigmemlong, kb_import, ...), parameter "error" of type "JobError" -> structure: parameter "code" of type "JobErrorCode", parameter "message" of String, parameter "service_error" of type "JSONRPC11Error" -> structure: parameter "code" of Long, parameter "message" of String, parameter "error" of unspecified object, parameter "termination" of type "JobTermination" -> structure: parameter "code" of type "JobTerminationCode", parameter "message" of String, parameter "app" of type "AppInfo" -> structure: parameter "module_name" of String, parameter "function_name" of String, parameter "title" of String, parameter "client_groups" of list of String, parameter "context" of type "JobContext" (The JobContext represents the context in which the Job was run. The `type` field Every job is run with some context. A) -> structure: parameter "type" of type "JobContextType" (narrative, export, workspace, unknown), parameter "workspace" of type "WorkspaceInfo" (Information about the workspace the job is associated with. Most, but not all, jobs are associated with a workspace. Note that only minimal information is exposed here, since this is all the the job browser requires. The design philosopy of this module is minimal support of the associated ui component.) -> structure: parameter "id" of Long, parameter "is_accessible" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "name" of String, parameter "is_deleted" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "narrative" of type "NarrativeInfo" (Information about the narrative with which the job is associated, if the workspace it is associated with is also a Narrative. Note that only minimal information is available at this time, since this is all that is required of a job browser. Future enhancments of a job browser may require additional fields here.) -> structure: parameter "title" of String, parameter "is_temporary" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) """ # ctx is the context object # return variables are: result # BEGIN get_jobs self.validation.validate_params('get_jobs', params) model = Model(config=self.config, context=ctx, timeout=params['timeout']).get_model(ctx) jobs, stats = model.get_jobs(params) result = {'jobs': jobs, 'stats': stats} self.validation.validate_result('get_jobs', result) return result # END get_jobs def query_jobs(self, ctx, params): """ :param params: instance of type "QueryJobsParams" (TODO: expand to match the filtering, sorting, searching of kb_metrics) -> structure: parameter "jobs" of list of type "JobID" (A job id is a uuid), parameter "sort" of list of type "SortSpec" -> structure: parameter "key" of type "SortKey" (behaves as an enum: narrative, app, submitted, status), parameter "direction" of type "SortDirection" (behaves as an enum: ascending, descending), parameter "search" of type "SearchSpec" -> structure: parameter "terms" of list of String, parameter "filter" of type "FilterSpec" -> structure: parameter "workspace_id" of list of Long, parameter "status" of list of String, parameter "username" of list of String, parameter "app_id" of list of String, parameter "job_id" of list of String, parameter "error_code" of list of Long, parameter "terminated_code" of list of Long, parameter "time_span" of type "TimeSpanSpec" -> structure: parameter "from" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "to" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "client_groups" of list of type "ClientGroup" (njs, bigmem, bigmemlong, kb_import, ...), parameter "offset" of Long, parameter "limit" of Long, parameter "admin" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) :returns: instance of type "QueryJobsResult" -> structure: parameter "jobs" of list of type "JobInfo" -> structure: parameter "job_id" of type "JobID" (A job id is a uuid), parameter "owner" of type "User" -> structure: parameter "username" of type "username" (A KBase username), parameter "realname" of String, parameter "state" of type "JobState" (Superset of all fields used to represent job state See the TS typing and json-schema) -> structure: parameter "status" of type "JobStatus" (create | queue | run | complete | error | terminate), parameter "create_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "queue_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "run_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "finish_at" of type "epoch_time" (Time represented as epoch time in milliseconds), parameter "client_group" of type "ClientGroup" (njs, bigmem, bigmemlong, kb_import, ...), parameter "error" of type "JobError" -> structure: parameter "code" of type "JobErrorCode", parameter "message" of String, parameter "service_error" of type "JSONRPC11Error" -> structure: parameter "code" of Long, parameter "message" of String, parameter "error" of unspecified object, parameter "termination" of type "JobTermination" -> structure: parameter "code" of type "JobTerminationCode", parameter "message" of String, parameter "app" of type "AppInfo" -> structure: parameter "module_name" of String, parameter "function_name" of String, parameter "title" of String, parameter "client_groups" of list of String, parameter "context" of type "JobContext" (The JobContext represents the context in which the Job was run. The `type` field Every job is run with some context. A) -> structure: parameter "type" of type "JobContextType" (narrative, export, workspace, unknown), parameter "workspace" of type "WorkspaceInfo" (Information about the workspace the job is associated with. Most, but not all, jobs are associated with a workspace. Note that only minimal information is exposed here, since this is all the the job browser requires. The design philosopy of this module is minimal support of the associated ui component.) -> structure: parameter "id" of Long, parameter "is_accessible" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "name" of String, parameter "is_deleted" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "narrative" of type "NarrativeInfo" (Information about the narrative with which the job is associated, if the workspace it is associated with is also a Narrative. Note that only minimal information is available at this time, since this is all that is required of a job browser. Future enhancments of a job browser may require additional fields here.) -> structure: parameter "title" of String, parameter "is_temporary" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "found_count" of Long, parameter "total_count" of Long """ # ctx is the context object # return variables are: result # BEGIN query_jobs self.validation.validate_params('query_jobs', params) model = Model(self.config, ctx).get_model(ctx) jobs, found_count, total_count, stats = model.query_jobs(params) result = { 'jobs': jobs, 'found_count': found_count, 'total_count': total_count, 'stats': stats } self.validation.validate_result('query_jobs', result) return result # END query_jobs def get_job_log(self, ctx, params): """ :param params: instance of type "GetJobLogParams" -> structure: parameter "job_id" of type "JobID" (A job id is a uuid), parameter "search" of type "SearchSpec" -> structure: parameter "terms" of list of String, parameter "level" of list of type "LogLevel" (enum-like: default, error), parameter "offset" of Long, parameter "limit" of Long, parameter "admin" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) :returns: instance of type "GetJobLogResult" -> structure: parameter "log" of list of type "LogEntry" -> structure: parameter "entry_number" of Long, parameter "created" of Long, parameter "entry" of String, parameter "level" of type "LogLevel" (enum-like: default, error), parameter "total_count" of Long """ # ctx is the context object # return variables are: result # BEGIN get_job_log self.validation.validate_params('get_job_log', params) model = Model(config=self.config, context=ctx, timeout=params['timeout']).get_model(ctx) result = model.get_job_log(params) self.validation.validate_result('get_job_log', result) return result # END get_job_log def cancel_job(self, ctx, params): """ :param params: instance of type "CancelJobParams" (cancel_job Given a job id, attempt to cancel the associated job. Params: - job_id: The id for the job to cancel Returns: - nothing. Throws: - 10 - Job not found: If the given job id was not found Note that attempting to cancel a job which is not cancelable will not throw an error. This behavior may change in the future. At present one upstream service (njsw) ignores this condition, but another (ee2) returns an error. For ee2 that error is ignored.) -> structure: parameter "job_id" of type "JobID" (A job id is a uuid), parameter "admin" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0), parameter "timeout" of Long :returns: instance of type "CancelJobResult" -> structure: parameter "canceled" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) """ # ctx is the context object # return variables are: result # BEGIN cancel_job self.validation.validate_params('cancel_job', params) model = Model(config=self.config, context=ctx, timeout=params['timeout']).get_model(ctx) result = model.cancel_job(params) self.validation.validate_result('cancel_job', result) return result # END cancel_job def get_job_types(self, ctx): """ :returns: instance of type "GetJobTypesResult" (********* * get_job_types *********) -> structure: parameter "job_type_definitions" of list of type "DomainDefinition" -> structure: parameter "code" of String, parameter "description" of String, parameter "notes" of String """ # ctx is the context object # return variables are: result # BEGIN get_job_types # No params to validate! d = self.definitions.get('job_types') result = {'job_types': d} self.validation.validate_result('get_job_types', result) return result # END get_job_types def get_job_states(self, ctx): """ :returns: instance of type "GetJobStatesResult" (********* * get_job_states *********) -> structure: parameter "job_states" of list of type "DomainDefinition" -> structure: parameter "code" of String, parameter "description" of String, parameter "notes" of String """ # ctx is the context object # return variables are: result # BEGIN get_job_states d = self.definitions.get('job_states') result = {'job_states': d} self.validation.validate_result('get_job_states', result) return result # END get_job_states def get_client_groups(self, ctx): """ :returns: instance of type "GetClientGroupsResult" (********* * get_client_groups *********) -> structure: parameter "client_groups" of list of type "ClientGroup" (njs, bigmem, bigmemlong, kb_import, ...) """ # ctx is the context object # return variables are: result # BEGIN get_client_groups model = Model(self.config, ctx).get_model(ctx) result = model.get_client_groups() self.validation.validate_result('get_client_groups', result) return result # END get_client_groups def get_searchable_job_fields(self, ctx): """ :returns: instance of type "GetSearchableJobFieldsResult" (********* * get_searchable_job_fields *********) -> structure: parameter "searchable_job_fields" of list of type "DomainDefinition" -> structure: parameter "code" of String, parameter "description" of String, parameter "notes" of String """ # ctx is the context object # return variables are: result # BEGIN get_searchable_job_fields d = self.definitions.get('searchable_job_fields') result = {'searchable_job_fields': d} self.validation.validate_result('get_searchable_job_fields', result) return result # END get_searchable_job_fields def get_sort_specs(self, ctx): """ :returns: instance of type "GetSortSpecsResult" -> structure: parameter "sort_fields" of list of type "SortSpecDefinition" (********* * get_sort_keys *********) -> structure: parameter "key" of String, parameter "fields" of list of String, parameter "description" of String """ # ctx is the context object # return variables are: result # BEGIN get_sort_specs d = self.definitions.get('sort_specs') result = {'sort_specs': d} self.validation.validate_result('get_sort_specs', result) return result # END get_sort_specs def get_log_levels(self, ctx): """ :returns: instance of type "GetLogLevelsResult" (********* * get_log_levels *********) -> structure: parameter "log_levels" of list of type "OrderedDomainDefinition" -> structure: parameter "code" of String, parameter "order" of Long, parameter "description" of String, parameter "notes" of String """ # ctx is the context object # return variables are: result # BEGIN get_log_levels d = self.definitions.get('log_levels') result = {'log_levels': d} self.validation.validate_result('get_log_levels', result) return result # END get_log_levels def is_admin(self, ctx): """ :returns: instance of type "IsAdminResult" (********* * is_admin *********) -> structure: parameter "is_admin" of type "bool" (In kb_sdk boolean values are represented as integer 1 and 0) """ # ctx is the context object # return variables are: result # BEGIN is_admin model = Model(self.config, ctx).get_model(ctx) is_admin = model.is_admin() result = {'is_admin': is_admin} self.validation.validate_result('is_admin', result) return result # END is_admin def status(self, ctx): # BEGIN_STATUS returnVal = { 'state': "OK", 'message': "", 'version': self.VERSION, 'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH } return returnVal
import os import json from JobBrowserBFF.Validation import Validation validation = Validation(schema_dir='ee2_api', load_schemas=True) def validate_job(job): job_id = job['job_id'] # print(f'validate? {job_id}') try: validation.validate({'$ref': 'base.json#definitions/job_state'}, job) # print(f'OK: {job_id}') print('.', end='', flush=True) except Exception as ex: print() print(f'ERROR {job_id}: {str(ex)}') print(type(ex)) print(job) print() def count_all_jobs(env): entries = os.scandir(f'temp/jobs/{env}') total = 0 for entry in entries: total = total + 1 return total def validate_all_jobs(env):