def test_list_config(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
expected = {
"external-url": "https://ci.kbase.us/services/ee2",
"kbase-endpoint": "https://ci.kbase.us/services",
"workspace-url": "https://ci.kbase.us/services/ws",
"catalog-url": "https://ci.kbase.us/services/catalog",
"shock-url": "https://ci.kbase.us/services/shock-api",
"handle-url": "https://ci.kbase.us/services/handle_service",
"srv-wiz-url": "https://ci.kbase.us/services/service_wizard",
"auth-service-url":
"https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login",
"auth-service-url-v2": "https://ci.kbase.us/services/auth/api/V2/token",
"auth-service-url-allow-insecure": "false",
"scratch": "/kb/module/work/tmp",
"executable": "execute_runner.sh",
"docker_timeout": "604805",
"initialdir": "/condor_shared",
"transfer_input_files": "/condor_shared/JobRunner.tgz"
}
config = api.list_config()
self.assertEqual(config, expected)
except Exception as ex:
self.assert_no_exception(ex)
def cancel_job(self, params):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
admin = params.get('admin', False)
if admin:
as_admin = 1
terminated_code = params.get('code', 1)
else:
as_admin = 0
terminated_code = params.get('code', 0)
try:
api.cancel_job({
'job_id': params['job_id'],
'terminated_code': terminated_code,
'as_admin': as_admin
})
return {'canceled': True}
except ServiceError as se:
if re.search(('A job with status .+ cannot be terminated. '
'It is already cancelled.'), se.message):
return {'canceled': False}
elif re.search('Cannot find job with ids:', se.message):
raise ServiceError(
code=10,
message="The job specified for cancelation does not exist",
data={'job_id': params['job_id']})
else:
raise se
def is_admin(self):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
result = api.is_admin()
if result == 1:
return True
else:
return False
def test_status(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
status = api.status()
self.assertEqual(status['version'], '0.0.1')
self.assertEqual(status['service'], 'KBase Execution Engine')
self.assertIsInstance(status['server_time'], float)
self.assertIsInstance(status['git_commit'], str)
except Exception as ex:
self.assert_no_exception(ex)
def test_get_admin_permission_admin(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, 'admin')
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
result = api.get_admin_permission()
schema.validate('get_admin_permission', result)
self.assertEqual(result['permission'], 'w')
except Exception as ex:
self.assert_no_exception(ex)
def test_is_admin_is_not(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
result = api.is_admin()
schema.validate('is_admin', result)
self.assertEqual(result, False)
except Exception as ex:
self.assert_no_exception(ex)
def test_ver(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
# impl, context = self.impl_for(ENV, USER_CLASS)
# params = {
# 'job_ids': [JOB_ID_HAPPY],
# 'timeout': TIMEOUT
# }
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
version = api.ver()
self.assertEqual(version, '0.0.1')
except Exception as ex:
self.assert_no_exception(ex)
def test_check_jobs(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
params = {
'job_ids': [JOB_ID_HAPPY]
}
jobs = api.check_jobs(params)
schema.validate('check_jobs', jobs)
self.assertEqual(jobs['job_states'][0]['user'], 'kbaseuitest')
except Exception as ex:
self.assert_no_exception(ex)
def test_get_job_logs(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
params = {
'job_id': JOB_ID_LOG_HAPPY
}
result = api.get_job_logs(params)
schema.validate('get_job_logs', result)
self.assertEqual(result['lines'][0]['ts'], 1585858534288)
except Exception as ex:
self.assert_no_exception(ex)
def test_check_job_canceled(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, USER_CLASS)
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
params = {
'job_id': JOB_ID_HAPPY
}
result = api.check_job_canceled(params)
schema.validate('check_job_canceled', result)
self.assertEqual(result['canceled'], False)
except Exception as ex:
self.assert_no_exception(ex)
def get_job_log(self, params):
job_id = get_param(params, 'job_id')
offset = get_param(params, 'offset')
limit = get_param(params, 'limit')
# search = params.get('search')
# level = params.get('level')
if params.get('admin', False):
as_admin = 1
else:
as_admin = 0
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
try:
# Note plural form of get_job_log. The upstream apis (njs, ee2 copying it)
# mistakenly use the plural form ... it is a log of a job, not a logs of a job.
result = api.get_job_logs({
'job_id': job_id,
'offset': offset,
'limit': limit,
'as_admin': as_admin
})
except ServiceError as se:
# handle specific error mesages
if se.code == -32000:
if re.search('Cannot find job log with id[s]?:', se.message):
return {'log': [], 'total_count': 0}
# raise ServiceError(
# code=30,
# message='The requested job log could not be found',
# data={
# 'job_id': job_id
# })
raise se
else:
entries = [
raw_log_line_to_entry(line, index, offset)
for index, line in enumerate(result['lines'])
]
if limit is not None:
entries = entries[slice(0, limit)]
# entries = list(map(raw_log_line_to_entry, result['lines']))
return {'log': entries, 'total_count': result['count']}
def test_check_jobs_date_range_for_all(self):
self.set_config('upstream-service', UPSTREAM_SERVICE)
token = self.token_for(ENV, 'admin')
schema = Schema(schema_dir="ee2_api", load_schemas=True)
try:
api = EE2Api(self.get_config('ee2-url'), token, TIMEOUT)
params = {
'start_time': START_TIME_1,
'end_time': END_TIME_1,
'user': USER,
'offset': 0,
'limit': 10
}
result = api.check_jobs_date_range_for_all(params)
schema.validate('check_jobs_date_range_for_all', result)
self.assertEqual(result['jobs'][0]['job_id'], '54b02da8e4b06e6b5555476d')
except Exception as ex:
self.assert_no_exception(ex)
def ee2_get_jobs(self, params):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
if params.get('admin', False):
as_admin = 1
else:
as_admin = 0
try:
jobs = api.check_jobs({
'job_ids': params['job_ids'],
'as_admin': as_admin
})
return jobs['job_states']
except ServiceError as se:
if se.code == -32000:
if re.search('Cannot find job with ids:', se.message):
# Hmm, wonder if the missing job ids are returned in the exception?
raise ServiceError(code=10,
message='Job not found',
data={'message': se.message})
else:
raise ServiceError(code=1,
message='Unknown error occurred',
data={
'upstream_error': {
'code': se.code,
'message': se.message,
'data': se.data
}
})
else:
raise
except Exception as ex:
raise ServiceError(code=1,
message='Unknown error',
data={'original_message': str(ex)})
from JobBrowserBFF.model.EE2Api import EE2Api
import os
import json
import datetime
import time
import math
url = 'https://ci.kbase.us/services/ee2'
token = os.environ['KBASE_TOKEN']
client = EE2Api(url=url, token=token, timeout=60)
version = client.ver()
print('VERSION')
print(version)
START_TIME = 0 # 1970/1/1
END_TIME = 1609459200000 # 2021/1/1
BATCH_SIZE = 1000
def save_job(data, iter):
job_id = data['job']['job_id']
with open('temp/drift/{}-{:03d}.json'.format(job_id, iter), 'w') as f:
f.write(json.dumps(data, indent=4))
def get_last_job():
# get first batch
params = {
'start_time': START_TIME,
'end_time': END_TIME,
'offset': 0,
def ee2_query_jobs(self,
offset,
limit,
time_span=None,
search=None,
filter=None,
sort=None,
admin=False):
# TODO: timeout global or timeout per call?
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
params = {
'start_time': time_span['from'],
'end_time': time_span['to'],
'offset': offset,
'limit': limit
}
if filter is not None:
raw_query = []
status = filter.get('status', [])
if len(status) > 0:
status_transform = {
'create': 'created',
'queue': 'queued',
'run': 'running',
'complete': 'completed',
'error': 'error',
'terminate': 'terminated'
}
# value = list(map(lambda x: status_transform.get(x, x), value))
status = [status_transform.get(x, x) for x in status]
raw_query.append({'status': {'$in': status}})
# parse and reformat the filters...
workspace_id = filter.get('workspace_id', [])
if len(workspace_id) > 0:
raw_query.append({'wsid': {'$in': workspace_id}})
user = filter.get('user', [])
if len(user) > 0:
raw_query.append({'user': {'$in': user}})
client_group = filter.get('client_group', [])
if len(client_group) > 0:
raw_query.append({
'job_input.requirements.clientgroup': {
'$in': client_group
}
})
app_id = filter.get('app_id', [])
if len(app_id) > 0:
raw_query.append({'job_input.app_id': {'$in': app_id}})
app_module = filter.get('app_module', [])
if len(app_module) > 0:
raw_query.append({
'$or': [{
'job_input.app_id': {
'$regex': f'^{module_name}/',
'$options': 'i'
}
} for module_name in app_module]
})
app_function = filter.get('app_function', [])
if len(app_function) > 0:
raw_query.append({
'$or': [{
'job_input.app_id': {
'$regex': f'/{function_name}$',
'$options': 'i'
}
} for function_name in app_function]
})
# wrap it in a raw query for mongoengine
# TODO: upstream ee2 service should not be exposed like this!
if len(raw_query) > 0:
filter_query = {'__raw__': {'$and': raw_query}}
params['filter'] = filter_query
if sort is not None:
# sort specs are not supported for ee2 (for now)
# rather sorting is always by created timestamp
# defaulting to ascending but reversable with the
# ascending parameter set to 0.
if len(sort) == 0:
pass
elif len(sort) > 1:
raise ServiceError(code=40000,
message="Only one sort spec supported",
data={})
elif sort[0].get('key') != 'created':
raise ServiceError(
code=40000,
message="The sort spec must be for the 'created' key")
if sort[0].get('direction') == 'ascending':
ascending = 1
else:
ascending = 0
params['ascending'] = ascending
try:
if admin:
result = api.check_jobs_date_range_for_all(params)
else:
result = api.check_jobs_date_range_for_user(params)
return result['jobs'], result['query_count']
except ServiceError:
raise
except Exception as ex:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(ex)})
def get_client_groups(self):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
result = api.get_client_groups()
return {'client_groups': result}