def get_users(self, user_ids):
url = self.config['auth-url']
header = {
'Accept': 'application/json',
'Authorization': self.token,
}
endpoint = url + '/api/V2/users/?list=' + ','.join(user_ids)
response = requests.get(endpoint,
headers=header,
timeout=self.timeout / 1000)
if response.status_code != 200:
raise ServiceError(code=40000,
message='Error fetching users',
data={'user_id': user_ids})
else:
try:
result = response.json()
retval = dict()
for username, realname in result.items():
retval[username] = {'realname': realname}
return retval
except Exception as err:
raise ServiceError(code=40000,
message='Bad response',
data={
'user_id': user_ids,
'original_message': str(err)
})
def get_user(self):
url = self.config['auth-url']
header = {
'Accept': 'application/json',
'Authorization': self.token,
}
endpoint = url + '/api/V2/me'
response = requests.get(endpoint, headers=header, timeout=10)
if response.status_code != 200:
raise ServiceError(code=40000, message='Error fetching me')
else:
try:
result = response.json()
return {
'username': result['user'],
'realname': result['display']
}
except Exception as err:
raise ServiceError(code=40000,
message='Bad response',
data={'original_message': str(err)})
def cancel_job(self, params):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
admin = params.get('admin', False)
if admin:
as_admin = 1
terminated_code = params.get('code', 1)
else:
as_admin = 0
terminated_code = params.get('code', 0)
try:
api.cancel_job({
'job_id': params['job_id'],
'terminated_code': terminated_code,
'as_admin': as_admin
})
return {'canceled': True}
except ServiceError as se:
if re.search(('A job with status .+ cannot be terminated. '
'It is already cancelled.'), se.message):
return {'canceled': False}
elif re.search('Cannot find job with ids:', se.message):
raise ServiceError(
code=10,
message="The job specified for cancelation does not exist",
data={'job_id': params['job_id']})
else:
raise se
def check_job_canceled(self, params):
try:
return self.rpc.call_func('check_job_canceled', params)
except ServiceError:
raise
except Exception as err:
raise ServiceError(code=1,
message='Unknown error',
data={'original_message': str(err)})
def get_job_status(self, params):
try:
return self.rpc.call_func('get_job_status', params)
except ServiceError:
raise
except Exception as err:
raise ServiceError(code=1,
message='Unknown error',
data={'original_message': str(err)})
def cancel_job(self, params):
try:
return self.rpc.call_func('cancel_job', params)
except ServiceError:
raise
except Exception as ex:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(ex)})
def status(self):
try:
return self.rpc.call_func('status')
except ServiceError:
raise
except Exception as err:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(err)})
def get_client_groups(self):
try:
return self.rpc.call_func('get_client_groups')
except ServiceError:
raise
except Exception as err:
raise ServiceError(code=1,
message='Unknown error',
data={'original_message': str(err)})
def check_jobs_date_range_for_all(self, params):
try:
return self.rpc.call_func('check_jobs_date_range_for_all', params)
except ServiceError:
raise
except Exception as ex:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(ex)})
def is_admin(self):
try:
result = self.rpc.call_func('is_admin')
return result
except ServiceError:
raise
except Exception as ex:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(ex)})
def get_job_log(self, job_id, offset, limit, search=None, level=None):
# TODO: enforce perms
# First, get the job and ensure the user has access to the logs.
jobs_collection = self.db['jobs']
raw_job = jobs_collection.find_one({'job_id': job_id}, {'_id': False})
if raw_job is None:
raise ServiceError(
code=10,
message='Job {} could not be found'.format(job_id),
data={'job_id': job_id})
job = json.loads(json_util.dumps(raw_job))
if job['owner']['username'] != self.username:
raise ServiceError(
code=40,
message='Access denied to job {}'.format(job_id),
data={
'job_id': job_id,
'username': self.username
})
# Now get the logs
logs_collection = self.db['job_logs']
filters = []
filters.append({'job_id': job_id})
cursor = logs_collection.find({'$and': filters})
total_count = cursor.count()
cursor.sort([('id', 1)])
cursor.skip(offset)
cursor.limit(limit)
log = json.loads(json_util.dumps(cursor))
entries = [
raw_log_entry_to_entry(line, index, offset)
for index, line in enumerate(log)
]
return {'job': job, 'log': entries, 'total_count': total_count}
def get_job(self, job_id):
# First, get the job and ensure the user has access to the logs.
jobs_collection = self.db['jobs']
raw_job = jobs_collection.find_one({'job_id': job_id}, {'_id': False})
if raw_job is None:
raise ServiceError(
code=10,
message='Job {} could not be found'.format(job_id),
data={'job_id': job_id})
job = json.loads(json_util.dumps(raw_job))
if job['owner']['username'] != self.username:
raise ServiceError(
code=40,
message='Access denied to job {}'.format(job_id),
data={
'job_id': job_id,
'username': self.username
})
return job
def ee2_get_jobs(self, params):
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
if params.get('admin', False):
as_admin = 1
else:
as_admin = 0
try:
jobs = api.check_jobs({
'job_ids': params['job_ids'],
'as_admin': as_admin
})
return jobs['job_states']
except ServiceError as se:
if se.code == -32000:
if re.search('Cannot find job with ids:', se.message):
# Hmm, wonder if the missing job ids are returned in the exception?
raise ServiceError(code=10,
message='Job not found',
data={'message': se.message})
else:
raise ServiceError(code=1,
message='Unknown error occurred',
data={
'upstream_error': {
'code': se.code,
'message': se.message,
'data': se.data
}
})
else:
raise
except Exception as ex:
raise ServiceError(code=1,
message='Unknown error',
data={'original_message': str(ex)})
def cancel_job(self, params):
admin = params.get('admin', False)
if admin:
as_admin = 1
terminated_code = params.get('code', 1)
else:
as_admin = 0
terminated_code = params.get('code', 0)
job_id = params['job_id']
job = self.get_job(job_id)
if job['state']['status'] not in ['create', 'queue', 'run']:
return ServiceError(code=60,
message='Job is not cancelable',
data={
'job_id': job_id,
'job_status': job['state']['status']
})
def get_jobs(self, params):
jobs_collection = self.db['jobs']
filters = []
if params.get('admin', False):
if not self.is_admin():
raise ServiceError(
code=50,
message='Permission denied for this operation',
data={})
else:
filters.append({'owner.username': self.username})
filters.append({'job_id': {'$in': params['job_ids']}})
query = {'$and': filters}
cursor = jobs_collection.find(query, {'_id': False})
jobs = json.loads(json_util.dumps(cursor))
return jobs
def call_func(self, func_name, params=None, timeout=None):
# Since this is designed to call KBase JSONRPC 1.1 services, we
# follow the KBase conventions:
# - params are always wrapped in an array; this emulates positional arguments
# - a method with no arguments is called as either missing params in the call
# or the value None, and represented as an empty array in the service call
# - a method with params is called with a single argument value which must be
# convertable to JSON; it is represented in the call to the service as an array
# wrapping that value.
# Note that KBase methods can take multiple positional arguments (multiple elements
# in the array), but by far most take just a single argument; this library makes that
# simplifying assumption.
if params is None:
wrapped_params = []
else:
wrapped_params = [params]
# The standard jsonrpc 1.1 calling object, with KBase conventions.
# - id should be unique for this call (thus uuid), but is unused; it isn't
# really relevant for jsonrpc over http since each request always has a
# corresponding response; but it could aid in debugging since it connects
# the request and response.
# - method is always the module name, a period, and the function name
# - version is always 1.1 (even though there was no officially published jsonrpc 1.1)
# - params as discussed above.
call_params = {
'id': str(uuid.uuid4()),
'method': self.module + '.' + func_name,
'version': '1.1',
'params': wrapped_params
}
header = {
'Content-Type': 'application/json'
}
# Calls may be authorized or not with a KBase token.
if self.token:
header['Authorization'] = self.token
timeout = timeout or self.timeout
# Note that timeout should be set here (except for type errors).
# The constructor requires it, and it can be overridden in the call
# to this method.
try:
response = requests.post(self.url, headers=header,
data=json.dumps(call_params), timeout=timeout)
except requests.exceptions.ReadTimeout as rte:
# Note that ServiceError mirrors (and is eventually converted to)
# a JSONRPC 2.0 error.
raise ServiceError(
code=100,
message='Timeout calling service endpoint',
data={
'url': self.url,
'headers': header,
'timeout': timeout,
'python_exception_string': str(rte)
})
except requests.exceptions.ConnectionError as rte:
# Note that ServiceError mirrors (and is eventually converted to)
# a JSONRPC 2.0 error.
raise ServiceError(
code=100,
message='Timeout calling service endpoint',
data={
'url': self.url,
'headers': header,
'timeout': timeout,
'python_exception_string': str(rte)
})
except Exception as ex:
raise ServiceError(
code=101,
message='Error calling service endpoint: ' + ex.message,
data={
'url': self.url,
'headers': header,
})
else:
if response.headers.get('content-type', '').startswith('application/json'):
try:
response_data = response.json()
except json.decoder.JSONDecodeError as err:
raise ServiceError(
code=102,
message='Invalid response from upstream service - not json',
data={
'url': self.url,
'headers': header,
'decoding_error': str(err),
'response_text': response.content
})
else:
error = response_data.get('error')
if error:
# Here we convert from the upstream KBase JSONRPC 1.1 error response
# to a JSONRPC 2.0 compatible exception. We pluck off commonly used
# error properties and put them into the data property. We do
# retain the error code (which should be JSONRPC 1.1 & 2.0 compatible).
error_data = {
'stack': error.get('error'),
'name': error.get('name')
}
raise ServiceError(
code=error.get('code'),
message=error.get('message', error.get('name')),
data=error_data)
result = response_data.get('result')
# The normal response has a result property which, like the params, is
# wrapped in an array. In this case the array emulates multiple value return.
# By far most KBase services keep that to a single array element, and provide
# multiple values if need be (most do) by usage of an object.
if isinstance(result, list):
if len(result) > 0:
return result[0]
else:
return None
# The one exception is when the result is just "null", which emulates a method
# with no, or void, result value.
elif result is None:
return result
else:
raise ServiceError(
code=103,
message=('Unexpected type in upstream service result; '
'must be array or null'),
data={
'url': self.url,
'headers': header,
'result': result,
'result_type': type(result).__name__
})
# Otherwise, if the service does not return json and has a 4xx or 5xx response,
# raise an HTTPError from requests. This will be caught by the caller and
# converted to a general purpose "unknown error" jsonrpc error response.
response.raise_for_status()
# If we get here, the response status is < 400 and not of type application/json,
# thus an invalid response.
raise ServiceError(
code=104,
message=('Invalid response from upstream service; '
'not application/json, not an error status'),
data={
'url': self.url,
'headers': header,
'response_status': response.status_code,
'response_content_type': response.headers.get('content-type', None),
'response_content': response.content
})
def ee2_query_jobs(self,
offset,
limit,
time_span=None,
search=None,
filter=None,
sort=None,
admin=False):
# TODO: timeout global or timeout per call?
api = EE2Api(url=self.config['ee2-url'],
token=self.token,
timeout=self.timeout)
params = {
'start_time': time_span['from'],
'end_time': time_span['to'],
'offset': offset,
'limit': limit
}
if filter is not None:
raw_query = []
status = filter.get('status', [])
if len(status) > 0:
status_transform = {
'create': 'created',
'queue': 'queued',
'run': 'running',
'complete': 'completed',
'error': 'error',
'terminate': 'terminated'
}
# value = list(map(lambda x: status_transform.get(x, x), value))
status = [status_transform.get(x, x) for x in status]
raw_query.append({'status': {'$in': status}})
# parse and reformat the filters...
workspace_id = filter.get('workspace_id', [])
if len(workspace_id) > 0:
raw_query.append({'wsid': {'$in': workspace_id}})
user = filter.get('user', [])
if len(user) > 0:
raw_query.append({'user': {'$in': user}})
client_group = filter.get('client_group', [])
if len(client_group) > 0:
raw_query.append({
'job_input.requirements.clientgroup': {
'$in': client_group
}
})
app_id = filter.get('app_id', [])
if len(app_id) > 0:
raw_query.append({'job_input.app_id': {'$in': app_id}})
app_module = filter.get('app_module', [])
if len(app_module) > 0:
raw_query.append({
'$or': [{
'job_input.app_id': {
'$regex': f'^{module_name}/',
'$options': 'i'
}
} for module_name in app_module]
})
app_function = filter.get('app_function', [])
if len(app_function) > 0:
raw_query.append({
'$or': [{
'job_input.app_id': {
'$regex': f'/{function_name}$',
'$options': 'i'
}
} for function_name in app_function]
})
# wrap it in a raw query for mongoengine
# TODO: upstream ee2 service should not be exposed like this!
if len(raw_query) > 0:
filter_query = {'__raw__': {'$and': raw_query}}
params['filter'] = filter_query
if sort is not None:
# sort specs are not supported for ee2 (for now)
# rather sorting is always by created timestamp
# defaulting to ascending but reversable with the
# ascending parameter set to 0.
if len(sort) == 0:
pass
elif len(sort) > 1:
raise ServiceError(code=40000,
message="Only one sort spec supported",
data={})
elif sort[0].get('key') != 'created':
raise ServiceError(
code=40000,
message="The sort spec must be for the 'created' key")
if sort[0].get('direction') == 'ascending':
ascending = 1
else:
ascending = 0
params['ascending'] = ascending
try:
if admin:
result = api.check_jobs_date_range_for_all(params)
else:
result = api.check_jobs_date_range_for_user(params)
return result['jobs'], result['query_count']
except ServiceError:
raise
except Exception as ex:
raise ServiceError(code=40000,
message='Unknown error',
data={'original_message': str(ex)})
def query_jobs(self, params):
jobs_collection = self.db['jobs']
is_admin = params.get('admin', False)
filters = []
# Enforce the single user model if not admin.
if is_admin:
if not self.is_admin():
raise ServiceError(
code=50,
message='Permission denied for this operation',
data={})
else:
filters.append({'owner.username': self.username})
# Get the total # jobs for this user, as the "total_count"
if len(filters) > 0:
cursor = jobs_collection.find({'$and': filters})
total_count = cursor.count()
else:
total_count = jobs_collection.count()
# Apply the time range.
filters.append({
'$and': [{
'state.create_at': {
'$gte': params['time_span']['from']
}
}, {
'state.create_at': {
'$lt': params['time_span']['to']
}
}]
})
# Handle field-specific filters
if 'filter' in params:
for key, value in params['filter'].items():
if key == 'status':
filters.append({'state.status': value})
elif key == 'app':
# filters.append({'$or': [
# {'app.function_name': value},
# {'app.module_name': value}
# ]})
filters.append({'app.id': value})
elif key == 'workspace_id':
filters.append({'context.workspace.id': value})
elif key == 'job_id':
filters.append({'job_id': value})
elif key == 'client_group':
filters.append({'state.client_group': value})
# Handle free-text search
if 'search' in params:
term_expressions = []
for term in params['search']['terms']:
term_expressions.append({
'$or': [{
'job_id': {
'$regex': term,
'$options': 'i'
}
}, {
'app.id': {
'$regex': term,
'$options': 'i'
}
}, {
'state.status': {
'$regex': term,
'$options': 'i'
}
}, {
'state.client_group': {
'$regex': term,
'$options': 'i'
}
}, {
'owner.username': {
'$regex': term,
'$options': 'i'
}
}, {
'msg': {
'$regex': term,
'$options': 'i'
}
}]
})
filters.append({'$and': term_expressions})
# Put it all together and do it.
query = {'$and': filters}
cursor = jobs_collection.find(query, {'_id': False})
found_count = cursor.count()
# Handle sorting.
if 'sort' in params:
sort = []
for sort_spec in params['sort']:
key = sort_spec['key']
direction = sort_spec['direction']
if direction == 'ascending':
sort_direction = 1
else:
sort_direction = -1
if key == 'created':
sort_key = 'state.create_at'
sort.append((sort_key, sort_direction))
cursor.sort(sort)
cursor.skip(params['offset'])
cursor.limit(params['limit'])
# TODO: filter on username and check if admin.
jobs = json.loads(json_util.dumps(cursor))
# Fix up workspace info relative to the current user.
self.fix_workspaces(jobs)
return jobs, found_count, total_count