def get_queue_names(_id):
"""
List the queues available for job-running
Note: does not return celery internal queues
@param _id - identity of job
@return: list of queues
"""
queues = set(get_all_queues(
app.config["RABBITMQ_ADMIN_API"])) # Non-celery queues set
protected = set(app.config["PROTECTED_QUEUES"])
visible = queues - protected # Visible generic queues
spec = {}
try:
spec = mozart_es.get_by_id(index="job_specs", id=_id, ignore=404)
if spec['found'] is True:
spec = spec['_source']
except Exception as e:
app.logger.warn(
"Failed to get job-spec: {0} proceeding without it. {1}:{2}".
format(_id, type(e), e))
# adding backwards compatibility to queues
required = set(spec.get("required-queues", spec.get("required_queues",
[])))
recommended = set(
spec.get("recommended-queues", spec.get("recommended_queues", [])))
queue_config = {
"queues": sorted(visible | required | recommended),
"recommended": sorted(required | recommended)
}
return queue_config
def get(self, _id):
doc_fields = ['status', 'job.job_info.metrics.products_staged']
prod = mozart_es.get_by_id(index=JOB_STATUS_INDEX, id=_id, _source_includes=doc_fields, ignore=404)
app.logger.info('fetch products staged for %s' % _id)
if prod['found'] is False:
return {
'success': False,
'message': 'Job id not found: %s' % _id
}, 404
doc = prod['_source']
status = doc['status']
if status not in {'job-failed', 'job-completed'}:
return {
'success': False,
'message': 'job has not been completed'
}
try:
return {
'success': True,
'message': status,
'results': doc['job']['job_info']['metrics']['products_staged']
}
except (KeyError, Exception):
app.logger.warning('%s does not have any products_staged' % _id)
return {
'success': True,
'message': status,
'results': []
}
def get(self):
"""Get information on container by ID"""
_id = request.form.get('id', request.args.get('id', None))
container = mozart_es.get_by_id(index=CONTAINERS_INDEX,
id=_id,
ignore=404)
if container['found'] is False:
return {'success': False, 'message': ""}, 404
return {'success': True, 'message': "", 'result': container['_source']}
def get(self):
"""Gets a HySDS-IO specification by ID"""
_id = request.form.get('id', request.args.get('id', None))
if _id is None:
return {'success': False, 'message': 'missing parameter: id'}, 400
hysds_io = mozart_es.get_by_id(index=HYSDS_IOS_INDEX,
id=_id,
ignore=404)
if hysds_io['found'] is False:
return {'success': False, 'message': ""}, 404
return {'success': True, 'message': "", 'result': hysds_io['_source']}
def get(self):
"""Retrieve user rule(s)"""
_id = request.args.get("id", None)
_rule_name = request.args.get("rule_name", None)
user_rules_index = app.config['USER_RULES_INDEX']
if _id:
rule = mozart_es.get_by_id(index=user_rules_index, id=_id, ignore=404)
if rule.get("found", False) is False:
return {
'success': False,
'message': rule['message']
}, 404
else:
rule = {**rule, **rule["_source"]}
rule.pop("_source", None)
return {
'success': True,
'rule': rule
}
elif _rule_name:
result = mozart_es.search(index=user_rules_index, q="rule_name:{}".format(_rule_name), ignore=404)
if result.get("hits", {}).get("total", {}).get("value", 0) == 0:
return {
"success": False,
"message": "rule {} not found".format(_rule_name)
}, 404
rule = result.get("hits").get("hits")[0]
rule = {**rule, **rule["_source"]}
rule.pop("_source", None)
return {
"success": True,
"rule": rule
}
user_rules = mozart_es.query(index=user_rules_index)
parsed_user_rules = []
for rule in user_rules:
rule_copy = rule.copy()
rule_temp = {**rule_copy, **rule['_source']}
rule_temp.pop('_source')
parsed_user_rules.append(rule_temp)
return {
'success': True,
'rules': parsed_user_rules
}
def get(self):
"""Gets the status of a submitted job based on job id"""
_id = request.form.get('id', request.args.get('id', None))
if _id is None:
return {'success': False, 'message': 'id not supplied'}, 400
job_status = mozart_es.get_by_id(index=JOB_STATUS_INDEX, id=_id, ignore=404, _source=['status'])
if job_status['found'] is False:
return {
'success': False,
'message': 'job status not found: %s' % _id
}, 404
return {
'success': True,
'message': "",
'status': job_status['_source']['status']
}
def delete(self):
"""remove user defined tag for job record"""
_id = request.args.get('id')
_index = request.args.get('index')
tag = request.args.get('tag')
app.logger.info('_id: %s _index: %s tag: %s' % (_id, _index, tag))
if _index != 'job_status-current':
app.logger.error('user tags only for index: job_status-current')
return {
'success': False,
'message': 'user tags only for index: job_status-current'
}, 400
if _id is None or _index is None:
return {
'success': False,
'message': 'id and index must be supplied'
}, 400
dataset = mozart_es.get_by_id(index=_index, id=_id, ignore=404)
if dataset['found'] is False:
return {'success': False, 'message': "dataset not found"}, 404
source = dataset['_source']
user_tags = source.get('user_tags', [])
app.logger.info('found user tags %s' % str(user_tags))
if tag in user_tags:
user_tags.remove(tag)
app.logger.info('tags after removing: %s' % str(user_tags))
else:
app.logger.warning('tag not found: %s' % tag)
update_doc = {'doc_as_upsert': True, 'doc': {'user_tags': user_tags}}
mozart_es.update_document(index=_index,
id=_id,
body=update_doc,
refresh=True)
return {'success': True, 'tags': user_tags}
def get(self):
"""Get complete info for submitted job based on id"""
_id = request.form.get('id', request.args.get('id', None))
if _id is None:
return {
'success': False,
'message': 'id must be supplied (as query param or url param)'
}, 400
info = mozart_es.get_by_id(index=JOB_STATUS_INDEX, id=_id, ignore=404)
if info['found'] is False:
return {
'success': False,
'message': 'job info not found: %s' % _id
}, 404
return {
'success': True,
'message': "",
'result': info['_source']
}
def get(self):
"""Gets a Job Type specification object for the given ID."""
_id = request.form.get('id', request.args.get('id', None))
if _id is None:
return {'success': False, 'message': 'missing parameter: id'}, 400
job_spec = mozart_es.get_by_id(index=JOB_SPECS_INDEX,
id=_id,
ignore=404)
if job_spec['found'] is False:
app.logger.error('job_spec not found %s' % _id)
return {
'success': False,
'message': 'Failed to retrieve job_spec: %s' % _id
}, 404
return {
'success': True,
'message': "job spec for %s" % _id,
'result': job_spec['_source']
}
def put(self):
request_data = request.json or request.form
_id = request_data.get('id')
_index = request_data.get('index')
tag = request_data.get('tag')
app.logger.info('_id: %s\n _index: %s\n tag: %s' % (_id, _index, tag))
if _index != 'job_status-current':
app.logger.error('user tags only for index: job_status-current')
return {
'success': False,
'message': 'user tags only for index: job_status-current'
}, 400
if _id is None or _index is None or tag is None:
return {
'success': False,
'message': 'id, index and tag must be supplied'
}, 400
dataset = mozart_es.get_by_id(index=_index, id=_id, ignore=404)
if dataset['found'] is False:
return {'success': False, 'message': "dataset not found"}, 404
source = dataset['_source']
user_tags = source.get('user_tags', [])
app.logger.info('found user tags: %s' % str(user_tags))
if tag not in user_tags:
user_tags.append(tag)
app.logger.info('tags after adding: %s' % str(user_tags))
update_doc = {'doc_as_upsert': True, 'doc': {'user_tags': user_tags}}
mozart_es.update_document(index=_index,
id=_id,
body=update_doc,
refresh=True)
return {'success': True, 'tags': user_tags}
def get(self):
job_type = request.args.get('job_type')
if not job_type:
return {'success': False, 'message': 'job_type not provided'}, 400
query = {
"query": {
"term": {"job-specification.keyword": job_type}
}
}
hysds_io = mozart_es.search(index=HYSDS_IOS_INDEX, body=query)
if hysds_io['hits']['total']['value'] == 0:
return {
'success': False,
'message': '%s not found in hysds_ios' % job_type
}, 404
hysds_io = hysds_io['hits']['hits'][0]
job_params = hysds_io['_source']['params']
job_params = list(filter(lambda x: x['from'] == 'submitter', job_params))
job_spec = mozart_es.get_by_id(index=JOB_SPECS_INDEX, id=job_type, ignore=404)
if job_spec.get('found', False) is False:
return {
'success': False,
'message': '%s not found in job_specs' % job_type
}, 404
return {
'success': True,
'submission_type': hysds_io['_source'].get('submission_type'),
'hysds_io': hysds_io['_source']['id'],
'params': job_params,
'time_limit': job_spec['_source']['time_limit'],
'soft_time_limit': job_spec['_source']['soft_time_limit'],
'disk_usage': job_spec['_source']['disk_usage'],
'enable_dedup': hysds_io['_source'].get('enable_dedup', True)
}
def put(self):
"""edit existing user rule"""
request_data = request.json or request.form
_id = request_data.get("id", None)
_rule_name = request_data.get("rule_name", None)
if not _id and not _rule_name:
return {
"success": False,
"message": "Must specify id or rule_name in the request"
}, 400
user_rules_index = app.config['USER_RULES_INDEX']
rule_name = request_data.get('rule_name')
hysds_io = request_data.get('workflow')
job_spec = request_data.get('job_spec')
priority = request_data.get('priority')
query_string = request_data.get('query_string')
kwargs = request_data.get('kwargs')
queue = request_data.get('queue')
enabled = request_data.get('enabled')
tags = request_data.get('tags')
time_limit = request_data.get('time_limit', None)
soft_time_limit = request_data.get('soft_time_limit', None)
disk_usage = request_data.get('disk_usage', None)
enable_dedup = request_data.get('enable_dedup')
if enable_dedup is not None:
try:
enable_dedup = inputs.boolean(enable_dedup)
except ValueError as e:
return {'success': False, 'message': str(e)}, 400
# check if job_type (hysds_io) exists in elasticsearch (only if we're updating job_type)
if hysds_io:
job_type = mozart_es.get_by_id(index=HYSDS_IOS_INDEX,
id=hysds_io,
ignore=404)
if job_type.get("found", False) is False:
return {
'success': False,
'message': 'job_type not found: %s' % hysds_io
}, 404
if _id:
app.logger.info('finding existing user rule: %s' % _id)
existing_rule = mozart_es.get_by_id(index=user_rules_index,
id=_id,
ignore=404)
if existing_rule.get("found", False) is False:
app.logger.info('rule not found %s' % _id)
return {
'result': False,
'message': 'user rule not found: %s' % _id
}, 404
elif _rule_name:
app.logger.info('finding existing user rule: %s' % _rule_name)
result = mozart_es.search(index=user_rules_index,
q="rule_name:{}".format(_rule_name),
ignore=404)
if result.get("hits", {}).get("total", {}).get("value", 0) == 0:
return {
'success': False,
'message': 'rule %s not found' % _rule_name
}, 404
else:
_id = result.get("hits").get("hits")[0].get("_id")
update_doc = {}
if rule_name:
if len(rule_name) > 64:
return {
"success": False,
"message": "rule_name needs to be less than 64 characters",
"result": None,
}, 400
update_doc['rule_name'] = rule_name
if hysds_io:
update_doc['workflow'] = hysds_io
update_doc['job_type'] = hysds_io
if job_spec:
update_doc['job_spec'] = job_spec
if priority:
update_doc['priority'] = int(priority)
if query_string:
update_doc['query_string'] = query_string
try:
json.loads(query_string)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid elasticsearch query JSON'
}, 400
if kwargs:
update_doc['kwargs'] = kwargs
try:
json.loads(kwargs)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid JSON: kwargs'
}, 400
if queue:
update_doc['queue'] = queue
if enabled is not None:
if isinstance(enabled, str):
if enabled.lower() == "false":
value = False
else:
value = True
update_doc["enabled"] = value
else:
update_doc["enabled"] = enabled
if tags is not None:
if type(tags) == str:
tags = [tags]
update_doc['tags'] = tags
update_doc['modified_time'] = datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%SZ')
if 'time_limit' in request_data: # if submitted in editor
if time_limit is None:
update_doc['time_limit'] = None
else:
if isinstance(time_limit, int) and 0 < time_limit <= 86400 * 7:
update_doc['time_limit'] = time_limit
else:
return {
'success': False,
'message':
'time_limit must be between 0 and 604800 (sec)'
}, 400
if 'soft_time_limit' in request_data: # if submitted in editor
if soft_time_limit is None:
update_doc['soft_time_limit'] = None
else:
if isinstance(soft_time_limit,
int) and 0 < soft_time_limit <= 86400 * 7:
update_doc['soft_time_limit'] = soft_time_limit
else:
return {
'success': False,
'message':
'time_limit must be between 0 and 604800 (sec)'
}, 400
if 'disk_usage' in request_data:
update_doc['disk_usage'] = disk_usage
if 'enable_dedup' in request_data:
update_doc['enable_dedup'] = enable_dedup
app.logger.info('editing document id %s in user_rule index' % _id)
doc = {'doc_as_upsert': True, 'doc': update_doc}
result = mozart_es.update_document(index=user_rules_index,
id=_id,
body=doc,
refresh=True)
app.logger.info(result)
app.logger.info('document updated: %s' % _id)
return {'success': True, 'id': _id, 'updated': update_doc}
def post(self):
"""create new user rule"""
user_rules_index = app.config['USER_RULES_INDEX']
request_data = request.json or request.form
rule_name = request_data.get('rule_name')
hysds_io = request_data.get('workflow')
job_spec = request_data.get('job_spec')
priority = int(request_data.get('priority', 0))
query_string = request_data.get('query_string')
kwargs = request_data.get('kwargs', '{}')
queue = request_data.get('queue')
tags = request_data.get('tags', [])
time_limit = request_data.get('time_limit', None)
soft_time_limit = request_data.get('soft_time_limit', None)
disk_usage = request_data.get('disk_usage', None)
enable_dedup = request_data.get('enable_dedup')
if enable_dedup is not None:
try:
enable_dedup = inputs.boolean(enable_dedup)
except ValueError as e:
return {'success': False, 'message': str(e)}, 400
username = "******" # TODO: add user role and permissions, hard coded to "ops" for now
if not rule_name or not hysds_io or not job_spec or not query_string or not queue:
missing_params = []
if not rule_name:
missing_params.append('rule_name')
if not hysds_io:
missing_params.append('workflow')
if not job_spec:
missing_params.append('job_spec')
if not query_string:
missing_params.append('query_string')
if not queue:
missing_params.append('queue')
return {
'success': False,
'message':
'Params not specified: %s' % ', '.join(missing_params),
'result': None,
}, 400
if len(rule_name) > 64:
return {
"success": False,
"message": "rule_name needs to be less than 64 characters",
"result": None,
}, 400
try:
json.loads(query_string)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid elasticsearch query JSON'
}, 400
try:
json.loads(kwargs)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {'success': False, 'message': 'invalid JSON: kwargs'}, 400
# check if rule name already exists
rule_exists_query = {"query": {"term": {"rule_name": rule_name}}}
existing_rules_count = mozart_es.get_count(index=user_rules_index,
body=rule_exists_query)
if existing_rules_count > 0:
return {
'success': False,
'message': 'user rule already exists: %s' % rule_name
}, 409
# check if job_type (hysds_io) exists in elasticsearch
job_type = mozart_es.get_by_id(index=HYSDS_IOS_INDEX,
id=hysds_io,
ignore=404)
if job_type['found'] is False:
return {
'success': False,
'message': '%s not found' % hysds_io
}, 400
params = job_type['_source']['params']
is_passthrough_query = check_passthrough_query(params)
if type(tags) == str:
tags = [tags]
now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
new_doc = {
"workflow": hysds_io,
"job_spec": job_spec,
"priority": priority,
"rule_name": rule_name,
"username": username,
"query_string": query_string,
"kwargs": kwargs,
"job_type": hysds_io,
"enabled": True,
"passthru_query": is_passthrough_query,
"query_all": False,
"queue": queue,
"modified_time": now,
"creation_time": now,
"tags": tags
}
if time_limit and isinstance(time_limit, int):
if time_limit <= 0 or time_limit > 86400 * 7:
return {
'success': False,
'message': 'time_limit must be between 0 and 604800 (sec)'
}, 400
else:
new_doc['time_limit'] = time_limit
if soft_time_limit and isinstance(soft_time_limit, int):
if soft_time_limit <= 0 or soft_time_limit > 86400 * 7:
return {
'success': False,
'message':
'soft_time_limit must be between 0 and 604800 (sec)'
}, 400
else:
new_doc['soft_time_limit'] = soft_time_limit
if disk_usage:
new_doc['disk_usage'] = disk_usage
if enable_dedup is not None:
new_doc['enable_dedup'] = enable_dedup
result = mozart_es.index_document(index=user_rules_index,
body=new_doc,
refresh=True)
return {'success': True, 'message': 'rule created', 'result': result}
def post(self):
"""
submits on demand job
:return: submit job id
"""
request_data = request.json
if not request_data:
request_data = request.form
tag = request_data.get('tags')
job_type = request_data.get('job_type')
hysds_io = request_data.get('hysds_io')
queue = request_data.get('queue')
priority = int(request_data.get('priority', 0))
query_string = request_data.get('query')
kwargs = request_data.get('kwargs', '{}')
time_limit = request_data.get('time_limit')
soft_time_limit = request_data.get('soft_time_limit')
disk_usage = request_data.get('disk_usage')
enable_dedup = request_data.get('enable_dedup')
if enable_dedup is not None:
try:
enable_dedup = inputs.boolean(enable_dedup)
except ValueError as e:
return {
'success': False,
'message': str(e)
}, 400
try:
query = json.loads(query_string)
query_string = json.dumps(query)
except (ValueError, TypeError, Exception) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid JSON query'
}, 400
if tag is None or job_type is None or hysds_io is None or queue is None or query_string is None:
return {
'success': False,
'message': 'missing field: [tags, job_type, hysds_io, queue, query]'
}, 400
doc = mozart_es.get_by_id(index=HYSDS_IOS_INDEX, id=hysds_io, ignore=404)
if doc['found'] is False:
app.logger.error('failed to fetch %s, not found in hysds_ios' % hysds_io)
return {
'success': False,
'message': '%s not found' % hysds_io
}, 404
params = doc['_source']['params']
is_passthrough_query = check_passthrough_query(params)
rule = {
'username': '******',
'workflow': hysds_io,
'priority': priority,
'enabled': True,
'job_type': job_type,
'rule_name': tag,
'kwargs': kwargs,
'query_string': query_string,
'query': query,
'passthru_query': is_passthrough_query,
'query_all': False,
'queue': queue
}
if time_limit and isinstance(time_limit, int):
if time_limit <= 0 or time_limit > 86400 * 7:
return {
'success': False,
'message': 'time_limit must be between 0 and 604800 (sec)'
}, 400
else:
rule['time_limit'] = time_limit
if soft_time_limit and isinstance(soft_time_limit, int):
if soft_time_limit <= 0 or soft_time_limit > 86400 * 7:
return {
'success': False,
'message': 'soft_time_limit must be between 0 and 604800 (sec)'
}, 400
else:
rule['soft_time_limit'] = soft_time_limit
if disk_usage:
rule['disk_usage'] = disk_usage
if enable_dedup is not None:
rule['enable_dedup'] = enable_dedup
payload = {
'type': 'job_iterator',
'function': 'hysds_commons.job_iterator.iterate',
'args': ["figaro", rule],
}
on_demand_job_queue = celery_app.conf['ON_DEMAND_JOB_QUEUE']
celery_task = do_submit_task(payload, on_demand_job_queue)
return {
'success': True,
'message': 'task submitted successfully',
'result': celery_task.id
}