def get(self):
"""retrieve user defined tags for trigger rules"""
index = app.config['USER_RULES_INDEX']
body = {
"size": 0,
"aggs": {
"my_buckets": {
"composite": {
"size": 1000,
"sources": [{
"tags": {
"terms": {
"field": "tags"
}
}
}]
}
}
}
}
results = mozart_es.search(index=index, body=body)
buckets = results['aggregations']['my_buckets']['buckets']
buckets = sorted(buckets, key=lambda k: k['doc_count'], reverse=True)
app.logger.info(buckets)
return {
'success':
True,
'tags': [{
'key': tag['key']['tags'],
'count': tag['doc_count']
} for tag in buckets]
}
def get(self):
"""Retrieve user rule(s)"""
_id = request.args.get("id", None)
_rule_name = request.args.get("rule_name", None)
user_rules_index = app.config['USER_RULES_INDEX']
if _id:
rule = mozart_es.get_by_id(index=user_rules_index, id=_id, ignore=404)
if rule.get("found", False) is False:
return {
'success': False,
'message': rule['message']
}, 404
else:
rule = {**rule, **rule["_source"]}
rule.pop("_source", None)
return {
'success': True,
'rule': rule
}
elif _rule_name:
result = mozart_es.search(index=user_rules_index, q="rule_name:{}".format(_rule_name), ignore=404)
if result.get("hits", {}).get("total", {}).get("value", 0) == 0:
return {
"success": False,
"message": "rule {} not found".format(_rule_name)
}, 404
rule = result.get("hits").get("hits")[0]
rule = {**rule, **rule["_source"]}
rule.pop("_source", None)
return {
"success": True,
"rule": rule
}
user_rules = mozart_es.query(index=user_rules_index)
parsed_user_rules = []
for rule in user_rules:
rule_copy = rule.copy()
rule_temp = {**rule_copy, **rule['_source']}
rule_temp.pop('_source')
parsed_user_rules.append(rule_temp)
return {
'success': True,
'rules': parsed_user_rules
}
def get(self):
job_type = request.args.get('job_type')
if not job_type:
return {'success': False, 'message': 'job_type not provided'}, 400
query = {
"query": {
"term": {"job-specification.keyword": job_type}
}
}
hysds_io = mozart_es.search(index=HYSDS_IOS_INDEX, body=query)
if hysds_io['hits']['total']['value'] == 0:
return {
'success': False,
'message': '%s not found in hysds_ios' % job_type
}, 404
hysds_io = hysds_io['hits']['hits'][0]
job_params = hysds_io['_source']['params']
job_params = list(filter(lambda x: x['from'] == 'submitter', job_params))
job_spec = mozart_es.get_by_id(index=JOB_SPECS_INDEX, id=job_type, ignore=404)
if job_spec.get('found', False) is False:
return {
'success': False,
'message': '%s not found in job_specs' % job_type
}, 404
return {
'success': True,
'submission_type': hysds_io['_source'].get('submission_type'),
'hysds_io': hysds_io['_source']['id'],
'params': job_params,
'time_limit': job_spec['_source']['time_limit'],
'soft_time_limit': job_spec['_source']['soft_time_limit'],
'disk_usage': job_spec['_source']['disk_usage'],
'enable_dedup': hysds_io['_source'].get('enable_dedup', True)
}
def put(self):
"""edit existing user rule"""
request_data = request.json or request.form
_id = request_data.get("id", None)
_rule_name = request_data.get("rule_name", None)
if not _id and not _rule_name:
return {
"success": False,
"message": "Must specify id or rule_name in the request"
}, 400
user_rules_index = app.config['USER_RULES_INDEX']
rule_name = request_data.get('rule_name')
hysds_io = request_data.get('workflow')
job_spec = request_data.get('job_spec')
priority = request_data.get('priority')
query_string = request_data.get('query_string')
kwargs = request_data.get('kwargs')
queue = request_data.get('queue')
enabled = request_data.get('enabled')
tags = request_data.get('tags')
time_limit = request_data.get('time_limit', None)
soft_time_limit = request_data.get('soft_time_limit', None)
disk_usage = request_data.get('disk_usage', None)
enable_dedup = request_data.get('enable_dedup')
if enable_dedup is not None:
try:
enable_dedup = inputs.boolean(enable_dedup)
except ValueError as e:
return {'success': False, 'message': str(e)}, 400
# check if job_type (hysds_io) exists in elasticsearch (only if we're updating job_type)
if hysds_io:
job_type = mozart_es.get_by_id(index=HYSDS_IOS_INDEX,
id=hysds_io,
ignore=404)
if job_type.get("found", False) is False:
return {
'success': False,
'message': 'job_type not found: %s' % hysds_io
}, 404
if _id:
app.logger.info('finding existing user rule: %s' % _id)
existing_rule = mozart_es.get_by_id(index=user_rules_index,
id=_id,
ignore=404)
if existing_rule.get("found", False) is False:
app.logger.info('rule not found %s' % _id)
return {
'result': False,
'message': 'user rule not found: %s' % _id
}, 404
elif _rule_name:
app.logger.info('finding existing user rule: %s' % _rule_name)
result = mozart_es.search(index=user_rules_index,
q="rule_name:{}".format(_rule_name),
ignore=404)
if result.get("hits", {}).get("total", {}).get("value", 0) == 0:
return {
'success': False,
'message': 'rule %s not found' % _rule_name
}, 404
else:
_id = result.get("hits").get("hits")[0].get("_id")
update_doc = {}
if rule_name:
if len(rule_name) > 64:
return {
"success": False,
"message": "rule_name needs to be less than 64 characters",
"result": None,
}, 400
update_doc['rule_name'] = rule_name
if hysds_io:
update_doc['workflow'] = hysds_io
update_doc['job_type'] = hysds_io
if job_spec:
update_doc['job_spec'] = job_spec
if priority:
update_doc['priority'] = int(priority)
if query_string:
update_doc['query_string'] = query_string
try:
json.loads(query_string)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid elasticsearch query JSON'
}, 400
if kwargs:
update_doc['kwargs'] = kwargs
try:
json.loads(kwargs)
except (ValueError, TypeError) as e:
app.logger.error(e)
return {
'success': False,
'message': 'invalid JSON: kwargs'
}, 400
if queue:
update_doc['queue'] = queue
if enabled is not None:
if isinstance(enabled, str):
if enabled.lower() == "false":
value = False
else:
value = True
update_doc["enabled"] = value
else:
update_doc["enabled"] = enabled
if tags is not None:
if type(tags) == str:
tags = [tags]
update_doc['tags'] = tags
update_doc['modified_time'] = datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%SZ')
if 'time_limit' in request_data: # if submitted in editor
if time_limit is None:
update_doc['time_limit'] = None
else:
if isinstance(time_limit, int) and 0 < time_limit <= 86400 * 7:
update_doc['time_limit'] = time_limit
else:
return {
'success': False,
'message':
'time_limit must be between 0 and 604800 (sec)'
}, 400
if 'soft_time_limit' in request_data: # if submitted in editor
if soft_time_limit is None:
update_doc['soft_time_limit'] = None
else:
if isinstance(soft_time_limit,
int) and 0 < soft_time_limit <= 86400 * 7:
update_doc['soft_time_limit'] = soft_time_limit
else:
return {
'success': False,
'message':
'time_limit must be between 0 and 604800 (sec)'
}, 400
if 'disk_usage' in request_data:
update_doc['disk_usage'] = disk_usage
if 'enable_dedup' in request_data:
update_doc['enable_dedup'] = enable_dedup
app.logger.info('editing document id %s in user_rule index' % _id)
doc = {'doc_as_upsert': True, 'doc': update_doc}
result = mozart_es.update_document(index=user_rules_index,
id=_id,
body=doc,
refresh=True)
app.logger.info(result)
app.logger.info('document updated: %s' % _id)
return {'success': True, 'id': _id, 'updated': update_doc}
def get(self, user):
"""
return user submitted jobs from ElasticSearch (sorted by @timestamp desc)
"""
offset = request.args.get('offset')
page_size = request.args.get('page_size')
job_type = request.args.get('job_type')
tag = request.args.get('tag')
queue = request.args.get('queue')
priority = request.args.get('priority')
start_time = request.args.get('start_time')
end_time = request.args.get('end_time')
status = request.args.get('status')
if offset:
try:
offset = int(offset)
except (ValueError, TypeError):
return {'success': False, 'message': 'offset must be an int'}, 400
if page_size:
try:
page_size = int(page_size)
if page_size > 250:
page_size = 250
except (ValueError, TypeError):
return {'success': False, 'message': 'page_size must be an int'}, 400
if priority:
try:
priority = int(priority)
if priority > 9:
priority = 9
except (ValueError, TypeError):
return {'success': False, 'message': 'priority must be an int'}, 400
query = {
"sort": [
{"@timestamp": {"order": "desc"}}
],
"query": {
"bool": {
"must": [
{
"term": {
"job.username": user
}
}
]
}
}
}
if offset:
query['from'] = offset
if page_size:
query['size'] = page_size
if tag:
query['query']['bool']['must'].append({"term": {"tags.keyword": tag}})
if job_type:
query['query']['bool']['must'].append({"term": {"job.type": job_type}})
if priority:
query['query']['bool']['must'].append({"term": {"job.priority": priority}})
if status:
query['query']['bool']['must'].append({"term": {"status": status}})
if queue:
query['query']['bool']['must'].append({"term": {"job.job_info.job_queue": queue}})
if start_time is not None or end_time is not None:
datetime_filter = {'range': {'@timestamp': {}}}
if start_time:
if start_time.isdigit():
start_time = int(start_time)
datetime_filter['range']['@timestamp']['gte'] = start_time
if end_time:
if end_time.isdigit():
end_time = int(end_time)
datetime_filter['range']['@timestamp']['lte'] = end_time
query['query']['bool']['must'].append(datetime_filter)
try:
res = mozart_es.search(index=JOB_STATUS_INDEX, body=query, _source=['tags'])
except Exception as e:
return {'success': False, 'message': str(e), 'result': []}, 400
return {
'success': True,
'result': [{
'id': doc['_id'],
'tags': doc['_source']['tags']
} for doc in res['hits']['hits']]
}