def submit_job():
if request.is_json:
request.json["metadata_uri"] = app.config["METADATA_URI"]
app.logger.debug('Submitting metadata job %s', request.json)
try:
analysis = app.config["HIVE_ANALYSIS"]
job = get_hive().create_job(analysis, request.json)
except ValueError as e:
raise HTTPRequestError(str(e), 404)
results = {"job_id": job.job_id}
return jsonify(results), 201
else:
error_msg = 'Could not handle input of type %s', request.headers['Content-Type']
app.logger.error(error_msg)
raise HTTPRequestError(error_msg)
def dropdown(src_host=None, src_port=None):
try:
src_name = request.args.get('name', None)
search = request.args.get('search', None)
if src_name:
res = requests.get(f"{cfg.copy_uri_dropdown}api/dbcopy/srchost",
params={'name': src_name})
res.raise_for_status()
return jsonify(res.json())
elif src_host and src_port and search:
res = requests.get(
f"{cfg.copy_uri_dropdown}api/dbcopy/databases/{src_host}/{src_port}",
params={'search': search})
res.raise_for_status()
return jsonify(res.json())
else:
raise Exception('required params not provided')
except HTTPError as http_err:
raise HTTPRequestError(f'{http_err}', 404)
except Exception as e:
logging.fatal(str(e))
return jsonify({
"count": 0,
"next": None,
"previous": None,
"results": [],
"error": str(e)
})
def delete_job(job_id):
try:
job = get_hive().get_job_by_id(job_id)
get_hive().delete_job(job, child=True)
except ValueError as e:
raise HTTPRequestError(str(e), 404)
return jsonify({"id": job_id})
def failure(job_id):
app.logger.info('Retrieving failure for job with ID %s', job_id)
try:
job_failure = get_hive().get_job_failure_msg_by_id(job_id, child=True)
except ValueError as e:
raise HTTPRequestError(str(e), 404)
return jsonify({"msg": job_failure.msg})
def job_result(job_id):
fmt = request.args.get('format')
app.logger.debug('Format %s', fmt)
if fmt == 'email':
email = request.args.get('email')
return job_email(email, job_id)
elif fmt == 'failures':
return failure(job_id)
elif fmt is None:
app.logger.info('Retrieving job with ID %s', job_id)
try:
result = get_hive().get_result_for_job_id(job_id, child=True)
except ValueError as e:
raise HTTPRequestError(str(e), 404)
return jsonify(result)
else:
raise HTTPRequestError("Format " + fmt + " not valid")
def dropdown(src_host=None, src_port=None):
try:
search = request.args.get('search', None)
if src_host and src_port and search:
res = requests.get(
f"{DatacheckConfig.COPY_URI_DROPDOWN}api/dbcopy/databases/{src_host}/{src_port}",
params={'search': search})
res.raise_for_status()
return jsonify(res.json())
else:
raise Exception('required params not provided')
except HTTPError as http_err:
raise HTTPRequestError(f'{http_err}', 404)
except Exception as e:
print(str(e))
return jsonify([])
def job_email(email, job_id):
app.logger.info('Retrieving job with ID %s for %s', job_id, email)
try:
results = get_hive().get_result_for_job_id(job_id, child=True)
if results['status'] == 'complete':
results['subject'] = 'Metadata load for database %s is successful' % (results['output']['database_uri'])
results['body'] = "Metadata load for database %s is successful\n" % (results['output']['database_uri'])
results['body'] += "Load took %s" % (results['output']['runtime'])
elif results['status'] == 'failed':
job_failure = get_hive().get_job_failure_msg_by_id(job_id, child=True)
results['subject'] = 'Metadata load for %s failed' % (results['input']['database_uri'])
results['body'] = 'Metadata load failed with following message:\n'
results['body'] += '%s' % job_failure.msg
except ValueError as e:
raise HTTPRequestError(str(e), 404)
results['output'] = None
return jsonify(results)
def stop_handover(handover_token=None):
"""
Endpoint to delete all the reports linked to a handover_token
This is using docstring for specifications
---
tags:
- handovers
parameters:
- name: handover_token
in: path
type: string
required: true
default: 15ce20fd-68cd-11e8-8117-005056ab00f0
description: handover token for the database handed over
operationId: handovers
consumes:
- application/json
produces:
- application/json
security:
delete_auth:
- 'write:delete'
- 'read:delete'
schemes: ['http', 'https']
deprecated: false
externalDocs:
description: Project repository
url: http://github.com/rochacbruno/flasgger
definitions:
handover_token:
type: object
properties:
handover_token:
type: integer
items:
$ref: '#/definitions/handover_token'
id:
type: integer
properties:
id:
type: integer
items:
$ref: '#/definitions/id'
responses:
200:
description: handover_token of the reports that need deleting
schema:
$ref: '#/definitions/handover_token'
examples:
id: 15ce20fd-68cd-11e8-8117-005056ab00f0
"""
try:
handover_token = request.args.get('handover_token', handover_token)
if handover_token is None:
raise ValueError('required handover_token')
status = stop_handover_job(handover_token)
return status
except NotFoundError as e:
raise HTTPRequestError(
'Error while looking for handover token: {} - {}:{}'.format(
handover_token, e.error, e.info['error']['reason']), 404)
def delete_handover(handover_token):
"""
Endpoint to delete all the reports linked to a handover_token
This is using docstring for specifications
---
tags:
- handovers
parameters:
- name: handover_token
in: path
type: string
required: true
default: 15ce20fd-68cd-11e8-8117-005056ab00f0
description: handover token for the database handed over
operationId: handovers
consumes:
- application/json
produces:
- application/json
security:
delete_auth:
- 'write:delete'
- 'read:delete'
schemes: ['http', 'https']
deprecated: false
externalDocs:
description: Project repository
url: http://github.com/rochacbruno/flasgger
definitions:
handover_token:
type: object
properties:
handover_token:
type: integer
items:
$ref: '#/definitions/handover_token'
id:
type: integer
properties:
id:
type: integer
items:
$ref: '#/definitions/id'
responses:
200:
description: handover_token of the reports that need deleting
schema:
$ref: '#/definitions/handover_token'
examples:
id: 15ce20fd-68cd-11e8-8117-005056ab00f0
"""
try:
app.logger.info('Retrieving handover data with token %s',
handover_token)
es = Elasticsearch([{'host': es_host, 'port': es_port}])
es.delete_by_query(index=es_index,
doc_type='report',
body={
"query": {
"bool": {
"must": [{
"term": {
"params.handover_token.keyword":
str(handover_token)
}
}]
}
}
})
return jsonify(str(handover_token))
except NotFoundError as e:
raise HTTPRequestError(
'Error while looking for handover token: {} - {}:{}'.format(
handover_token, e.error, e.info['error']['reason']), 404)
def handover_result(handover_token=''):
"""
Endpoint to get an handover job detail
This is using docstring for specifications
---
tags:
- handovers
parameters:
- name: handover_token
in: path
type: string
required: true
default: 15ce20fd-68cd-11e8-8117-005056ab00f0
description: handover token for the database handed over
operationId: handovers
consumes:
- application/json
produces:
- application/json
security:
handovers_auth:
- 'write:handovers'
- 'read:handovers'
schemes: ['http', 'https']
deprecated: false
externalDocs:
description: Project repository
url: http://github.com/rochacbruno/flasgger
definitions:
handovers:
title: Get a handover job details
description: This will retrieve a handover job details
type: object
required:
-handover_token
properties:
handover_token:
type: string
example: '15ce20fd-68cd-11e8-8117-005056ab00f0'
responses:
200:
description: Retrieve an handover job ticket
schema:
$ref: '#/definitions/jobs'
examples:
[{"comment": "handover new Tiger database", "contact": "*****@*****.**", "handover_token": "605f1191-7a13-11e8-aa7e-005056ab00f0", "id": "X1qcQWQBiZ0vMed2vaAt", "message": "Metadata load complete, Handover successful", "progress_total": 3, "report_time": "2018-06-27T15:19:08.459", "src_uri": "mysql://ensro@mysql-ens-general-prod-1:4525/panthera_tigris_altaica_core_93_1", "tgt_uri": "mysql://ensro@mysql-ens-general-dev-1:4484/panthera_tigris_altaica_core_93_1"} ]
"""
fmt = request.args.get('format', None)
# TODO Move this into core (potential usage on every flask app)
# renter bootstrap table
app.logger.info("Request Headers %s", request.headers)
if fmt != 'json' and not request.is_json:
return render_template('result.html', handover_token=handover_token)
es = Elasticsearch([{'host': es_host, 'port': es_port}])
handover_detail = []
res = es.search(index=es_index,
body={
"size": 0,
"query": {
"bool": {
"must": [
{
"term": {
"params.handover_token.keyword":
str(handover_token)
}
},
{
"query_string": {
"fields": ["report_type"],
"query": "(INFO|ERROR)",
"analyze_wildcard": "true"
}
},
]
}
},
"aggs": {
"top_result": {
"top_hits": {
"size": 1,
"sort": {
"report_time": "desc"
}
}
}
},
"sort": [{
"report_time": {
"order": "desc"
}
}]
})
for doc in res['aggregations']['top_result']['hits']['hits']:
result = {"id": doc['_id']}
if 'job_progress' in doc['_source']['params']:
result['job_progress'] = doc['_source']['params']['job_progress']
result['message'] = doc['_source']['message']
result['comment'] = doc['_source']['params']['comment']
result['handover_token'] = doc['_source']['params']['handover_token']
result['contact'] = doc['_source']['params']['contact']
result['src_uri'] = doc['_source']['params']['src_uri']
result['tgt_uri'] = doc['_source']['params']['tgt_uri']
result['progress_complete'] = doc['_source']['params'][
'progress_complete']
result['progress_total'] = doc['_source']['params']['progress_total']
result['report_time'] = doc['_source']['report_time']
handover_detail.append(result)
if len(handover_detail) == 0:
raise HTTPRequestError('Handover token %s not found' % handover_token,
404)
else:
return jsonify(handover_detail)
def handover_status_update():
"""update handover status to success"""
try:
if json_pattern.match(request.headers['Content-Type']):
handover_token = request.json.get('handover_token')
else:
raise HTTPRequestError('Could not handle input of type %s' %
request.headers['Content-Type'])
es = Elasticsearch([{'host': es_host, 'port': es_port}])
res_error = es.search(
index=es_index,
body={
"query": {
"bool": {
"must": [{
"term": {
"params.handover_token.keyword":
str(handover_token)
}
}, {
"term": {
"report_type.keyword": "INFO"
}
}, {
"query_string": {
"fields": ["message"],
"query": "*Metadata load failed*"
}
}],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 1,
"sort": [{
"report_time": {
"order": "desc"
}
}],
"aggs": {}
})
if len(res_error['hits']['hits']) == 0:
raise HTTPRequestError('No Hits Found for Handover Token : %s' %
handover_token)
# set handover message to success
result = res_error['hits']['hits'][0]['_source']
h_id = res_error['hits']['hits'][0]['_id']
result['report_time'] = str(datetime.datetime.now().isoformat())[:-3]
result['message'] = 'Metadata load complete, Handover successful'
result['report_type'] = 'INFO'
res = es.update(index=es_index,
id=h_id,
doc_type='report',
body={"doc": result})
except Exception as e:
raise HTTPRequestError('%s' % str(e))
return res
def handovers():
"""
Endpoint to submit an handover job
This is using docstring for specifications
---
tags:
- handovers
parameters:
- in: body
name: body
description: DC object
required: false
schema:
$ref: '#/definitions/jobs'
operationId: handovers
consumes:
- application/json
produces:
- application/json
security:
handovers_auth:
- 'write:jobs'
- 'read:jobs'
schemes: ['http', 'https']
deprecated: false
externalDocs:
description: Project repository
url: http://github.com/rochacbruno/flasgger
definitions:
handovers:
title: handover job
description: A job to handover a database, the database will be healthchecked, copied and added to metadata database
type: object
required:
-src_uri
-contact
-type
-comment
properties:
src_uri:
type: string
example: 'mysql://user@server:port/saccharomyces_cerevisiae_core_91_4'
comment:
type: string
example: 'handover new Panda OF'
contact:
type: string
example: '*****@*****.**'
responses:
200:
description: submit of an handover job
schema:
$ref: '#/definitions/jobs'
examples:
{src_uri: "mysql://user@server:port/saccharomyces_cerevisiae_core_91_4", contact: "*****@*****.**", comment: "handover new Panda OF"}
"""
try:
if not cfg.compara_species:
# Empty list of compara
raise MissingDispatchException
# get form data
if form_pattern.match(request.headers['Content-Type']):
spec = request.form.to_dict(flat=True)
elif json_pattern.match(request.headers['Content-Type']):
spec = request.json
else:
raise HTTPRequestError('Could not handle input of type %s' %
request.headers['Content-Type'])
if 'src_uri' not in spec or 'contact' not in spec or 'comment' not in spec:
raise HTTPRequestError(
"Handover specification incomplete - please specify src_uri, contact and comment"
)
app.logger.debug('Submitting handover request %s', spec)
ticket = handover_database(spec)
app.logger.info('Ticket: %s', ticket)
except Exception as e:
raise HTTPRequestError(str(e), 400)
return jsonify(ticket)