def handover_database(spec):
""" Method to accept a new database for incorporation into the system
Argument is a dict with the following keys:
* src_uri - URI to database to handover (required)
* tgt_uri - URI to copy database to (optional - generated from staging and src_uri if not set)
* contact - email address of submitter (required)
* type - string describing type of update (required)
* comment - additional information about submission (required)
The following keys are added during the handover process:
* handover_token - unique identifier for this particular handover invocation
* hc_job_id - job ID for healthcheck process
* db_job_id - job ID for database copy process
"""
# TODO verify dict
reporting.set_logger_context(get_logger(), spec['src_uri'], spec)
get_logger().info("Handling " + str(spec))
if 'tgt_uri' not in spec:
spec['tgt_uri'] = get_tgt_uri(spec['src_uri'])
# create unique identifier
spec['handover_token'] = str(uuid.uuid1())
check_db(spec['src_uri'])
groups = groups_for_uri(spec['src_uri'])
if (groups == None):
get_logger().info("No HCs needed, starting copy")
submit_copy(spec)
else:
get_logger().info("Starting HCs")
submit_hc(spec, groups)
return spec['handover_token']
def process_copied_db(self, copy_job_id, spec):
"""Wait for copy to complete and then respond accordingly:
* if success, submit to metadata database
* if failure, flag error using email"""
reporting.set_logger_context(get_logger(), spec['src_uri'], spec)
# allow infinite retries
self.max_retries = None
get_logger().info("Checking " + str(spec) + " using " + str(copy_job_id))
result = db_copy_client.retrieve_job(copy_job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().info("Job incomplete, retrying")
raise self.retry()
if (result['status'] == 'failed'):
get_logger().info("Copy failed")
msg = """
Copying %s to %s failed.
Please see %s
""" % (spec['src_uri'], spec['tgt_uri'], cfg.copy_web_uri + str(copy_job_id))
send_email(to_address=spec['contact'],
subject='Database copy failed',
body=msg,
smtp_server=cfg.smtp_server)
return
else:
get_logger().info("Copying complete, need to submit metadata")
metadata_job_id = submit_metadata_update(spec)
task_id = process_db_metadata.delay(metadata_job_id, spec)
get_logger().debug("Submitted DB for meta update as " + str(task_id))
return task_id
def process_db_metadata(self, metadata_job_id, spec):
"""Wait for metadata update to complete and then respond accordingly:
* if success, submit event to event handler for further processing
* if failure, flag error using email"""
reporting.set_logger_context(get_logger(), spec['tgt_uri'], spec)
# allow infinite retries
self.max_retries = None
get_logger().info("Checking " + str(spec) + " using " +
str(metadata_job_id))
result = metadata_client.retrieve_job(metadata_job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().info("Job incomplete, retrying")
raise self.retry()
if (result['status'] == 'failed'):
get_logger().info("Metadata load failed")
msg = """
Metadata load of %s failed.
Please see %s
""" % (spec['tgt_uri'], cfg.meta_web_uri + str(metadata_job_id))
send_email(to_address=spec['contact'],
subject='Metadata load failed',
body=msg,
smtp_server=cfg.smtp_server)
return
else:
get_logger().info(
"Metadata load complete, retrieving event from metadata database")
# TODO wait for job to complete
# TODO retrieve event from metadata job
# TODO pass event to event handler endpoint to trigger more processing
return
def process_result(self, event, process, job_id):
"""
Wait for the completion of the job and then process any output further
"""
reporting.set_logger_context(get_logger(), event['genome'], event)
# allow infinite retries
self.max_retries = None
get_logger().info("Checking {} event {}".format(process, job_id))
result = event_client.retrieve_job(process, job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().info("Job incomplete, retrying")
raise self.retry()
get_logger().debug("Handling result for " + json.dumps(event))
if result['status'] == 'failure':
get_logger().fatal("Event failed: " + json.dumps(result))
else:
get_logger().info("Event succeeded: " + json.dumps(result))
# TODO
# 1. update metadata
# 2. schedule new events as required
return event['event_id']
def handover_database(spec):
""" Method to accept a new database for incorporation into the system
Argument is a dict with the following keys:
* src_uri - URI to database to handover (required)
* tgt_uri - URI to copy database to (optional - generated from staging and src_uri if not set)
* contact - email address of submitter (required)
* type - string describing type of update (required)
* comment - additional information about submission (required)
The following keys are added during the handover process:
* handover_token - unique identifier for this particular handover invocation
* hc_job_id - job ID for healthcheck process
* db_job_id - job ID for database copy process
* metadata_job_id - job ID for the metadata loading process
* progress_total - Total number of task to do
* progress_complete - Total number of task completed
"""
# TODO verify dict
reporting.set_logger_context(get_logger(), spec['src_uri'], spec)
# create unique identifier
spec['handover_token'] = str(uuid.uuid1())
if 'tgt_uri' not in spec:
spec['tgt_uri'] = get_tgt_uri(spec['src_uri'])
get_logger().info("Handling " + str(spec))
check_db(spec['src_uri'])
(groups, compara_uri, production_uri, staging_uri,
live_uri) = groups_for_uri(spec['src_uri'])
if (compara_uri == None):
compara_uri = cfg.compara_uri + 'ensembl_compara_master'
spec['progress_total'] = 3
spec['progress_complete'] = 1
submit_hc(spec, groups, compara_uri, production_uri, staging_uri, live_uri)
return spec['handover_token']
def process_copied_db(self, copy_job_id, spec):
"""Wait for copy to complete and then respond accordingly:
* if success, submit to metadata database
* if failure, flag error using email"""
reporting.set_logger_context(get_logger(), spec['src_uri'], spec)
# allow infinite retries
self.max_retries = None
get_logger().info("Copying in progress, please see: " + cfg.copy_web_uri +
str(copy_job_id))
result = db_copy_client.retrieve_job(copy_job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().debug(
"Database copy job incomplete, checking again later")
raise self.retry()
if (result['status'] == 'failed'):
get_logger().info("Copy failed, please see: " + cfg.copy_web_uri +
str(copy_job_id))
msg = """
Copying %s to %s failed.
Please see %s
""" % (spec['src_uri'], spec['tgt_uri'], cfg.copy_web_uri + str(copy_job_id))
send_email(to_address=spec['contact'],
subject='Database copy failed',
body=msg,
smtp_server=cfg.smtp_server)
return
else:
get_logger().info("Copying complete, submitting metadata job")
spec['progress_complete'] = 3
submit_metadata_update(spec)
def process_checked_db(self, hc_job_id, spec):
""" Task to wait until HCs finish and then respond e.g.
* submit copy if HCs succeed
* send error email if not
"""
reporting.set_logger_context(get_logger(), spec['src_uri'], spec)
# allow infinite retries
self.max_retries = None
get_logger().info("HCs in progress, please see: " + cfg.hc_web_uri +
str(hc_job_id))
result = hc_client.retrieve_job(hc_job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().debug("HC Job incomplete, checking again later")
raise self.retry()
# check results
if (result['status'] == 'failed'):
get_logger().info("HCs failed to run, please see: " + cfg.hc_web_uri +
str(hc_job_id))
msg = """
Running healthchecks on %s failed to execute.
Please see %s
""" % (spec['src_uri'], cfg.hc_web_uri + str(hc_job_id))
send_email(to_address=spec['contact'],
subject='HC failed to run',
body=msg,
smtp_server=cfg.smtp_server)
return
elif (result['output']['status'] == 'failed'):
get_logger().info("HCs found problems, please see: " + cfg.hc_web_uri +
str(hc_job_id))
msg = """
Running healthchecks on %s completed but found failures.
Please see %s
""" % (spec['src_uri'], cfg.hc_web_uri + str(hc_job_id))
send_email(to_address=spec['contact'],
subject='HC ran but failed',
body=msg,
smtp_server=cfg.smtp_server)
return
else:
get_logger().info("HCs fine, starting copy")
spec['progress_complete'] = 2
submit_copy(spec)
def process_db_metadata(self, metadata_job_id, spec):
"""Wait for metadata update to complete and then respond accordingly:
* if success, submit event to event handler for further processing
* if failure, flag error using email"""
reporting.set_logger_context(get_logger(), spec['tgt_uri'], spec)
# allow infinite retries
self.max_retries = None
get_logger().info("Loading into metadata database, please see: " +
cfg.meta_uri + "jobs/" + str(metadata_job_id))
result = metadata_client.retrieve_job(metadata_job_id)
if (result['status']
== 'incomplete') or (result['status']
== 'running') or (result['status']
== 'submitted'):
get_logger().debug(
"Metadata load Job incomplete, checking again later")
raise self.retry()
if (result['status'] == 'failed'):
get_logger().info("Metadata load failed, please see " + cfg.meta_uri +
'jobs/' + str(metadata_job_id) + '?format=failures')
msg = """
Metadata load of %s failed.
Please see %s
""" % (spec['tgt_uri'],
cfg.meta_uri + 'jobs/' + str(metadata_job_id) + '?format=failures')
send_email(to_address=spec['contact'],
subject='Metadata load failed, please see: ' +
cfg.meta_uri + 'jobs/' + str(metadata_job_id) +
'?format=failures',
body=msg,
smtp_server=cfg.smtp_server)
return
else:
#get_logger().info("Metadata load complete, submitting event")
get_logger().info("Metadata load complete, Handover successful")
#submit_event(spec,result)
return