def _update_status_db(self, status, msg):
"""
Update workflow status in DB.
Args:
self: class instance
status: Workflow status
msg: Success, error or warning message
Returns:
On success: True.
On failure: False.
"""
try:
data_source = DataSource(self._config['database'])
except DataSourceException as err:
msg = 'data source initialization error [{}]'.format(str(err))
Log.an().error(msg)
return False
# set start time (if started, or errored immediatedly)
if (
status in ['RUNNING', 'ERROR']
and self._status == 'PENDING'
):
if not data_source.set_job_started(self._job_id):
Log.a().warning('cannot set job start time in data source')
data_source.rollback()
# set finished time (even on error)
if status in ['FINISHED', 'ERROR']:
if not data_source.set_job_finished(self._job_id):
Log.a().warning('cannot set job finish time in data source')
data_source.rollback()
# if state change, contact notification endpoint
if status != self._status:
if self._job['notifications']:
self._send_notifications(status)
# update database
self._status = status
if not data_source.update_job_status(self._job_id, status, msg):
Log.a().warning('cannot update job status in data source')
data_source.rollback()
data_source.commit()
return True
def run_pending(args):
"""
Run any jobs in database in the PENDING state.
Args:
args.config_file: GeneFlow config file path.
args.environment: Config environment.
Returns:
On success: True.
On failure: False.
"""
config_file = args.config_file
environment = args.environment
log_location = args.log_location
# load config file
cfg = Config()
if not cfg.load(config_file):
Log.an().error('cannot load config file: %s', config_file)
return False
config_dict = cfg.config(environment)
if not config_dict:
Log.an().error('invalid config environment: %s', environment)
return False
# connect to data source
try:
data_source = DataSource(config_dict['database'])
except DataSourceException as err:
Log.an().error('data source initialization error [%s]', str(err))
return False
# get pending jobs from database
pending_jobs = data_source.get_pending_jobs()
if pending_jobs is False:
Log.an().error('cannot query for pending jobs')
return False
if not pending_jobs:
# no jobs found
return True
Log.some().info('pending jobs found:\n%s', pprint.pformat(pending_jobs))
# set job status to QUEUED to minimize the chance that another
# process will try to run it
for job in pending_jobs:
if not data_source.update_job_status(job['id'], 'QUEUED', ''):
Log.a().warning('cannot update job status in data source')
data_source.rollback()
data_source.commit()
# create a thread pool to run at most 5 jobs concurrently
pool = Pool(min(5, len(pending_jobs)))
jobs = [{
'name': job['name'],
'id': job['id'],
'log': str(Path(log_location) / (job['id'] + '.log'))
} for job in pending_jobs]
result = pool.map(
partial(geneflow.cli.common.run_workflow,
config=config_dict,
log_level=args.log_level), jobs)
pool.close()
pool.join()
if not all(result):
Log.an().error('some jobs failed')
return result