def _update_status_db(self, status, msg): """ Update workflow status in DB. Args: self: class instance status: Workflow status msg: Success, error or warning message Returns: On success: True. On failure: False. """ try: data_source = DataSource(self._config['database']) except DataSourceException as err: msg = 'data source initialization error [{}]'.format(str(err)) Log.an().error(msg) return False # set start time (if started, or errored immediatedly) if ( status in ['RUNNING', 'ERROR'] and self._status == 'PENDING' ): if not data_source.set_job_started(self._job_id): Log.a().warning('cannot set job start time in data source') data_source.rollback() # set finished time (even on error) if status in ['FINISHED', 'ERROR']: if not data_source.set_job_finished(self._job_id): Log.a().warning('cannot set job finish time in data source') data_source.rollback() # if state change, contact notification endpoint if status != self._status: if self._job['notifications']: self._send_notifications(status) # update database self._status = status if not data_source.update_job_status(self._job_id, status, msg): Log.a().warning('cannot update job status in data source') data_source.rollback() data_source.commit() return True
def run_pending(args): """ Run any jobs in database in the PENDING state. Args: args.config_file: GeneFlow config file path. args.environment: Config environment. Returns: On success: True. On failure: False. """ config_file = args.config_file environment = args.environment log_location = args.log_location # load config file cfg = Config() if not cfg.load(config_file): Log.an().error('cannot load config file: %s', config_file) return False config_dict = cfg.config(environment) if not config_dict: Log.an().error('invalid config environment: %s', environment) return False # connect to data source try: data_source = DataSource(config_dict['database']) except DataSourceException as err: Log.an().error('data source initialization error [%s]', str(err)) return False # get pending jobs from database pending_jobs = data_source.get_pending_jobs() if pending_jobs is False: Log.an().error('cannot query for pending jobs') return False if not pending_jobs: # no jobs found return True Log.some().info('pending jobs found:\n%s', pprint.pformat(pending_jobs)) # set job status to QUEUED to minimize the chance that another # process will try to run it for job in pending_jobs: if not data_source.update_job_status(job['id'], 'QUEUED', ''): Log.a().warning('cannot update job status in data source') data_source.rollback() data_source.commit() # create a thread pool to run at most 5 jobs concurrently pool = Pool(min(5, len(pending_jobs))) jobs = [{ 'name': job['name'], 'id': job['id'], 'log': str(Path(log_location) / (job['id'] + '.log')) } for job in pending_jobs] result = pool.map( partial(geneflow.cli.common.run_workflow, config=config_dict, log_level=args.log_level), jobs) pool.close() pool.join() if not all(result): Log.an().error('some jobs failed') return result