def run_submission(eval_id, submission_id, wes_id='local'): """ For a single submission to a single evaluation queue, run the workflow in a single environment. """ submission = eval.get_submission_bundle(eval_id, submission_id) if submission['wes_id'] is not None: wes_id = submission['wes_id'] logger.info( "Submitting job '{}' for eval '{}' to WES endpoint '{}'".format( submission_id, eval_id, wes_id)) client = WESClient(**config.wes_config()[wes_id]) run_data = client.run_workflow(submission['data']) run_data['start_time'] = dt.datetime.now().ctime() eval.update_submission_run(eval_id, submission_id, run_data) eval.update_submission_status(eval_id, submission_id, 'SUBMITTED') return eval.get_submission_bundle(eval_id, submission_id)
def monitor(submissions): """ Monitor progress of workflow jobs. """ current = dt.datetime.now() statuses = [] for submission_dict in submissions: status_dict = {} for eval_id in submission_dict: for submission_id, bundle in submission_dict[eval_id].items(): client = WESClient(**config.wes_config()[bundle['wes_id']]) run = client.get_workflow_run_status(bundle['run_id']) if run['state'] in ['QUEUED', 'INITIALIZING', 'RUNNING']: etime = util.convert_timedelta( current - util.ctime2datetime(bundle['start_time'])) elif 'elapsed_time' not in bundle: etime = 0 else: etime = bundle['elapsed_time'] status_dict.setdefault(eval_id, {})[submission_id] = { 'queue_id': eval_id, 'job': bundle['job'], 'submission_status': bundle['submission_status'], 'wes_id': bundle['wes_id'], 'run_id': run['workflow_id'], 'run_status': run['state'], 'start_time': bundle['start_time'], 'elapsed_time': etime } statuses.append(status_dict) status_df = pd.DataFrame.from_dict( {(i, j): status[i][j] for status in statuses for i in status.keys() for j in status[i].keys()}, orient='index') clear_output(wait=True) display(status_df) sys.stdout.flush() if any(status_df['run_status'].isin(['QUEUED', 'INITIALIZING', 'RUNNING'])): time.sleep(1) monitor(statuses) else: print("Done")
def run_submission(wes_id, submission_id): """ For a single submission to a single evaluation queue, run the workflow in a single environment. """ submission = get_submission_bundle(wes_id, submission_id) logger.info(" Submitting to WES endpoint '{}':" " \n - submission ID: {}".format(wes_id, submission_id)) client = WESClient(wes_config()[wes_id]) run_data = client.run(submission['data']['wf'], submission['data']['jsonyaml'], submission['data']['attachments']) run_data['start_time'] = dt.datetime.now().ctime() update_submission(wes_id, submission_id, 'run', run_data) update_submission(wes_id, submission_id, 'status', 'SUBMITTED') return run_data
def monitor_service(wf_service): """ Returns a dictionary of all of the jobs under a single wes service appropriate for displaying as a pandas dataframe. :param wf_service: :return: """ status_dict = {} submissions = get_json(queue_path()) for run_id in submissions[wf_service]: sample_name = submissions[wf_service][run_id]['sample'] if 'run' not in submissions[wf_service][run_id]: status_dict.setdefault(wf_service, {})[run_id] = { 'wf_id': submissions[wf_service][run_id]['wf_id'], 'run_id': '-', 'sample_name': sample_name, 'run_status': 'QUEUED', 'start_time': '-', 'elapsed_time': '-' } else: if submissions[wf_service][run_id]['status'] in [ 'COMPLETE', 'SYSTEM_ERROR', 'EXECUTOR_ERROR' ]: run = submissions[wf_service][run_id]['run'] try: wf_id = run['workflow_id'] except KeyError: wf_id = run['run_id'] status_dict.setdefault(wf_service, {})[run_id] = { 'wf_id': submissions[wf_service][run_id]['wf_id'], 'run_id': wf_id, 'sample_name': sample_name, 'run_status': submissions[wf_service][run_id]['status'], 'start_time': run['start_time'], 'elapsed_time': run['elapsed_time'] } else: try: run = submissions[wf_service][run_id]['run'] if 'run_id' not in run and 'workflow_id' not in run: status_dict.setdefault(wf_service, {})[run_id] = { 'wf_id': submissions[wf_service][run_id]['wf_id'], 'run_id': '-', 'sample_name': sample_name, 'run_status': 'INITIALIZING', 'start_time': '-', 'elapsed_time': '-' } else: client = WESClient(wes_config()[wf_service]) try: wf_id = run['workflow_id'] except KeyError: wf_id = run['run_id'] if 'state' not in run: run['state'] = client.get_run_status( wf_id)['state'].upper() elif run['state'].upper() not in [ 'COMPLETED', 'OK', 'EXECUTOR_ERROR', 'SYSTEM_ERROR' ]: run['state'] = client.get_run_status( wf_id)['state'].upper() if run['state'] in [ 'QUEUED', 'INITIALIZING', 'RUNNING' ]: etime = convert_timedelta( dt.datetime.now() - ctime2datetime(run['start_time'])) elif 'elapsed_time' not in run: etime = '0h:0m:0s' else: update_submission(wf_service, run_id, 'status', run['state']) etime = run['elapsed_time'] update_submission_run(wf_service, run_id, 'elapsed_time', etime) status_dict.setdefault(wf_service, {})[run_id] = { 'wf_id': submissions[wf_service][run_id]['wf_id'], 'run_id': wf_id, 'sample_name': sample_name, 'run_status': run['state'], 'start_time': run['start_time'], 'elapsed_time': etime } except ConnectionError: status_dict.setdefault(wf_service, {})[run_id] = { 'wf_id': 'ConnectionError', 'run_id': '-', 'sample_name': sample_name, 'run_status': '-', 'start_time': '-', 'elapsed_time': '-' } return status_dict