예제 #1
0
def run_submission(eval_id, submission_id, wes_id='local'):
    """
    For a single submission to a single evaluation queue, run
    the workflow in a single environment.
    """
    submission = eval.get_submission_bundle(eval_id, submission_id)
    if submission['wes_id'] is not None:
        wes_id = submission['wes_id']
    logger.info(
        "Submitting job '{}' for eval '{}' to WES endpoint '{}'".format(
            submission_id, eval_id, wes_id))
    client = WESClient(**config.wes_config()[wes_id])
    run_data = client.run_workflow(submission['data'])
    run_data['start_time'] = dt.datetime.now().ctime()
    eval.update_submission_run(eval_id, submission_id, run_data)
    eval.update_submission_status(eval_id, submission_id, 'SUBMITTED')
    return eval.get_submission_bundle(eval_id, submission_id)
예제 #2
0
def monitor(submissions):
    """
    Monitor progress of workflow jobs.
    """
    current = dt.datetime.now()
    statuses = []
    for submission_dict in submissions:
        status_dict = {}
        for eval_id in submission_dict:
            for submission_id, bundle in submission_dict[eval_id].items():
                client = WESClient(**config.wes_config()[bundle['wes_id']])
                run = client.get_workflow_run_status(bundle['run_id'])
                if run['state'] in ['QUEUED', 'INITIALIZING', 'RUNNING']:
                    etime = util.convert_timedelta(
                        current - util.ctime2datetime(bundle['start_time']))
                elif 'elapsed_time' not in bundle:
                    etime = 0
                else:
                    etime = bundle['elapsed_time']
                status_dict.setdefault(eval_id, {})[submission_id] = {
                    'queue_id': eval_id,
                    'job': bundle['job'],
                    'submission_status': bundle['submission_status'],
                    'wes_id': bundle['wes_id'],
                    'run_id': run['workflow_id'],
                    'run_status': run['state'],
                    'start_time': bundle['start_time'],
                    'elapsed_time': etime
                }
        statuses.append(status_dict)

    status_df = pd.DataFrame.from_dict(
        {(i, j): status[i][j]
         for status in statuses for i in status.keys()
         for j in status[i].keys()},
        orient='index')

    clear_output(wait=True)
    display(status_df)
    sys.stdout.flush()
    if any(status_df['run_status'].isin(['QUEUED', 'INITIALIZING',
                                         'RUNNING'])):
        time.sleep(1)
        monitor(statuses)
    else:
        print("Done")
예제 #3
0
def run_submission(wes_id, submission_id):
    """
    For a single submission to a single evaluation queue, run
    the workflow in a single environment.
    """
    submission = get_submission_bundle(wes_id, submission_id)

    logger.info(" Submitting to WES endpoint '{}':"
                " \n - submission ID: {}".format(wes_id, submission_id))

    client = WESClient(wes_config()[wes_id])
    run_data = client.run(submission['data']['wf'],
                          submission['data']['jsonyaml'],
                          submission['data']['attachments'])
    run_data['start_time'] = dt.datetime.now().ctime()
    update_submission(wes_id, submission_id, 'run', run_data)
    update_submission(wes_id, submission_id, 'status', 'SUBMITTED')
    return run_data
예제 #4
0
def monitor_service(wf_service):
    """
    Returns a dictionary of all of the jobs under a single wes service appropriate
    for displaying as a pandas dataframe.

    :param wf_service:
    :return:
    """
    status_dict = {}
    submissions = get_json(queue_path())
    for run_id in submissions[wf_service]:
        sample_name = submissions[wf_service][run_id]['sample']
        if 'run' not in submissions[wf_service][run_id]:
            status_dict.setdefault(wf_service, {})[run_id] = {
                'wf_id': submissions[wf_service][run_id]['wf_id'],
                'run_id': '-',
                'sample_name': sample_name,
                'run_status': 'QUEUED',
                'start_time': '-',
                'elapsed_time': '-'
            }
        else:
            if submissions[wf_service][run_id]['status'] in [
                    'COMPLETE', 'SYSTEM_ERROR', 'EXECUTOR_ERROR'
            ]:
                run = submissions[wf_service][run_id]['run']
                try:
                    wf_id = run['workflow_id']
                except KeyError:
                    wf_id = run['run_id']
                status_dict.setdefault(wf_service, {})[run_id] = {
                    'wf_id': submissions[wf_service][run_id]['wf_id'],
                    'run_id': wf_id,
                    'sample_name': sample_name,
                    'run_status': submissions[wf_service][run_id]['status'],
                    'start_time': run['start_time'],
                    'elapsed_time': run['elapsed_time']
                }
            else:
                try:
                    run = submissions[wf_service][run_id]['run']
                    if 'run_id' not in run and 'workflow_id' not in run:
                        status_dict.setdefault(wf_service, {})[run_id] = {
                            'wf_id': submissions[wf_service][run_id]['wf_id'],
                            'run_id': '-',
                            'sample_name': sample_name,
                            'run_status': 'INITIALIZING',
                            'start_time': '-',
                            'elapsed_time': '-'
                        }
                    else:
                        client = WESClient(wes_config()[wf_service])
                        try:
                            wf_id = run['workflow_id']
                        except KeyError:
                            wf_id = run['run_id']
                        if 'state' not in run:
                            run['state'] = client.get_run_status(
                                wf_id)['state'].upper()
                        elif run['state'].upper() not in [
                                'COMPLETED', 'OK', 'EXECUTOR_ERROR',
                                'SYSTEM_ERROR'
                        ]:
                            run['state'] = client.get_run_status(
                                wf_id)['state'].upper()

                        if run['state'] in [
                                'QUEUED', 'INITIALIZING', 'RUNNING'
                        ]:
                            etime = convert_timedelta(
                                dt.datetime.now() -
                                ctime2datetime(run['start_time']))
                        elif 'elapsed_time' not in run:
                            etime = '0h:0m:0s'
                        else:
                            update_submission(wf_service, run_id, 'status',
                                              run['state'])
                            etime = run['elapsed_time']
                        update_submission_run(wf_service, run_id,
                                              'elapsed_time', etime)
                        status_dict.setdefault(wf_service, {})[run_id] = {
                            'wf_id': submissions[wf_service][run_id]['wf_id'],
                            'run_id': wf_id,
                            'sample_name': sample_name,
                            'run_status': run['state'],
                            'start_time': run['start_time'],
                            'elapsed_time': etime
                        }
                except ConnectionError:
                    status_dict.setdefault(wf_service, {})[run_id] = {
                        'wf_id': 'ConnectionError',
                        'run_id': '-',
                        'sample_name': sample_name,
                        'run_status': '-',
                        'start_time': '-',
                        'elapsed_time': '-'
                    }

    return status_dict