Пример #1
0
def test_workflow():
    path = abspath(join(__file__, '../../test_runtime/wf_tests.yaml'))

    doc = from_url(path)
    tests = doc['tests']
    for test_name, test in six.iteritems(tests):
        context = init_context(test['job'])
        job = Job.from_dict(context, test['job'])

        for inp in job.app.inputs:
            construct_files(job.inputs.get(inp.id), inp.validator)
        yield assert_execution, job, test['outputs']
Пример #2
0
def get_inputs(args, inputs, basedir=None):

    basedir = basedir or os.path.abspath('.')
    constructed = {}
    for i in inputs:
        val = args.get(i.id)
        if i.depth == 0:
            cons = construct_files(val, i.validator)
        else:
            cons = [construct_files(e, i.validator)
                    for e in val] if val else []
        if cons:
            constructed[i.id] = cons
    return map_rec_collection(lambda v: rebase_path(v, basedir), constructed)
Пример #3
0
def get_inputs(args, inputs, basedir=None):

    basedir = basedir or os.path.abspath('.')
    constructed = {}
    for i in inputs:
        val = args.get(i.id)
        if i.depth == 0:
            cons = construct_files(val, i.validator)
        else:
            cons = [construct_files(e, i.validator) for e in val] if val else []
        if cons:
            constructed[i.id] = cons
    return map_rec_collection(
        lambda v: rebase_path(v, basedir),
        constructed
    )
Пример #4
0
def get_command_line():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {strip_prefix(k): construct_files(v, tool._inputs[k].validator)
              for k, v in data['input_map'].iteritems()}
    job = Job('Fake job ID', tool, inputs,  {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    return json.dumps({
        'arguments': cli_job.make_arg_list(),
        'stdin': cli_job.stdin,
        'stdout': cli_job.stdout,
    }, indent=2)
Пример #5
0
def get_outputs():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {strip_prefix(k): construct_files(v, tool._inputs[k].validator)
              for k, v in data['input_map'].iteritems()}
    job = Job('Fake job ID', tool, inputs,  {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
        else 'FAILURE'
    return json.dumps({
        'status': status,
        'outputs': cli_job.get_outputs(data['job_dir'], job),
    }, indent=2)
Пример #6
0
def get_command_line():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {
        strip_prefix(k): construct_files(v, tool._inputs[k].validator)
        for k, v in data['input_map'].iteritems()
    }
    job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    return json.dumps(
        {
            'arguments': cli_job.make_arg_list(),
            'stdin': cli_job.stdin,
            'stdout': cli_job.stdout,
        },
        indent=2)
Пример #7
0
def get_outputs():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {
        strip_prefix(k): construct_files(v, tool._inputs[k].validator)
        for k, v in data['input_map'].iteritems()
    }
    job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
        else 'FAILURE'
    return json.dumps(
        {
            'status': status,
            'outputs': cli_job.get_outputs(data['job_dir'], job),
        },
        indent=2)
Пример #8
0
def get_outputs():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {strip_prefix(k): construct_files(v, tool._inputs[strip_prefix(k)].validator) for k, v in data['input_map'].iteritems()}
    job = Job('Fake job ID', tool, inputs,  {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    print data
#    status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0]) else 'FAILURE'
    status = 'SUCCESS'
    outputs = cli_job.get_outputs(data['job_dir'], job)
    if inputs:
      key = inputs.keys()[0]
      prefix = key[0:key.rfind('.')+1]
      outputs = {prefix + k: v for k, v in outputs.iteritems()}
    return json.dumps({
        'status': status,
        'outputs': outputs,
    })
Пример #9
0
    def run(self, job, job_dir=None):
        job_dir = os.path.abspath(job_dir or job.id)
        if not job_dir.endswith('/'):
            job_dir += '/'

        os.mkdir(job_dir)
        os.chmod(job_dir,
                 os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
        self.cli_job = CLIJob(job)

        if self.container:
            self.ensure_files(job, job_dir)
            abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
                              job.allocated_resources, job.context)
            self.install(job=job)

            cmd_line = self.command_line(job, job_dir)
            self.job_dump(job, job_dir)
            self.container.run(cmd_line, job_dir)
            result_path = os.path.abspath(job_dir) + '/result.cwl.json'
            if os.path.exists(result_path):
                with open(result_path, 'r') as res:
                    outputs = json.load(res)
            else:
                with open(result_path, 'w') as res:
                    outputs = self.cli_job.get_outputs(
                        os.path.abspath(job_dir), abspath_job)
                    json.dump(outputs, res)

            outputs = {
                o.id: construct_files(outputs.get(o.id), o.validator)
                for o in job.app.outputs
            }

            self.unmap_paths(outputs)

            def write_rbx(f):
                if isinstance(f, File):
                    with open(f.path + '.rbx.json', 'w') as rbx:
                        json.dump(f.to_dict(), rbx)

            map_rec_collection(write_rbx, outputs)

            return outputs
Пример #10
0
    def run(self, job, job_dir=None):
        job_dir = os.path.abspath(job_dir or job.id)
        if not job_dir.endswith('/'):
            job_dir += '/'

        os.mkdir(job_dir)
        os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH |
                 stat.S_IWOTH)
        self.cli_job = CLIJob(job)

        if self.container:
            self.ensure_files(job, job_dir)
            abspath_job = Job(
                job.id, job.app, copy.deepcopy(job.inputs),
                job.allocated_resources, job.context
            )
            self.install(job=job)

            cmd_line = self.command_line(job, job_dir)
            self.job_dump(job, job_dir)
            self.container.run(cmd_line, job_dir)
            result_path = os.path.abspath(job_dir) + '/result.cwl.json'
            if os.path.exists(result_path):
                with open(result_path, 'r') as res:
                    outputs = json.load(res)
            else:
                with open(result_path, 'w') as res:
                    outputs = self.cli_job.get_outputs(
                        os.path.abspath(job_dir), abspath_job)
                    json.dump(outputs, res)

            outputs = {o.id: construct_files(outputs.get(o.id), o.validator)
                       for o in job.app.outputs}

            self.unmap_paths(outputs)

            def write_rbx(f):
                if isinstance(f, File):
                    with open(f.path + '.rbx.json', 'w') as rbx:
                        json.dump(f.to_dict(), rbx)

            map_rec_collection(write_rbx, outputs)

            return outputs