Пример #1
0
    def split_job(job):
        parallel_input = None
        for input_name, input_val in six.iteritems(job.inputs):
            io = job.app.get_input(input_name)
            val_d = Executor.depth(input_val)
            if val_d == io.depth:
                continue
            if val_d > io.depth + 1:
                raise RabixError("Depth difference to large")
            if val_d < io.depth:
                raise RabixError("Insufficient dimensionality")
            if parallel_input:
                raise RabixError("Already parallelized by input '%%s'" % parallel_input)

            parallel_input = input_name

        if parallel_input:
            jobs = []
            for i, val in enumerate(job.inputs[parallel_input]):
                inputs = copy.deepcopy(job.inputs)
                inputs[parallel_input] = val
                jobs.append(Job(job.id+"_"+six.text_type(i), job.app, inputs, {}, job.context))
            return jobs
        else:
            return job
Пример #2
0
def conformance_test(context, app, job_dict, basedir):
    job_dict['class'] = 'Job'
    job_dict['id'] = basedir
    job_dict['app'] = app

    if not app.outputs:
        app.outputs = []

    job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
    job = Job.from_dict(context, job_dict)

    adapter = CLIJob(job)

    result = {
        'args': adapter.make_arg_list(),
        'stdin': adapter.stdin,
        'stdout': adapter.stdout,
    }

    cfr = app.get_requirement(CreateFileRequirement)
    e = ValueResolver(job)

    if cfr:
        result['createfiles'] = {
            name: content
            for name, content in cfr.resolve_file_defs(e)
        }

    print(json.dumps(result))
Пример #3
0
def test_workflow():
    path = abspath(join(__file__, '../../test_runtime/wf_tests.yaml'))

    doc = from_url(path)
    tests = doc['tests']
    for test_name, test in six.iteritems(tests):
        context = init_context(test['job'])
        job = Job.from_dict(context, test['job'])

        for inp in job.app.inputs:
            construct_files(job.inputs.get(inp.id), inp.validator)
        yield assert_execution, job, test['outputs']
Пример #4
0
def get_command_line():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {
        strip_prefix(k): construct_files(v, tool._inputs[k].validator)
        for k, v in data['input_map'].iteritems()
    }
    job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    return json.dumps(
        {
            'arguments': cli_job.make_arg_list(),
            'stdin': cli_job.stdin,
            'stdout': cli_job.stdout,
        },
        indent=2)
Пример #5
0
def get_outputs():
    data = request.get_json(force=True)
    tool = process_builder(ctx, data['tool_cfg'])
    inputs = {
        strip_prefix(k): construct_files(v, tool._inputs[k].validator)
        for k, v in data['input_map'].iteritems()
    }
    job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
    cli_job = CLIJob(job)
    status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
        else 'FAILURE'
    return json.dumps(
        {
            'status': status,
            'outputs': cli_job.get_outputs(data['job_dir'], job),
        },
        indent=2)
Пример #6
0
def conformance_test(context, app, job_dict, basedir):
    job_dict['class'] = 'Job'
    job_dict['id'] = basedir
    job_dict['app'] = app

    if not app.outputs:
        app.outputs = []

    job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
    job = Job.from_dict(context, job_dict)

    adapter = CLIJob(job)

    print(json.dumps({
        'args': adapter.make_arg_list(),
        'stdin': adapter.stdin,
        'stdout': adapter.stdout,
    }))
Пример #7
0
    def run(self, job, job_dir=None):
        job_dir = os.path.abspath(job_dir or job.id)
        if not job_dir.endswith('/'):
            job_dir += '/'

        os.mkdir(job_dir)
        os.chmod(job_dir,
                 os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
        self.cli_job = CLIJob(job)

        if self.container:
            self.ensure_files(job, job_dir)
            abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
                              job.allocated_resources, job.context)
            self.install(job=job)

            cmd_line = self.command_line(job, job_dir)
            self.job_dump(job, job_dir)
            self.container.run(cmd_line, job_dir)
            result_path = os.path.abspath(job_dir) + '/result.cwl.json'
            if os.path.exists(result_path):
                with open(result_path, 'r') as res:
                    outputs = json.load(res)
            else:
                with open(result_path, 'w') as res:
                    outputs = self.cli_job.get_outputs(
                        os.path.abspath(job_dir), abspath_job)
                    json.dump(outputs, res)

            outputs = {
                o.id: construct_files(outputs.get(o.id), o.validator)
                for o in job.app.outputs
            }

            self.unmap_paths(outputs)

            def write_rbx(f):
                if isinstance(f, File):
                    with open(f.path + '.rbx.json', 'w') as rbx:
                        json.dump(f.to_dict(), rbx)

            map_rec_collection(write_rbx, outputs)

            return outputs
Пример #8
0
def conformance_test(context, app, job_dict, basedir):
    job_dict['class'] = 'Job'
    job_dict['id'] = basedir
    job_dict['app'] = app

    if not app.outputs:
        app.outputs = []

    job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
    job = Job.from_dict(context, job_dict)

    adapter = CLIJob(job)

    print(
        json.dumps({
            'args': adapter.make_arg_list(),
            'stdin': adapter.stdin,
            'stdout': adapter.stdout,
        }))
Пример #9
0
 def job(self):
     return Job(None, self.app, self.inputs, {}, self.context)
Пример #10
0
def main():
    disable_warnings()
    logging.basicConfig(level=logging.WARN)
    if len(sys.argv) == 1:
        print(USAGE)
        return

    usage = USAGE.format(resources=make_resources_usage_string(),
                         inputs='<inputs>')
    app_usage = usage

    if len(sys.argv) == 2 and \
            (sys.argv[1] == '--help' or sys.argv[1] == '-h'):
        print(USAGE)
        return

    dry_run_args = dry_run_parse()
    if not dry_run_args:
        print(USAGE)
        return

    if not (dry_run_args['<tool>']):
        print('You have to specify a tool, with --tool option')
        print(usage)
        return

    tool = get_tool(dry_run_args)
    if not tool:
        fail("Couldn't find tool.")

    if isinstance(tool, list):
        tool = loader.index.get('#main')

    if 'class' not in tool:
        fail("Document must have a 'class' field")

    if 'id' not in tool:
        tool['id'] = dry_run_args['<tool>']

    context = init_context(tool)

    app = process_builder(context, tool)
    job = None

    if isinstance(app, Job):
        job = app
        app = job.app

    rabix.expressions.update_engines(app)

    if dry_run_args['--install']:
        app.install()
        print("Install successful.")
        return

    if dry_run_args['--conformance-test']:
        job_dict = from_url(dry_run_args['<job>'])
        conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
        return

    try:
        args = docopt.docopt(usage, version=version, help=False)
        job_dict = copy.deepcopy(TEMPLATE_JOB)
        logging.root.setLevel(log_level(dry_run_args['--verbose']))

        input_file_path = args.get('<inp>') or args.get('--inp-file')
        if input_file_path:
            basedir = os.path.dirname(os.path.abspath(input_file_path))
            input_file = from_url(input_file_path)
            inputs = get_inputs(input_file, app.inputs, basedir)
            job_dict['inputs'].update(inputs)

        input_usage = job_dict['inputs']

        if job:
            basedir = os.path.dirname(args.get('<tool>'))
            job.inputs = get_inputs(job.inputs, app.inputs, basedir)
            input_usage.update(job.inputs)

        app_inputs_usage = make_app_usage_string(
            app, template=TOOL_TEMPLATE, inp=input_usage)

        app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])

        try:
            app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
        except docopt.DocoptExit:
            if not job:
                raise
            for inp in job.app.inputs:
                if inp.required and inp.id not in job.inputs:
                    raise
            app_inputs = {}

        if args['--help']:
            print(app_usage)
            return
        # trim leading --, and ignore empty arays
        app_inputs = {
            k[2:]: v
            for k, v in six.iteritems(app_inputs)
            if v != []
        }

        inp = get_inputs(app_inputs, app.inputs)
        if not job:
            job_dict['id'] = args.get('--outdir') or args.get('--dir')
            job_dict['app'] = app
            job = Job.from_dict(context, job_dict)

        job.inputs.update(inp)

        if args['--print-cli']:
            if not isinstance(app, CommandLineTool):
                fail(dry_run_args['<tool>'] + " is not a command line app")

            print(CLIJob(job).cmd_line())
            return

        if args['--pretty-print']:
            fmt = partial(result_str, job.id)
        else:
            fmt = lambda result: json.dumps(context.to_primitive(result))

        if not job.inputs and not args['--'] and not args['--quiet']:
            print(app_usage)
            return

        try:
            context.executor.execute(job, lambda _, result: print(fmt(result)))
        except RabixError as err:
            fail(err.message)

    except docopt.DocoptExit:
        fail(app_usage)
Пример #11
0
def main():
    logging.basicConfig(level=logging.WARN)
    if len(sys.argv) == 1:
        print(USAGE)
        return

    usage = USAGE.format(resources=make_resources_usage_string(),
                         inputs='<inputs>')
    app_usage = usage

    if len(sys.argv) == 2 and \
            (sys.argv[1] == '--help' or sys.argv[1] == '-h'):
        print(USAGE)
        return

    dry_run_args = dry_run_parse()
    if not dry_run_args:
        print(USAGE)
        return

    if not (dry_run_args['<tool>']):
        print('You have to specify a tool, with --tool option')
        print(usage)
        return

    tool = get_tool(dry_run_args)
    if not tool:
        print("Couldn't find tool.")
        return

    fix_types(tool, dry_run_args.get('--type', 'CommandLine'))

    context = init_context()
    app = context.from_dict(tool)
    job = None

    if isinstance(app, Job):
        job = app
        app = job.app

    if dry_run_args['--install']:
        app.install()
        print("Install successful.")
        return

    if dry_run_args['--conformance-test']:
        job_dict = from_url(dry_run_args['<job>'])
        conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
        return

    try:
        args = docopt.docopt(usage, version=version, help=False)
        job_dict = TEMPLATE_JOB
        logging.root.setLevel(log_level(dry_run_args['--verbose']))

        if args['--inp-file']:
            basedir = os.path.dirname(args.get('--inp-file'))
            input_file = from_url(args.get('--inp-file'))
            rebased = rebase_paths(app, input_file, basedir)
            dot_update_dict(
                job_dict['inputs'],
                get_inputs(app, rebased)
            )

        input_usage = job_dict['inputs']

        if job:
            basedir = os.path.dirname(args.get('<tool>'))
            rebased = rebase_paths(app, job.inputs, basedir)
            dot_update_dict(job.inputs, rebased)
            job.inputs.update(get_inputs(app, job.inputs))
            input_usage.update(job.inputs)

        app_inputs_usage = make_app_usage_string(
            app, template=TOOL_TEMPLATE, inp=input_usage)

        app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])

        try:
            app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
        except docopt.DocoptExit:
            if not job:
                raise
            for inp in job.app.inputs:
                if inp.required and inp.id not in job.inputs:
                    raise
            app_inputs = {}

        if args['--help']:
            print(app_usage)
            return

        inp = get_inputs(app, app_inputs)
        if not job:
            job_dict['inputs'].update(inp)
            job_dict['@id'] = args.get('--dir')
            job_dict['app'] = app
            job = Job.from_dict(context, job_dict)
        else:
            job.inputs.update(inp)

        if args['--print-cli']:
            if not isinstance(app, CliApp):
                print(dry_run_args['<tool>'] + " is not a command line app")
                return

            print(CLIJob(job).cmd_line())
            return

        if not job.inputs and not args['--']:
            print(app_usage)
            return

        try:
            context.executor.execute(job, lambda _, result: print(result))
        except RabixError as err:
            print(err.message)
            sys.exit(1)

    except docopt.DocoptExit:
        print(app_usage)
        sys.exit(1)
Пример #12
0
def main():
    logging.basicConfig(level=logging.WARN)
    if len(sys.argv) == 1:
        print(USAGE)
        return

    usage = USAGE.format(resources=make_resources_usage_string(),
                         inputs='<inputs>')
    app_usage = usage

    if len(sys.argv) == 2 and \
            (sys.argv[1] == '--help' or sys.argv[1] == '-h'):
        print(USAGE)
        return

    dry_run_args = dry_run_parse()
    if not dry_run_args:
        print(USAGE)
        return

    if not (dry_run_args['<tool>']):
        print('You have to specify a tool, with --tool option')
        print(usage)
        return

    tool = get_tool(dry_run_args)
    if not tool:
        print("Couldn't find tool.")
        return

    fix_types(tool, dry_run_args.get('--type', 'CommandLine'))

    context = init_context()
    app = context.from_dict(tool)
    job = None

    if isinstance(app, Job):
        job = app
        app = job.app

    if dry_run_args['--install']:
        app.install()
        print("Install successful.")
        return

    if dry_run_args['--conformance-test']:
        job_dict = from_url(dry_run_args['<job>'])
        conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
        return

    try:
        args = docopt.docopt(usage, version=version, help=False)
        job_dict = copy.deepcopy(TEMPLATE_JOB)
        logging.root.setLevel(log_level(dry_run_args['--verbose']))

        if args['--inp-file']:
            basedir = os.path.dirname(args.get('--inp-file'))
            input_file = from_url(args.get('--inp-file'))
            inputs = get_inputs(app, input_file, basedir)
            job_dict['inputs'].update(inputs)

        input_usage = job_dict['inputs']

        if job:
            basedir = os.path.dirname(args.get('<tool>'))
            job.inputs = get_inputs(app, job.inputs, basedir)
            input_usage.update(job.inputs)

        app_inputs_usage = make_app_usage_string(
            app, template=TOOL_TEMPLATE, inp=input_usage)

        app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])

        try:
            app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
        except docopt.DocoptExit:
            if not job:
                raise
            for inp in job.app.inputs:
                if inp.required and inp.id not in job.inputs:
                    raise
            app_inputs = {}

        if args['--help']:
            print(app_usage)
            return
        # trim leading --, and ignore empty arays
        app_inputs = {
            k[2:]: v
            for k, v in six.iteritems(app_inputs)
            if v != []
        }

        inp = get_inputs(app, app_inputs)
        if not job:
            job_dict['@id'] = args.get('--dir')
            job_dict['app'] = app
            job = Job.from_dict(context, job_dict)

        job.inputs.update(inp)

        if args['--print-cli']:
            if not isinstance(app, CliApp):
                print(dry_run_args['<tool>'] + " is not a command line app")
                return

            print(CLIJob(job).cmd_line())
            return

        if not job.inputs and not args['--']:
            print(app_usage)
            return

        try:
            context.executor.execute(job, lambda _, result: print(
                result_str(job.id, result)))
        except RabixError as err:
            print(err.message)
            sys.exit(1)

    except docopt.DocoptExit:
        print(app_usage)
        sys.exit(1)
Пример #13
0
def main():
    disable_warnings()
    logging.basicConfig(level=logging.WARN)
    if len(sys.argv) == 1:
        print(USAGE)
        return

    usage = USAGE.format(resources=make_resources_usage_string(),
                         inputs='<inputs>')
    app_usage = usage

    if len(sys.argv) == 2 and \
            (sys.argv[1] == '--help' or sys.argv[1] == '-h'):
        print(USAGE)
        return

    dry_run_args = dry_run_parse()
    if not dry_run_args:
        print(USAGE)
        return

    if not (dry_run_args['<tool>']):
        print('You have to specify a tool, with --tool option')
        print(usage)
        return

    tool = get_tool(dry_run_args)
    if not tool:
        fail("Couldn't find tool.")

    if isinstance(tool, list):
        tool = loader.index.get('#main')

    if 'class' not in tool:
        fail("Document must have a 'class' field")

    if 'id' not in tool:
        tool['id'] = dry_run_args['<tool>']

    context = init_context(tool)

    app = process_builder(context, tool)
    job = None

    if isinstance(app, Job):
        job = app
        app = job.app

    rabix.expressions.update_engines(app)

    if dry_run_args['--install']:
        app.install()
        print("Install successful.")
        return

    if dry_run_args['--conformance-test']:
        job_dict = from_url(dry_run_args['<job>'])
        conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
        return

    try:
        args = docopt.docopt(usage, version=version, help=False)
        job_dict = copy.deepcopy(TEMPLATE_JOB)
        logging.root.setLevel(log_level(dry_run_args['--verbose']))

        input_file_path = args.get('<inp>') or args.get('--inp-file')
        if input_file_path:
            basedir = os.path.dirname(os.path.abspath(input_file_path))
            input_file = from_url(input_file_path)
            inputs = get_inputs(input_file, app.inputs, basedir)
            job_dict['inputs'].update(inputs)

        input_usage = job_dict['inputs']

        if job:
            basedir = os.path.dirname(args.get('<tool>'))
            job.inputs = get_inputs(job.inputs, app.inputs, basedir)
            input_usage.update(job.inputs)

        app_inputs_usage = make_app_usage_string(app,
                                                 template=TOOL_TEMPLATE,
                                                 inp=input_usage)

        app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])

        try:
            app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
        except docopt.DocoptExit:
            if not job:
                raise
            for inp in job.app.inputs:
                if inp.required and inp.id not in job.inputs:
                    raise
            app_inputs = {}

        if args['--help']:
            print(app_usage)
            return
        # trim leading --, and ignore empty arays
        app_inputs = {
            k[2:]: v
            for k, v in six.iteritems(app_inputs) if v != []
        }

        inp = get_inputs(app_inputs, app.inputs)
        if not job:
            job_dict['id'] = args.get('--outdir') or args.get('--dir')
            job_dict['app'] = app
            job = Job.from_dict(context, job_dict)

        job.inputs.update(inp)

        if args['--print-cli']:
            if not isinstance(app, CommandLineTool):
                fail(dry_run_args['<tool>'] + " is not a command line app")

            print(CLIJob(job).cmd_line())
            return

        if args['--pretty-print']:
            fmt = partial(result_str, job.id)
        else:
            fmt = lambda result: json.dumps(context.to_primitive(result))

        if not job.inputs and not args['--'] and not args['--quiet']:
            print(app_usage)
            return

        try:
            context.executor.execute(job, lambda _, result: print(fmt(result)))
        except RabixError as err:
            fail(err.message)

    except docopt.DocoptExit:
        fail(app_usage)
Пример #14
0
    def run(self, job, job_dir=None):
        self.load_input_content(job)
        job_dir = os.path.abspath(job_dir or job.id)
        if not job_dir.endswith('/'):
            job_dir += '/'

        if not os.path.exists(job_dir):
            os.mkdir(job_dir)

        os.chmod(job_dir,
                 os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
        self.cli_job = CLIJob(job)

        eval = ValueResolver(job)

        cfr = self.get_requirement_or_hint(CreateFileRequirement)
        if cfr:
            cfr.create_files(job_dir, eval)

        env = None
        evr = self.get_requirement_or_hint(EnvVarRequirement)
        if evr:
            env = evr.var_map(eval)

        self.ensure_files(job, job_dir)
        self.install(job=job)

        abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
                          job.allocated_resources, job.context)

        cmd_line = self.command_line(job, job_dir)
        log.info("Running: %s" % cmd_line)
        self.job_dump(job, job_dir)

        if self.container:
            self.container.run(cmd_line, job_dir, env)
        else:
            ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir)
            if ret != 0:
                raise RabixError("Command failed with exit status %s" % ret)

        result_path = os.path.abspath(job_dir) + '/cwl.output.json'
        if os.path.exists(result_path):
            with open(result_path, 'r') as res:
                outputs = json.load(res)
        else:
            with open(result_path, 'w') as res:
                outputs = self.cli_job.get_outputs(os.path.abspath(job_dir),
                                                   abspath_job)
                json.dump(job.context.to_primitive(outputs), res)

        self.unmap_paths(outputs)

        def write_rbx(f):
            if isinstance(f, File):
                with open(f.path + '.rbx.json', 'w') as rbx:
                    json.dump(f.to_dict(), rbx)

        map_rec_collection(write_rbx, outputs)

        return outputs