def test_remap_job(): job = TEMPLATE_JOB tool = from_url(join(dirname(__file__), 'bwa-mem.json#tool')) context = init_context(tool) app = context.from_dict(tool) input_file = from_url(join(dirname(__file__), 'inputs.json')) startdir = './'
def download(self, url, secondary_files=None, prompt=True): npath = to_abspath(self._download(url, metasearch=True)) rbx_path = npath + '.rbx.json' if isfile(rbx_path): file_dict = from_url(rbx_path) startdir = dirname(npath) file_dict['path'] = npath file = File(file_dict) file.secondary_files = [ File( self._download( URL(to_abspath(sf.path, startdir)), metasearch=False)) for sf in file.secondary_files ] else: file = File(npath) file.meta = self._meta(npath, prompt=prompt) if secondary_files: file.secondary_files = self._get_secondary_files( secondary_files, url, prompt=prompt) if prompt: self._rbx_dump(file) return file
def download(self, url, secondary_files=None, prompt=True): npath = to_abspath(self._download(url, metasearch=True)) rbx_path = npath + '.rbx.json' if isfile(rbx_path): file_dict = from_url(rbx_path) startdir = dirname(npath) file_dict['path'] = npath file = FileConstructor()(file_dict) file.secondary_files = [ File( self._download( URL(to_abspath(sf.path, startdir)), metasearch=False)) for sf in file.secondary_files ] else: file = File(npath) file.meta = self._meta(npath, prompt=prompt) if secondary_files: file.secondary_files = self._get_secondary_files( secondary_files, url, prompt=prompt) if prompt: self._rbx_dump(file) return file
def test_workflow(): path = abspath(join(__file__, '../../test_runtime/wf_tests.yaml')) context = init_context() doc = from_url(path) tests = doc['tests'] for test_name, test in six.iteritems(tests): features = test.get('requiresFeatures', []) if 'map' not in features: fix_types(test['job']['app']) yield assert_execution, context.from_dict(test['job']), test['outputs']
def test_workflow(): path = abspath(join(__file__, '../../test_runtime/wf_tests.yaml')) doc = from_url(path) tests = doc['tests'] for test_name, test in six.iteritems(tests): context = init_context(test['job']) job = Job.from_dict(context, test['job']) for inp in job.app.inputs: construct_files(job.inputs.get(inp.id), inp.validator) yield assert_execution, job, test['outputs']
def main(): disable_warnings() logging.basicConfig(level=logging.WARN) if len(sys.argv) == 1: print(USAGE) return usage = USAGE.format(resources=make_resources_usage_string(), inputs='<inputs>') app_usage = usage if len(sys.argv) == 2 and \ (sys.argv[1] == '--help' or sys.argv[1] == '-h'): print(USAGE) return dry_run_args = dry_run_parse() if not dry_run_args: print(USAGE) return if not (dry_run_args['<tool>']): print('You have to specify a tool, with --tool option') print(usage) return tool = get_tool(dry_run_args) if not tool: fail("Couldn't find tool.") if isinstance(tool, list): tool = loader.index.get('#main') if 'class' not in tool: fail("Document must have a 'class' field") if 'id' not in tool: tool['id'] = dry_run_args['<tool>'] context = init_context(tool) app = process_builder(context, tool) job = None if isinstance(app, Job): job = app app = job.app rabix.expressions.update_engines(app) if dry_run_args['--install']: app.install() print("Install successful.") return if dry_run_args['--conformance-test']: job_dict = from_url(dry_run_args['<job>']) conformance_test(context, app, job_dict, dry_run_args.get('--basedir')) return try: args = docopt.docopt(usage, version=version, help=False) job_dict = copy.deepcopy(TEMPLATE_JOB) logging.root.setLevel(log_level(dry_run_args['--verbose'])) input_file_path = args.get('<inp>') or args.get('--inp-file') if input_file_path: basedir = os.path.dirname(os.path.abspath(input_file_path)) input_file = from_url(input_file_path) inputs = get_inputs(input_file, app.inputs, basedir) job_dict['inputs'].update(inputs) input_usage = job_dict['inputs'] if job: basedir = os.path.dirname(args.get('<tool>')) job.inputs = get_inputs(job.inputs, app.inputs, basedir) input_usage.update(job.inputs) app_inputs_usage = make_app_usage_string( app, template=TOOL_TEMPLATE, inp=input_usage) app_usage = make_app_usage_string(app, USAGE, job_dict['inputs']) try: app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>']) except docopt.DocoptExit: if not job: raise for inp in job.app.inputs: if inp.required and inp.id not in job.inputs: raise app_inputs = {} if args['--help']: print(app_usage) return # trim leading --, and ignore empty arays app_inputs = { k[2:]: v for k, v in six.iteritems(app_inputs) if v != [] } inp = get_inputs(app_inputs, app.inputs) if not job: job_dict['id'] = args.get('--outdir') or args.get('--dir') job_dict['app'] = app job = Job.from_dict(context, job_dict) job.inputs.update(inp) if args['--print-cli']: if not isinstance(app, CommandLineTool): fail(dry_run_args['<tool>'] + " is not a command line app") print(CLIJob(job).cmd_line()) return if args['--pretty-print']: fmt = partial(result_str, job.id) else: fmt = lambda result: json.dumps(context.to_primitive(result)) if not job.inputs and not args['--'] and not args['--quiet']: print(app_usage) return try: context.executor.execute(job, lambda _, result: print(fmt(result))) except RabixError as err: fail(err.message) except docopt.DocoptExit: fail(app_usage)
def get_tool(args): if args['<tool>']: return from_url(args['<tool>'])
def main(): logging.basicConfig(level=logging.WARN) if len(sys.argv) == 1: print(USAGE) return usage = USAGE.format(resources=make_resources_usage_string(), inputs='<inputs>') app_usage = usage if len(sys.argv) == 2 and \ (sys.argv[1] == '--help' or sys.argv[1] == '-h'): print(USAGE) return dry_run_args = dry_run_parse() if not dry_run_args: print(USAGE) return if not (dry_run_args['<tool>']): print('You have to specify a tool, with --tool option') print(usage) return tool = get_tool(dry_run_args) if not tool: print("Couldn't find tool.") return fix_types(tool, dry_run_args.get('--type', 'CommandLine')) context = init_context() app = context.from_dict(tool) job = None if isinstance(app, Job): job = app app = job.app if dry_run_args['--install']: app.install() print("Install successful.") return if dry_run_args['--conformance-test']: job_dict = from_url(dry_run_args['<job>']) conformance_test(context, app, job_dict, dry_run_args.get('--basedir')) return try: args = docopt.docopt(usage, version=version, help=False) job_dict = TEMPLATE_JOB logging.root.setLevel(log_level(dry_run_args['--verbose'])) if args['--inp-file']: basedir = os.path.dirname(args.get('--inp-file')) input_file = from_url(args.get('--inp-file')) rebased = rebase_paths(app, input_file, basedir) dot_update_dict( job_dict['inputs'], get_inputs(app, rebased) ) input_usage = job_dict['inputs'] if job: basedir = os.path.dirname(args.get('<tool>')) rebased = rebase_paths(app, job.inputs, basedir) dot_update_dict(job.inputs, rebased) job.inputs.update(get_inputs(app, job.inputs)) input_usage.update(job.inputs) app_inputs_usage = make_app_usage_string( app, template=TOOL_TEMPLATE, inp=input_usage) app_usage = make_app_usage_string(app, USAGE, job_dict['inputs']) try: app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>']) except docopt.DocoptExit: if not job: raise for inp in job.app.inputs: if inp.required and inp.id not in job.inputs: raise app_inputs = {} if args['--help']: print(app_usage) return inp = get_inputs(app, app_inputs) if not job: job_dict['inputs'].update(inp) job_dict['@id'] = args.get('--dir') job_dict['app'] = app job = Job.from_dict(context, job_dict) else: job.inputs.update(inp) if args['--print-cli']: if not isinstance(app, CliApp): print(dry_run_args['<tool>'] + " is not a command line app") return print(CLIJob(job).cmd_line()) return if not job.inputs and not args['--']: print(app_usage) return try: context.executor.execute(job, lambda _, result: print(result)) except RabixError as err: print(err.message) sys.exit(1) except docopt.DocoptExit: print(app_usage) sys.exit(1)
def test_bwa_mem(): path = os.path.join(os.path.dirname(__file__), '../examples/bwa-mem.yml') doc = from_url(path) tool, job = doc['tool'], doc['job'] print(gen_cli(tool, job))
def main(): disable_warnings() logging.basicConfig(level=logging.WARN) if len(sys.argv) == 1: print(USAGE) return usage = USAGE.format(resources=make_resources_usage_string(), inputs='<inputs>') app_usage = usage if len(sys.argv) == 2 and \ (sys.argv[1] == '--help' or sys.argv[1] == '-h'): print(USAGE) return dry_run_args = dry_run_parse() if not dry_run_args: print(USAGE) return if not (dry_run_args['<tool>']): print('You have to specify a tool, with --tool option') print(usage) return tool = get_tool(dry_run_args) if not tool: fail("Couldn't find tool.") if isinstance(tool, list): tool = loader.index.get('#main') if 'class' not in tool: fail("Document must have a 'class' field") if 'id' not in tool: tool['id'] = dry_run_args['<tool>'] context = init_context(tool) app = process_builder(context, tool) job = None if isinstance(app, Job): job = app app = job.app rabix.expressions.update_engines(app) if dry_run_args['--install']: app.install() print("Install successful.") return if dry_run_args['--conformance-test']: job_dict = from_url(dry_run_args['<job>']) conformance_test(context, app, job_dict, dry_run_args.get('--basedir')) return try: args = docopt.docopt(usage, version=version, help=False) job_dict = copy.deepcopy(TEMPLATE_JOB) logging.root.setLevel(log_level(dry_run_args['--verbose'])) input_file_path = args.get('<inp>') or args.get('--inp-file') if input_file_path: basedir = os.path.dirname(os.path.abspath(input_file_path)) input_file = from_url(input_file_path) inputs = get_inputs(input_file, app.inputs, basedir) job_dict['inputs'].update(inputs) input_usage = job_dict['inputs'] if job: basedir = os.path.dirname(args.get('<tool>')) job.inputs = get_inputs(job.inputs, app.inputs, basedir) input_usage.update(job.inputs) app_inputs_usage = make_app_usage_string(app, template=TOOL_TEMPLATE, inp=input_usage) app_usage = make_app_usage_string(app, USAGE, job_dict['inputs']) try: app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>']) except docopt.DocoptExit: if not job: raise for inp in job.app.inputs: if inp.required and inp.id not in job.inputs: raise app_inputs = {} if args['--help']: print(app_usage) return # trim leading --, and ignore empty arays app_inputs = { k[2:]: v for k, v in six.iteritems(app_inputs) if v != [] } inp = get_inputs(app_inputs, app.inputs) if not job: job_dict['id'] = args.get('--outdir') or args.get('--dir') job_dict['app'] = app job = Job.from_dict(context, job_dict) job.inputs.update(inp) if args['--print-cli']: if not isinstance(app, CommandLineTool): fail(dry_run_args['<tool>'] + " is not a command line app") print(CLIJob(job).cmd_line()) return if args['--pretty-print']: fmt = partial(result_str, job.id) else: fmt = lambda result: json.dumps(context.to_primitive(result)) if not job.inputs and not args['--'] and not args['--quiet']: print(app_usage) return try: context.executor.execute(job, lambda _, result: print(fmt(result))) except RabixError as err: fail(err.message) except docopt.DocoptExit: fail(app_usage)
def test_tmap_mapall(): path = os.path.join(os.path.dirname(__file__), '../examples/tmap.yml') doc = from_url(path) tool, job = doc['mapall'], doc['exampleJob'] print(gen_cli(tool, job))
next = self.order.pop() return next, self.executables[next].job() def has_next(self): return len(self.order) > 0 @property def graph(self): return self.workflow.graph # Smoke test if __name__ == '__main__': from os.path import abspath, join from rabix.common.ref_resolver import from_url def root_relative(path): p = abspath(join(__file__, '../', path)) return abspath(join(__file__, '../../../', path)) doc = from_url(root_relative('tests/workflow.yml')) wf = Workflow(doc['workflows']['add_one_mul_two']['steps']) print(wf.graph.forw_topo_sort()) for edge in wf.graph.edges: print(wf.graph.describe_edge(edge)) print(wf.inputs) print(wf.outputs)
def main(): logging.basicConfig(level=logging.WARN) if len(sys.argv) == 1: print(USAGE) return usage = USAGE.format(resources=make_resources_usage_string(), inputs='<inputs>') app_usage = usage if len(sys.argv) == 2 and \ (sys.argv[1] == '--help' or sys.argv[1] == '-h'): print(USAGE) return dry_run_args = dry_run_parse() if not dry_run_args: print(USAGE) return if not (dry_run_args['<tool>']): print('You have to specify a tool, with --tool option') print(usage) return tool = get_tool(dry_run_args) if not tool: print("Couldn't find tool.") return fix_types(tool, dry_run_args.get('--type', 'CommandLine')) context = init_context() app = context.from_dict(tool) job = None if isinstance(app, Job): job = app app = job.app if dry_run_args['--install']: app.install() print("Install successful.") return if dry_run_args['--conformance-test']: job_dict = from_url(dry_run_args['<job>']) conformance_test(context, app, job_dict, dry_run_args.get('--basedir')) return try: args = docopt.docopt(usage, version=version, help=False) job_dict = copy.deepcopy(TEMPLATE_JOB) logging.root.setLevel(log_level(dry_run_args['--verbose'])) if args['--inp-file']: basedir = os.path.dirname(args.get('--inp-file')) input_file = from_url(args.get('--inp-file')) inputs = get_inputs(app, input_file, basedir) job_dict['inputs'].update(inputs) input_usage = job_dict['inputs'] if job: basedir = os.path.dirname(args.get('<tool>')) job.inputs = get_inputs(app, job.inputs, basedir) input_usage.update(job.inputs) app_inputs_usage = make_app_usage_string( app, template=TOOL_TEMPLATE, inp=input_usage) app_usage = make_app_usage_string(app, USAGE, job_dict['inputs']) try: app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>']) except docopt.DocoptExit: if not job: raise for inp in job.app.inputs: if inp.required and inp.id not in job.inputs: raise app_inputs = {} if args['--help']: print(app_usage) return # trim leading --, and ignore empty arays app_inputs = { k[2:]: v for k, v in six.iteritems(app_inputs) if v != [] } inp = get_inputs(app, app_inputs) if not job: job_dict['@id'] = args.get('--dir') job_dict['app'] = app job = Job.from_dict(context, job_dict) job.inputs.update(inp) if args['--print-cli']: if not isinstance(app, CliApp): print(dry_run_args['<tool>'] + " is not a command line app") return print(CLIJob(job).cmd_line()) return if not job.inputs and not args['--']: print(app_usage) return try: context.executor.execute(job, lambda _, result: print( result_str(job.id, result))) except RabixError as err: print(err.message) sys.exit(1) except docopt.DocoptExit: print(app_usage) sys.exit(1)