def split_job(job):
parallel_input = None
for input_name, input_val in six.iteritems(job.inputs):
io = job.app.get_input(input_name)
val_d = Executor.depth(input_val)
if val_d == io.depth:
continue
if val_d > io.depth + 1:
raise RabixError("Depth difference to large")
if val_d < io.depth:
raise RabixError("Insufficient dimensionality")
if parallel_input:
raise RabixError("Already parallelized by input '%%s'" % parallel_input)
parallel_input = input_name
if parallel_input:
jobs = []
for i, val in enumerate(job.inputs[parallel_input]):
inputs = copy.deepcopy(job.inputs)
inputs[parallel_input] = val
jobs.append(Job(job.id+"_"+six.text_type(i), job.app, inputs, {}, job.context))
return jobs
else:
return job
def conformance_test(context, app, job_dict, basedir):
job_dict['class'] = 'Job'
job_dict['id'] = basedir
job_dict['app'] = app
if not app.outputs:
app.outputs = []
job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
job = Job.from_dict(context, job_dict)
adapter = CLIJob(job)
result = {
'args': adapter.make_arg_list(),
'stdin': adapter.stdin,
'stdout': adapter.stdout,
}
cfr = app.get_requirement(CreateFileRequirement)
e = ValueResolver(job)
if cfr:
result['createfiles'] = {
name: content
for name, content in cfr.resolve_file_defs(e)
}
print(json.dumps(result))
def test_workflow():
path = abspath(join(__file__, '../../test_runtime/wf_tests.yaml'))
doc = from_url(path)
tests = doc['tests']
for test_name, test in six.iteritems(tests):
context = init_context(test['job'])
job = Job.from_dict(context, test['job'])
for inp in job.app.inputs:
construct_files(job.inputs.get(inp.id), inp.validator)
yield assert_execution, job, test['outputs']
def get_command_line():
data = request.get_json(force=True)
tool = process_builder(ctx, data['tool_cfg'])
inputs = {
strip_prefix(k): construct_files(v, tool._inputs[k].validator)
for k, v in data['input_map'].iteritems()
}
job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
cli_job = CLIJob(job)
return json.dumps(
{
'arguments': cli_job.make_arg_list(),
'stdin': cli_job.stdin,
'stdout': cli_job.stdout,
},
indent=2)
def get_outputs():
data = request.get_json(force=True)
tool = process_builder(ctx, data['tool_cfg'])
inputs = {
strip_prefix(k): construct_files(v, tool._inputs[k].validator)
for k, v in data['input_map'].iteritems()
}
job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
cli_job = CLIJob(job)
status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
else 'FAILURE'
return json.dumps(
{
'status': status,
'outputs': cli_job.get_outputs(data['job_dir'], job),
},
indent=2)
def conformance_test(context, app, job_dict, basedir):
job_dict['class'] = 'Job'
job_dict['id'] = basedir
job_dict['app'] = app
if not app.outputs:
app.outputs = []
job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
job = Job.from_dict(context, job_dict)
adapter = CLIJob(job)
print(json.dumps({
'args': adapter.make_arg_list(),
'stdin': adapter.stdin,
'stdout': adapter.stdout,
}))
def run(self, job, job_dir=None):
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
if self.container:
self.ensure_files(job, job_dir)
abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context)
self.install(job=job)
cmd_line = self.command_line(job, job_dir)
self.job_dump(job, job_dir)
self.container.run(cmd_line, job_dir)
result_path = os.path.abspath(job_dir) + '/result.cwl.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(
os.path.abspath(job_dir), abspath_job)
json.dump(outputs, res)
outputs = {
o.id: construct_files(outputs.get(o.id), o.validator)
for o in job.app.outputs
}
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def conformance_test(context, app, job_dict, basedir):
job_dict['class'] = 'Job'
job_dict['id'] = basedir
job_dict['app'] = app
if not app.outputs:
app.outputs = []
job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
job = Job.from_dict(context, job_dict)
adapter = CLIJob(job)
print(
json.dumps({
'args': adapter.make_arg_list(),
'stdin': adapter.stdin,
'stdout': adapter.stdout,
}))
def job(self):
return Job(None, self.app, self.inputs, {}, self.context)
def main():
disable_warnings()
logging.basicConfig(level=logging.WARN)
if len(sys.argv) == 1:
print(USAGE)
return
usage = USAGE.format(resources=make_resources_usage_string(),
inputs='<inputs>')
app_usage = usage
if len(sys.argv) == 2 and \
(sys.argv[1] == '--help' or sys.argv[1] == '-h'):
print(USAGE)
return
dry_run_args = dry_run_parse()
if not dry_run_args:
print(USAGE)
return
if not (dry_run_args['<tool>']):
print('You have to specify a tool, with --tool option')
print(usage)
return
tool = get_tool(dry_run_args)
if not tool:
fail("Couldn't find tool.")
if isinstance(tool, list):
tool = loader.index.get('#main')
if 'class' not in tool:
fail("Document must have a 'class' field")
if 'id' not in tool:
tool['id'] = dry_run_args['<tool>']
context = init_context(tool)
app = process_builder(context, tool)
job = None
if isinstance(app, Job):
job = app
app = job.app
rabix.expressions.update_engines(app)
if dry_run_args['--install']:
app.install()
print("Install successful.")
return
if dry_run_args['--conformance-test']:
job_dict = from_url(dry_run_args['<job>'])
conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
return
try:
args = docopt.docopt(usage, version=version, help=False)
job_dict = copy.deepcopy(TEMPLATE_JOB)
logging.root.setLevel(log_level(dry_run_args['--verbose']))
input_file_path = args.get('<inp>') or args.get('--inp-file')
if input_file_path:
basedir = os.path.dirname(os.path.abspath(input_file_path))
input_file = from_url(input_file_path)
inputs = get_inputs(input_file, app.inputs, basedir)
job_dict['inputs'].update(inputs)
input_usage = job_dict['inputs']
if job:
basedir = os.path.dirname(args.get('<tool>'))
job.inputs = get_inputs(job.inputs, app.inputs, basedir)
input_usage.update(job.inputs)
app_inputs_usage = make_app_usage_string(
app, template=TOOL_TEMPLATE, inp=input_usage)
app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])
try:
app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
except docopt.DocoptExit:
if not job:
raise
for inp in job.app.inputs:
if inp.required and inp.id not in job.inputs:
raise
app_inputs = {}
if args['--help']:
print(app_usage)
return
# trim leading --, and ignore empty arays
app_inputs = {
k[2:]: v
for k, v in six.iteritems(app_inputs)
if v != []
}
inp = get_inputs(app_inputs, app.inputs)
if not job:
job_dict['id'] = args.get('--outdir') or args.get('--dir')
job_dict['app'] = app
job = Job.from_dict(context, job_dict)
job.inputs.update(inp)
if args['--print-cli']:
if not isinstance(app, CommandLineTool):
fail(dry_run_args['<tool>'] + " is not a command line app")
print(CLIJob(job).cmd_line())
return
if args['--pretty-print']:
fmt = partial(result_str, job.id)
else:
fmt = lambda result: json.dumps(context.to_primitive(result))
if not job.inputs and not args['--'] and not args['--quiet']:
print(app_usage)
return
try:
context.executor.execute(job, lambda _, result: print(fmt(result)))
except RabixError as err:
fail(err.message)
except docopt.DocoptExit:
fail(app_usage)
def main():
logging.basicConfig(level=logging.WARN)
if len(sys.argv) == 1:
print(USAGE)
return
usage = USAGE.format(resources=make_resources_usage_string(),
inputs='<inputs>')
app_usage = usage
if len(sys.argv) == 2 and \
(sys.argv[1] == '--help' or sys.argv[1] == '-h'):
print(USAGE)
return
dry_run_args = dry_run_parse()
if not dry_run_args:
print(USAGE)
return
if not (dry_run_args['<tool>']):
print('You have to specify a tool, with --tool option')
print(usage)
return
tool = get_tool(dry_run_args)
if not tool:
print("Couldn't find tool.")
return
fix_types(tool, dry_run_args.get('--type', 'CommandLine'))
context = init_context()
app = context.from_dict(tool)
job = None
if isinstance(app, Job):
job = app
app = job.app
if dry_run_args['--install']:
app.install()
print("Install successful.")
return
if dry_run_args['--conformance-test']:
job_dict = from_url(dry_run_args['<job>'])
conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
return
try:
args = docopt.docopt(usage, version=version, help=False)
job_dict = TEMPLATE_JOB
logging.root.setLevel(log_level(dry_run_args['--verbose']))
if args['--inp-file']:
basedir = os.path.dirname(args.get('--inp-file'))
input_file = from_url(args.get('--inp-file'))
rebased = rebase_paths(app, input_file, basedir)
dot_update_dict(
job_dict['inputs'],
get_inputs(app, rebased)
)
input_usage = job_dict['inputs']
if job:
basedir = os.path.dirname(args.get('<tool>'))
rebased = rebase_paths(app, job.inputs, basedir)
dot_update_dict(job.inputs, rebased)
job.inputs.update(get_inputs(app, job.inputs))
input_usage.update(job.inputs)
app_inputs_usage = make_app_usage_string(
app, template=TOOL_TEMPLATE, inp=input_usage)
app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])
try:
app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
except docopt.DocoptExit:
if not job:
raise
for inp in job.app.inputs:
if inp.required and inp.id not in job.inputs:
raise
app_inputs = {}
if args['--help']:
print(app_usage)
return
inp = get_inputs(app, app_inputs)
if not job:
job_dict['inputs'].update(inp)
job_dict['@id'] = args.get('--dir')
job_dict['app'] = app
job = Job.from_dict(context, job_dict)
else:
job.inputs.update(inp)
if args['--print-cli']:
if not isinstance(app, CliApp):
print(dry_run_args['<tool>'] + " is not a command line app")
return
print(CLIJob(job).cmd_line())
return
if not job.inputs and not args['--']:
print(app_usage)
return
try:
context.executor.execute(job, lambda _, result: print(result))
except RabixError as err:
print(err.message)
sys.exit(1)
except docopt.DocoptExit:
print(app_usage)
sys.exit(1)
def main():
logging.basicConfig(level=logging.WARN)
if len(sys.argv) == 1:
print(USAGE)
return
usage = USAGE.format(resources=make_resources_usage_string(),
inputs='<inputs>')
app_usage = usage
if len(sys.argv) == 2 and \
(sys.argv[1] == '--help' or sys.argv[1] == '-h'):
print(USAGE)
return
dry_run_args = dry_run_parse()
if not dry_run_args:
print(USAGE)
return
if not (dry_run_args['<tool>']):
print('You have to specify a tool, with --tool option')
print(usage)
return
tool = get_tool(dry_run_args)
if not tool:
print("Couldn't find tool.")
return
fix_types(tool, dry_run_args.get('--type', 'CommandLine'))
context = init_context()
app = context.from_dict(tool)
job = None
if isinstance(app, Job):
job = app
app = job.app
if dry_run_args['--install']:
app.install()
print("Install successful.")
return
if dry_run_args['--conformance-test']:
job_dict = from_url(dry_run_args['<job>'])
conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
return
try:
args = docopt.docopt(usage, version=version, help=False)
job_dict = copy.deepcopy(TEMPLATE_JOB)
logging.root.setLevel(log_level(dry_run_args['--verbose']))
if args['--inp-file']:
basedir = os.path.dirname(args.get('--inp-file'))
input_file = from_url(args.get('--inp-file'))
inputs = get_inputs(app, input_file, basedir)
job_dict['inputs'].update(inputs)
input_usage = job_dict['inputs']
if job:
basedir = os.path.dirname(args.get('<tool>'))
job.inputs = get_inputs(app, job.inputs, basedir)
input_usage.update(job.inputs)
app_inputs_usage = make_app_usage_string(
app, template=TOOL_TEMPLATE, inp=input_usage)
app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])
try:
app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
except docopt.DocoptExit:
if not job:
raise
for inp in job.app.inputs:
if inp.required and inp.id not in job.inputs:
raise
app_inputs = {}
if args['--help']:
print(app_usage)
return
# trim leading --, and ignore empty arays
app_inputs = {
k[2:]: v
for k, v in six.iteritems(app_inputs)
if v != []
}
inp = get_inputs(app, app_inputs)
if not job:
job_dict['@id'] = args.get('--dir')
job_dict['app'] = app
job = Job.from_dict(context, job_dict)
job.inputs.update(inp)
if args['--print-cli']:
if not isinstance(app, CliApp):
print(dry_run_args['<tool>'] + " is not a command line app")
return
print(CLIJob(job).cmd_line())
return
if not job.inputs and not args['--']:
print(app_usage)
return
try:
context.executor.execute(job, lambda _, result: print(
result_str(job.id, result)))
except RabixError as err:
print(err.message)
sys.exit(1)
except docopt.DocoptExit:
print(app_usage)
sys.exit(1)
def main():
disable_warnings()
logging.basicConfig(level=logging.WARN)
if len(sys.argv) == 1:
print(USAGE)
return
usage = USAGE.format(resources=make_resources_usage_string(),
inputs='<inputs>')
app_usage = usage
if len(sys.argv) == 2 and \
(sys.argv[1] == '--help' or sys.argv[1] == '-h'):
print(USAGE)
return
dry_run_args = dry_run_parse()
if not dry_run_args:
print(USAGE)
return
if not (dry_run_args['<tool>']):
print('You have to specify a tool, with --tool option')
print(usage)
return
tool = get_tool(dry_run_args)
if not tool:
fail("Couldn't find tool.")
if isinstance(tool, list):
tool = loader.index.get('#main')
if 'class' not in tool:
fail("Document must have a 'class' field")
if 'id' not in tool:
tool['id'] = dry_run_args['<tool>']
context = init_context(tool)
app = process_builder(context, tool)
job = None
if isinstance(app, Job):
job = app
app = job.app
rabix.expressions.update_engines(app)
if dry_run_args['--install']:
app.install()
print("Install successful.")
return
if dry_run_args['--conformance-test']:
job_dict = from_url(dry_run_args['<job>'])
conformance_test(context, app, job_dict, dry_run_args.get('--basedir'))
return
try:
args = docopt.docopt(usage, version=version, help=False)
job_dict = copy.deepcopy(TEMPLATE_JOB)
logging.root.setLevel(log_level(dry_run_args['--verbose']))
input_file_path = args.get('<inp>') or args.get('--inp-file')
if input_file_path:
basedir = os.path.dirname(os.path.abspath(input_file_path))
input_file = from_url(input_file_path)
inputs = get_inputs(input_file, app.inputs, basedir)
job_dict['inputs'].update(inputs)
input_usage = job_dict['inputs']
if job:
basedir = os.path.dirname(args.get('<tool>'))
job.inputs = get_inputs(job.inputs, app.inputs, basedir)
input_usage.update(job.inputs)
app_inputs_usage = make_app_usage_string(app,
template=TOOL_TEMPLATE,
inp=input_usage)
app_usage = make_app_usage_string(app, USAGE, job_dict['inputs'])
try:
app_inputs = docopt.docopt(app_inputs_usage, args['<inputs>'])
except docopt.DocoptExit:
if not job:
raise
for inp in job.app.inputs:
if inp.required and inp.id not in job.inputs:
raise
app_inputs = {}
if args['--help']:
print(app_usage)
return
# trim leading --, and ignore empty arays
app_inputs = {
k[2:]: v
for k, v in six.iteritems(app_inputs) if v != []
}
inp = get_inputs(app_inputs, app.inputs)
if not job:
job_dict['id'] = args.get('--outdir') or args.get('--dir')
job_dict['app'] = app
job = Job.from_dict(context, job_dict)
job.inputs.update(inp)
if args['--print-cli']:
if not isinstance(app, CommandLineTool):
fail(dry_run_args['<tool>'] + " is not a command line app")
print(CLIJob(job).cmd_line())
return
if args['--pretty-print']:
fmt = partial(result_str, job.id)
else:
fmt = lambda result: json.dumps(context.to_primitive(result))
if not job.inputs and not args['--'] and not args['--quiet']:
print(app_usage)
return
try:
context.executor.execute(job, lambda _, result: print(fmt(result)))
except RabixError as err:
fail(err.message)
except docopt.DocoptExit:
fail(app_usage)
def run(self, job, job_dir=None):
self.load_input_content(job)
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
if not os.path.exists(job_dir):
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
eval = ValueResolver(job)
cfr = self.get_requirement_or_hint(CreateFileRequirement)
if cfr:
cfr.create_files(job_dir, eval)
env = None
evr = self.get_requirement_or_hint(EnvVarRequirement)
if evr:
env = evr.var_map(eval)
self.ensure_files(job, job_dir)
self.install(job=job)
abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context)
cmd_line = self.command_line(job, job_dir)
log.info("Running: %s" % cmd_line)
self.job_dump(job, job_dir)
if self.container:
self.container.run(cmd_line, job_dir, env)
else:
ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir)
if ret != 0:
raise RabixError("Command failed with exit status %s" % ret)
result_path = os.path.abspath(job_dir) + '/cwl.output.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(os.path.abspath(job_dir),
abspath_job)
json.dump(job.context.to_primitive(outputs), res)
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs