def get_outputs():
data = request.get_json(force=True)
tool = process_builder(ctx, data['tool_cfg'])
inputs = {strip_prefix(k): construct_files(v, tool._inputs[k].validator)
for k, v in data['input_map'].iteritems()}
job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
cli_job = CLIJob(job)
status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
else 'FAILURE'
return json.dumps({
'status': status,
'outputs': cli_job.get_outputs(data['job_dir'], job),
}, indent=2)
def get_outputs():
data = request.get_json(force=True)
tool = process_builder(ctx, data['tool_cfg'])
inputs = {
strip_prefix(k): construct_files(v, tool._inputs[k].validator)
for k, v in data['input_map'].iteritems()
}
job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
cli_job = CLIJob(job)
status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\
else 'FAILURE'
return json.dumps(
{
'status': status,
'outputs': cli_job.get_outputs(data['job_dir'], job),
},
indent=2)
def get_outputs():
data = request.get_json(force=True)
tool = process_builder(ctx, data['tool_cfg'])
inputs = {strip_prefix(k): construct_files(v, tool._inputs[strip_prefix(k)].validator) for k, v in data['input_map'].iteritems()}
job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx)
cli_job = CLIJob(job)
print data
# status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0]) else 'FAILURE'
status = 'SUCCESS'
outputs = cli_job.get_outputs(data['job_dir'], job)
if inputs:
key = inputs.keys()[0]
prefix = key[0:key.rfind('.')+1]
outputs = {prefix + k: v for k, v in outputs.iteritems()}
return json.dumps({
'status': status,
'outputs': outputs,
})
class CommandLineTool(Process):
WORKING_DIR = '/work'
def __init__(self,
process_id,
inputs,
outputs,
requirements,
hints,
label,
description,
base_command,
arguments=None,
stdin=None,
stdout=None):
super(CommandLineTool, self).__init__(process_id,
inputs,
outputs,
requirements=requirements,
hints=hints,
label=label,
description=description)
self.base_command = base_command
self.arguments = arguments or []
self.stdin = stdin
self.stdout = stdout
self.mappings = {}
self.cli_job = None
self._command_line = None
self.container = next(
(r for r in self.requirements if hasattr(r, 'run')),
next((r for r in self.hints if hasattr(r, 'run')), None))
def run(self, job, job_dir=None):
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
if self.container:
self.ensure_files(job, job_dir)
abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context)
self.install(job=job)
cmd_line = self.command_line(job, job_dir)
self.job_dump(job, job_dir)
self.container.run(cmd_line, job_dir)
result_path = os.path.abspath(job_dir) + '/result.cwl.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(
os.path.abspath(job_dir), abspath_job)
json.dump(outputs, res)
outputs = {
o.id: construct_files(outputs.get(o.id), o.validator)
for o in job.app.outputs
}
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def command_line(self, job, job_dir=None):
self.remap_paths(job.inputs, job_dir)
self._command_line = self.cli_job.cmd_line()
return self._command_line
def install(self, *args, **kwargs):
if self.container:
self.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.container:
self.container.ensure_files(job, job_dir)
def remap_paths(self, inputs, job_dir):
if self.container:
files = collect_files(inputs)
flatened = flatten_files(files)
paths = [os.path.dirname(f.path) for f in flatened] + [job_dir]
prefixes = collect_prefixes(paths)
self.mappings = self.container.get_mapping(prefixes)
for file in files:
file.remap(self.mappings)
def unmap_paths(self, outputs):
files = collect_files(outputs)
for file in files:
file.remap({v: k for k, v in six.iteritems(self.mappings)})
def to_dict(self, context=None):
d = super(CommandLineTool, self).to_dict(context)
d.update({
'class': 'CommandLineTool',
'baseCommand': self.base_command,
'arguments': self.arguments,
'stdin': self.stdin,
'stdout': self.stdout
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {k: context.from_dict(v) for k, v in six.iteritems(d)}
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'base_command':
converted['baseCommand'],
'arguments':
converted.get('arguments'),
'stdin':
converted.get('stdin'),
'stdout':
converted.get('stdout'),
'inputs': [
InputParameter.from_dict(context, inp)
for inp in converted.get('inputs', [])
],
'outputs': [
OutputParameter.from_dict(context, inp)
for inp in converted.get('outputs', [])
]
})
return cls(**kwargs)
class CommandLineTool(Process):
WORKING_DIR = '/work'
def __init__(
self, process_id, inputs, outputs, requirements, hints,
label, description, base_command, arguments=None,
stdin=None, stdout=None):
super(CommandLineTool, self).__init__(
process_id, inputs, outputs,
requirements=requirements,
hints=hints,
label=label,
description=description
)
self.base_command = base_command
self.arguments = arguments or []
self.stdin = stdin
self.stdout = stdout
self.mappings = {}
self.cli_job = None
self._command_line = None
self.container = next(
(r for r in self.requirements if hasattr(r, 'run')),
next((r for r in self.hints if hasattr(r, 'run')), None)
)
def run(self, job, job_dir=None):
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
os.mkdir(job_dir)
os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH |
stat.S_IWOTH)
self.cli_job = CLIJob(job)
if self.container:
self.ensure_files(job, job_dir)
abspath_job = Job(
job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context
)
self.install(job=job)
cmd_line = self.command_line(job, job_dir)
self.job_dump(job, job_dir)
self.container.run(cmd_line, job_dir)
result_path = os.path.abspath(job_dir) + '/result.cwl.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(
os.path.abspath(job_dir), abspath_job)
json.dump(outputs, res)
outputs = {o.id: construct_files(outputs.get(o.id), o.validator)
for o in job.app.outputs}
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def command_line(self, job, job_dir=None):
self.remap_paths(job.inputs, job_dir)
self._command_line = self.cli_job.cmd_line()
return self._command_line
def install(self, *args, **kwargs):
if self.container:
self.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.container:
self.container.ensure_files(job, job_dir)
def remap_paths(self, inputs, job_dir):
if self.container:
files = collect_files(inputs)
flatened = flatten_files(files)
paths = [os.path.dirname(f.path) for f in flatened] + [job_dir]
prefixes = collect_prefixes(paths)
self.mappings = self.container.get_mapping(prefixes)
for file in files:
file.remap(self.mappings)
def unmap_paths(self, outputs):
files = collect_files(outputs)
for file in files:
file.remap({v: k for k, v in six.iteritems(self.mappings)})
def to_dict(self, context=None):
d = super(CommandLineTool, self).to_dict(context)
d.update({
'class': 'CommandLineTool',
'baseCommand': self.base_command,
'arguments': self.arguments,
'stdin': self.stdin,
'stdout': self.stdout
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {k: context.from_dict(v) for k, v in six.iteritems(d)}
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'base_command': converted['baseCommand'],
'arguments': converted.get('arguments'),
'stdin': converted.get('stdin'),
'stdout': converted.get('stdout'),
'inputs': [InputParameter.from_dict(context, inp)
for inp in converted.get('inputs', [])],
'outputs': [OutputParameter.from_dict(context, inp)
for inp in converted.get('outputs', [])]
})
return cls(**kwargs)
class CommandLineTool(Process):
WORKING_DIR = '/work'
def __init__(
self, process_id, inputs, outputs, requirements, hints,
label, description, base_command, arguments=None,
stdin=None, stdout=None):
super(CommandLineTool, self).__init__(
process_id, inputs, outputs,
requirements=requirements,
hints=hints,
label=label,
description=description
)
self.base_command = base_command
self.arguments = [a if isinstance(a, dict) else {'valueFrom': a}
for a in (arguments or [])]
self.stdin = stdin
self.stdout = stdout
self.mappings = {}
self.cli_job = None
self._command_line = None
self.container = next(
(r for r in self.requirements if hasattr(r, 'run')),
next((r for r in self.hints if hasattr(r, 'run')), None)
)
def run(self, job, job_dir=None):
self.load_input_content(job)
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
if not os.path.exists(job_dir):
os.mkdir(job_dir)
os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH |
stat.S_IWOTH)
self.cli_job = CLIJob(job)
eval = ValueResolver(job)
cfr = self.get_requirement_or_hint(CreateFileRequirement)
if cfr:
cfr.create_files(job_dir, eval)
env = None
evr = self.get_requirement_or_hint(EnvVarRequirement)
if evr:
env = evr.var_map(eval)
self.ensure_files(job, job_dir)
self.install(job=job)
abspath_job = Job(
job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context
)
cmd_line = self.command_line(job, job_dir)
log.info("Running: %s" % cmd_line)
self.job_dump(job, job_dir)
if self.container:
self.container.run(cmd_line, job_dir, env)
else:
ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir)
if ret != 0:
raise RabixError("Command failed with exit status %s" % ret)
result_path = os.path.abspath(job_dir) + '/cwl.output.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(
os.path.abspath(job_dir), abspath_job)
json.dump(job.context.to_primitive(outputs), res)
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def command_line(self, job, job_dir=None):
self.remap_paths(job.inputs, job_dir)
self._command_line = self.cli_job.cmd_line()
return self._command_line
def install(self, *args, **kwargs):
if self.container:
self.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.container:
self.container.ensure_files(job, job_dir)
def remap_paths(self, inputs, job_dir):
if self.container:
files = collect_files(inputs)
flatened = flatten_files(files)
paths = [os.path.dirname(f.path) for f in flatened] + [job_dir]
prefixes = collect_prefixes(paths)
self.mappings = self.container.get_mapping(prefixes)
for file in files:
file.remap(self.mappings)
def unmap_paths(self, outputs):
files = collect_files(outputs)
for file in files:
file.remap({v: k for k, v in six.iteritems(self.mappings)})
def to_dict(self, context=None):
d = super(CommandLineTool, self).to_dict(context)
d.update({
'class': 'CommandLineTool',
'baseCommand': self.base_command,
'arguments': self.arguments,
'stdin': self.stdin,
'stdout': self.stdout
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {k: context.from_dict(v) for k, v in six.iteritems(d)}
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'base_command': converted['baseCommand'],
'arguments': converted.get('arguments'),
'stdin': converted.get('stdin'),
'stdout': converted.get('stdout'),
'inputs': [InputParameter.from_dict(context, inp)
for inp in converted.get('inputs', [])],
'outputs': [OutputParameter.from_dict(context, inp)
for inp in converted.get('outputs', [])]
})
return cls(**kwargs)
class CliApp(App):
WORKING_DIR = '/work'
def __init__(self, app_id, inputs, outputs,
app_description=None,
annotations=None,
platform_features=None,
adapter=None,
software_description=None,
requirements=None):
super(CliApp, self).__init__(
app_id, inputs, outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features
)
self.adapter = adapter
self.software_description = software_description
self.requirements = requirements
self.mappings = {}
self.cli_job = None
self._command_line = None
def run(self, job, job_dir=None):
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
os.mkdir(job_dir)
os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH |
stat.S_IWOTH)
self.cli_job = CLIJob(job)
if self.requirements.container:
self.ensure_files(job, job_dir)
abspath_job = copy.deepcopy(job)
self.install(job=job)
cmd_line = self.command_line(job, job_dir)
self.job_dump(job, job_dir)
self.requirements.container.run(cmd_line, job_dir)
result_path = os.path.abspath(job_dir) + '/result.cwl.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(
os.path.abspath(job_dir), abspath_job)
json.dump(outputs, res)
outputs = job.app.construct_outputs(outputs)
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def command_line(self, job, job_dir=None):
self.remap_paths(job.inputs, job_dir)
self._command_line = self.cli_job.cmd_line()
return self._command_line
def install(self, *args, **kwargs):
if self.requirements and self.requirements.container:
self.requirements.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.requirements and self.requirements.container:
self.requirements.container.ensure_files(job, job_dir)
def remap_paths(self, inputs, job_dir):
if self.requirements and self.requirements.container:
files = collect_files(inputs)
flatened = flatten_files(files)
paths = [os.path.dirname(f.path) for f in flatened] + [job_dir]
prefixes = collect_prefixes(paths)
self.mappings = self.requirements.container.get_mapping(prefixes)
for file in files:
file.remap(self.mappings)
def unmap_paths(self, outputs):
files = collect_files(outputs)
for file in files:
file.remap({v: k for k, v in six.iteritems(self.mappings)})
def to_dict(self, context=None):
d = super(CliApp, self).to_dict(context)
d.update({
"@type": "CommandLine",
'name': self.id,
'adapter': self.adapter,
'annotations': self.annotations,
'platform_features': self.platform_features,
'inputs': self.inputs.schema,
'outputs': self.outputs.schema,
'requirements': self.requirements.to_dict(context)
})
return d
@classmethod
def from_dict(cls, context, d):
return cls(d.get('@id', d.get('name')),
context.from_dict(d['inputs']),
context.from_dict(d.get('outputs', {})),
app_description=d.get('appDescription'),
annotations=d.get('annotations'),
platform_features=d.get('platform_features'),
adapter=context.from_dict(d.get('adapter')),
software_description=d.get('softwareDescription'),
requirements=Requirements.from_dict(
context, d.get('requirements', {})))
class CommandLineTool(Process):
WORKING_DIR = '/work'
def __init__(self,
process_id,
inputs,
outputs,
requirements,
hints,
label,
description,
base_command,
arguments=None,
stdin=None,
stdout=None):
super(CommandLineTool, self).__init__(process_id,
inputs,
outputs,
requirements=requirements,
hints=hints,
label=label,
description=description)
self.base_command = base_command
self.arguments = [
a if isinstance(a, dict) else {
'valueFrom': a
} for a in (arguments or [])
]
self.stdin = stdin
self.stdout = stdout
self.mappings = {}
self.cli_job = None
self._command_line = None
self.container = next(
(r for r in self.requirements if hasattr(r, 'run')),
next((r for r in self.hints if hasattr(r, 'run')), None))
def run(self, job, job_dir=None):
self.load_input_content(job)
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
if not os.path.exists(job_dir):
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
eval = ValueResolver(job)
cfr = self.get_requirement_or_hint(CreateFileRequirement)
if cfr:
cfr.create_files(job_dir, eval)
env = None
evr = self.get_requirement_or_hint(EnvVarRequirement)
if evr:
env = evr.var_map(eval)
self.ensure_files(job, job_dir)
self.install(job=job)
abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context)
cmd_line = self.command_line(job, job_dir)
log.info("Running: %s" % cmd_line)
self.job_dump(job, job_dir)
if self.container:
self.container.run(cmd_line, job_dir, env)
else:
ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir)
if ret != 0:
raise RabixError("Command failed with exit status %s" % ret)
result_path = os.path.abspath(job_dir) + '/cwl.output.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(os.path.abspath(job_dir),
abspath_job)
json.dump(job.context.to_primitive(outputs), res)
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
def command_line(self, job, job_dir=None):
self.remap_paths(job.inputs, job_dir)
self._command_line = self.cli_job.cmd_line()
return self._command_line
def install(self, *args, **kwargs):
if self.container:
self.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.container:
self.container.ensure_files(job, job_dir)
def remap_paths(self, inputs, job_dir):
if self.container:
files = collect_files(inputs)
flatened = flatten_files(files)
paths = [os.path.dirname(f.path) for f in flatened] + [job_dir]
prefixes = collect_prefixes(paths)
self.mappings = self.container.get_mapping(prefixes)
for file in files:
file.remap(self.mappings)
def unmap_paths(self, outputs):
files = collect_files(outputs)
for file in files:
file.remap({v: k for k, v in six.iteritems(self.mappings)})
def to_dict(self, context=None):
d = super(CommandLineTool, self).to_dict(context)
d.update({
'class': 'CommandLineTool',
'baseCommand': self.base_command,
'arguments': self.arguments,
'stdin': self.stdin,
'stdout': self.stdout
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {k: context.from_dict(v) for k, v in six.iteritems(d)}
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'base_command':
converted['baseCommand'],
'arguments':
converted.get('arguments'),
'stdin':
converted.get('stdin'),
'stdout':
converted.get('stdout'),
'inputs': [
InputParameter.from_dict(context, inp)
for inp in converted.get('inputs', [])
],
'outputs': [
OutputParameter.from_dict(context, inp)
for inp in converted.get('outputs', [])
]
})
return cls(**kwargs)
class CliApp(App):
WORKING_DIR = '/work'
def __init__(self,
app_id,
inputs,
outputs,
app_description=None,
annotations=None,
platform_features=None,
adapter=None,
software_description=None,
requirements=None):
super(CliApp, self).__init__(app_id,
inputs,
outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features)
self.adapter = adapter
self.software_description = software_description
self.requirements = requirements
self.cli_job = None
self._command_line = None
def run(self, job, job_dir=None):
job_dir = job_dir or job.id
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
if self.requirements.container:
self.ensure_files(job, job_dir)
abspath_job = copy.deepcopy(job)
self.install(job=job)
cmd_line = self.command_line(job, job_dir)
self.job_dump(job, job_dir)
self.requirements.container.run(cmd_line)
with open(os.path.abspath(job_dir) + '/result.cwl.json', 'w') as f:
outputs = self.cli_job.get_outputs(os.path.abspath(job_dir),
abspath_job)
json.dump(outputs, f)
for k, v in six.iteritems(outputs):
if isinstance(v, list):
for f in v:
with open(f['path'] + '.rbx.json', 'w') as rx:
json.dump(f, rx)
elif isinstance(v, dict):
with open(v['path'] + '.rbx.json', 'w') as rx:
json.dump(v, rx)
return outputs
def command_line(self, job, job_dir=None):
if not self._command_line:
self.set_config(job=job, job_dir=job_dir)
self._command_line = self.cli_job.cmd_line()
# print(self._command_line)
return self._command_line
def install(self, *args, **kwargs):
if self.requirements and self.requirements.container:
self.requirements.container.install(*args, **kwargs)
def ensure_files(self, job, job_dir):
if self.requirements and self.requirements.container:
self.requirements.container.ensure_files(job, job_dir)
def set_config(self, *args, **kwargs):
if self.requirements and self.requirements.container:
self.requirements.container.set_config(*args, **kwargs)
def to_dict(self, context=None):
d = super(CliApp, self).to_dict(context)
d.update({
"@type": "CommandLine",
'name': self.id,
'adapter': self.adapter,
'annotations': self.annotations,
'platform_features': self.platform_features,
'inputs': self.inputs.schema,
'outputs': self.outputs.schema,
'requirements': self.requirements.to_dict(context)
})
return d
@classmethod
def from_dict(cls, context, d):
return cls(d.get('@id', d.get('name')),
context.from_dict(d['inputs']),
context.from_dict(d.get('outputs', {})),
app_description=d.get('appDescription'),
annotations=d.get('annotations'),
platform_features=d.get('platform_features'),
adapter=context.from_dict(d.get('adapter')),
software_description=d.get('softwareDescription'),
requirements=Requirements.from_dict(
context, d.get('requirements', {})))