def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(
out_binding.get('outputEval'),
[r.to_dict() for r in res]
)
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res, list) else res
return result
def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [
File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files
]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(out_binding.get('outputEval'),
[r.to_dict() for r in res])
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res,
list) else res
return result
def __init__(self, job):
self.job = job
self.app = job.app
self._stdin = self.app.stdin
self._stdout = self.app.stdout
self.base_cmd = self.app.base_command
if isinstance(self.base_cmd, six.string_types):
self.base_cmd = self.base_cmd.split(' ')
self.args = self.app.arguments
self.eval = ValueResolver(job)
def conformance_test(context, app, job_dict, basedir):
job_dict['class'] = 'Job'
job_dict['id'] = basedir
job_dict['app'] = app
if not app.outputs:
app.outputs = []
job_dict['inputs'] = get_inputs(job_dict, app.inputs, basedir)
job = Job.from_dict(context, job_dict)
adapter = CLIJob(job)
result = {
'args': adapter.make_arg_list(),
'stdin': adapter.stdin,
'stdout': adapter.stdout,
}
cfr = app.get_requirement(CreateFileRequirement)
e = ValueResolver(job)
if cfr:
result['createfiles'] = {
name: content
for name, content in cfr.resolve_file_defs(e)
}
print(json.dumps(result))
def __init__(self, job):
self.job = job
self.app = job.app
self._stdin = self.app.stdin
self._stdout = self.app.stdout
self.base_cmd = self.app.base_command
if isinstance(self.base_cmd, six.string_types):
self.base_cmd = self.base_cmd.split(' ')
self.args = self.app.arguments
self.eval = ValueResolver(job)
class CLIJob(object):
def __init__(self, job):
self.job = job
self.app = job.app
self._stdin = self.app.stdin
self._stdout = self.app.stdout
self.base_cmd = self.app.base_command
if isinstance(self.base_cmd, six.string_types):
self.base_cmd = self.base_cmd.split(' ')
self.args = self.app.arguments
self.eval = ValueResolver(job)
@property
def stdin(self):
if self._stdin:
return self.eval.resolve(self._stdin)
@property
def stdout(self):
if self._stdout:
return self.eval.resolve(self._stdout)
def make_arg_list(self):
adapters = [InputAdapter(a.get('valueFrom'), self.eval, {}, a)
for a in self.args]
ia = InputAdapter(self.job.inputs, self.eval, self.app.inputs)
args = ia.as_toplevel(adapters)
base_cmd = [self.eval.resolve(item) for item in self.base_cmd]
return [six.text_type(arg) for arg in base_cmd + args]
def cmd_line(self):
a = self.make_arg_list()
if self._stdin:
a += ['<', self.stdin]
if self._stdout:
a += ['>', self.eval.resolve(self.stdout)]
return ' '.join(a) # TODO: escape
@staticmethod
def glob_or(pattern):
"""
>>> CLIJob.glob_or("simple")
['simple']
>>> CLIJob.glob_or("{option1,option2}")
['option1', 'option2']
:param pattern:
:return:
"""
if re.match('^\{[^,]+(,[^,]+)*\}$', pattern):
return pattern.strip('{}').split(',')
return [pattern]
def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(
out_binding.get('outputEval'),
[r.to_dict() for r in res]
)
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res, list) else res
return result
def run(self, job, job_dir=None):
self.load_input_content(job)
job_dir = os.path.abspath(job_dir or job.id)
if not job_dir.endswith('/'):
job_dir += '/'
if not os.path.exists(job_dir):
os.mkdir(job_dir)
os.chmod(job_dir,
os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH)
self.cli_job = CLIJob(job)
eval = ValueResolver(job)
cfr = self.get_requirement_or_hint(CreateFileRequirement)
if cfr:
cfr.create_files(job_dir, eval)
env = None
evr = self.get_requirement_or_hint(EnvVarRequirement)
if evr:
env = evr.var_map(eval)
self.ensure_files(job, job_dir)
self.install(job=job)
abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs),
job.allocated_resources, job.context)
cmd_line = self.command_line(job, job_dir)
log.info("Running: %s" % cmd_line)
self.job_dump(job, job_dir)
if self.container:
self.container.run(cmd_line, job_dir, env)
else:
ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir)
if ret != 0:
raise RabixError("Command failed with exit status %s" % ret)
result_path = os.path.abspath(job_dir) + '/cwl.output.json'
if os.path.exists(result_path):
with open(result_path, 'r') as res:
outputs = json.load(res)
else:
with open(result_path, 'w') as res:
outputs = self.cli_job.get_outputs(os.path.abspath(job_dir),
abspath_job)
json.dump(job.context.to_primitive(outputs), res)
self.unmap_paths(outputs)
def write_rbx(f):
if isinstance(f, File):
with open(f.path + '.rbx.json', 'w') as rbx:
json.dump(f.to_dict(), rbx)
map_rec_collection(write_rbx, outputs)
return outputs
class CLIJob(object):
def __init__(self, job):
self.job = job
self.app = job.app
self._stdin = self.app.stdin
self._stdout = self.app.stdout
self.base_cmd = self.app.base_command
if isinstance(self.base_cmd, six.string_types):
self.base_cmd = self.base_cmd.split(' ')
self.args = self.app.arguments
self.eval = ValueResolver(job)
@property
def stdin(self):
if self._stdin:
return self.eval.resolve(self._stdin)
@property
def stdout(self):
if self._stdout:
return self.eval.resolve(self._stdout)
def make_arg_list(self):
adapters = [
InputAdapter(a.get('valueFrom'), self.eval, {}, a)
for a in self.args
]
ia = InputAdapter(self.job.inputs, self.eval, self.app.inputs)
args = ia.as_toplevel(adapters)
base_cmd = [self.eval.resolve(item) for item in self.base_cmd]
return [six.text_type(arg) for arg in base_cmd + args]
def cmd_line(self):
a = self.make_arg_list()
if self._stdin:
a += ['<', self.stdin]
if self._stdout:
a += ['>', self.eval.resolve(self.stdout)]
return ' '.join(a) # TODO: escape
@staticmethod
def glob_or(pattern):
"""
>>> CLIJob.glob_or("simple")
['simple']
>>> CLIJob.glob_or("{option1,option2}")
['option1', 'option2']
:param pattern:
:return:
"""
if re.match('^\{[^,]+(,[^,]+)*\}$', pattern):
return pattern.strip('{}').split(',')
return [pattern]
def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [
File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files
]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(out_binding.get('outputEval'),
[r.to_dict() for r in res])
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res,
list) else res
return result
