def run(self, job, job_dir=None): job_dir = job_dir or job.id os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.requirements.container: self.ensure_files(job, job_dir) abspath_job = copy.deepcopy(job) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.requirements.container.run(cmd_line) with open(os.path.abspath(job_dir) + '/result.cwl.json', 'w') as f: outputs = self.cli_job.get_outputs(os.path.abspath(job_dir), abspath_job) json.dump(outputs, f) for k, v in six.iteritems(outputs): if isinstance(v, list): for f in v: with open(f['path'] + '.rbx.json', 'w') as rx: json.dump(f, rx) elif isinstance(v, dict): with open(v['path'] + '.rbx.json', 'w') as rx: json.dump(v, rx) return outputs
def get_command_line(): data = request.get_json(force=True) tool = process_builder(ctx, data['tool_cfg']) inputs = {strip_prefix(k): construct_files(v, tool._inputs[k].validator) for k, v in data['input_map'].iteritems()} job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx) cli_job = CLIJob(job) return json.dumps({ 'arguments': cli_job.make_arg_list(), 'stdin': cli_job.stdin, 'stdout': cli_job.stdout, }, indent=2)
def get_outputs(): data = request.get_json(force=True) tool = process_builder(ctx, data['tool_cfg']) inputs = {strip_prefix(k): construct_files(v, tool._inputs[k].validator) for k, v in data['input_map'].iteritems()} job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx) cli_job = CLIJob(job) status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\ else 'FAILURE' return json.dumps({ 'status': status, 'outputs': cli_job.get_outputs(data['job_dir'], job), }, indent=2)
def get_command_line(): data = request.get_json(force=True) tool = process_builder(ctx, data['tool_cfg']) inputs = { strip_prefix(k): construct_files(v, tool._inputs[k].validator) for k, v in data['input_map'].iteritems() } job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx) cli_job = CLIJob(job) return json.dumps( { 'arguments': cli_job.make_arg_list(), 'stdin': cli_job.stdin, 'stdout': cli_job.stdout, }, indent=2)
def get_outputs(): data = request.get_json(force=True) tool = process_builder(ctx, data['tool_cfg']) inputs = { strip_prefix(k): construct_files(v, tool._inputs[k].validator) for k, v in data['input_map'].iteritems() } job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx) cli_job = CLIJob(job) status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0])\ else 'FAILURE' return json.dumps( { 'status': status, 'outputs': cli_job.get_outputs(data['job_dir'], job), }, indent=2)
def get_outputs(): data = request.get_json(force=True) tool = process_builder(ctx, data['tool_cfg']) inputs = {strip_prefix(k): construct_files(v, tool._inputs[strip_prefix(k)].validator) for k, v in data['input_map'].iteritems()} job = Job('Fake job ID', tool, inputs, {'cpu': 1, 'mem': 1024}, ctx) cli_job = CLIJob(job) print data # status = 'SUCCESS' if data['exit_code'] in data['tool_cfg'].get('successCodes', [0]) else 'FAILURE' status = 'SUCCESS' outputs = cli_job.get_outputs(data['job_dir'], job) if inputs: key = inputs.keys()[0] prefix = key[0:key.rfind('.')+1] outputs = {prefix + k: v for k, v in outputs.iteritems()} return json.dumps({ 'status': status, 'outputs': outputs, })
def run(self, job, job_dir=None): job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.container: self.ensure_files(job, job_dir) abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.container.run(cmd_line, job_dir) result_path = os.path.abspath(job_dir) + '/result.cwl.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(outputs, res) outputs = { o.id: construct_files(outputs.get(o.id), o.validator) for o in job.app.outputs } self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs
def run(self, job, job_dir=None): job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.container: self.ensure_files(job, job_dir) abspath_job = Job( job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context ) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.container.run(cmd_line, job_dir) result_path = os.path.abspath(job_dir) + '/result.cwl.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(outputs, res) outputs = {o.id: construct_files(outputs.get(o.id), o.validator) for o in job.app.outputs} self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs
class CommandLineTool(Process): WORKING_DIR = '/work' def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, base_command, arguments=None, stdin=None, stdout=None): super(CommandLineTool, self).__init__(process_id, inputs, outputs, requirements=requirements, hints=hints, label=label, description=description) self.base_command = base_command self.arguments = arguments or [] self.stdin = stdin self.stdout = stdout self.mappings = {} self.cli_job = None self._command_line = None self.container = next( (r for r in self.requirements if hasattr(r, 'run')), next((r for r in self.hints if hasattr(r, 'run')), None)) def run(self, job, job_dir=None): job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.container: self.ensure_files(job, job_dir) abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.container.run(cmd_line, job_dir) result_path = os.path.abspath(job_dir) + '/result.cwl.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(outputs, res) outputs = { o.id: construct_files(outputs.get(o.id), o.validator) for o in job.app.outputs } self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs def command_line(self, job, job_dir=None): self.remap_paths(job.inputs, job_dir) self._command_line = self.cli_job.cmd_line() return self._command_line def install(self, *args, **kwargs): if self.container: self.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.container: self.container.ensure_files(job, job_dir) def remap_paths(self, inputs, job_dir): if self.container: files = collect_files(inputs) flatened = flatten_files(files) paths = [os.path.dirname(f.path) for f in flatened] + [job_dir] prefixes = collect_prefixes(paths) self.mappings = self.container.get_mapping(prefixes) for file in files: file.remap(self.mappings) def unmap_paths(self, outputs): files = collect_files(outputs) for file in files: file.remap({v: k for k, v in six.iteritems(self.mappings)}) def to_dict(self, context=None): d = super(CommandLineTool, self).to_dict(context) d.update({ 'class': 'CommandLineTool', 'baseCommand': self.base_command, 'arguments': self.arguments, 'stdin': self.stdin, 'stdout': self.stdout }) return d @classmethod def from_dict(cls, context, d): converted = {k: context.from_dict(v) for k, v in six.iteritems(d)} kwargs = Process.kwarg_dict(converted) kwargs.update({ 'base_command': converted['baseCommand'], 'arguments': converted.get('arguments'), 'stdin': converted.get('stdin'), 'stdout': converted.get('stdout'), 'inputs': [ InputParameter.from_dict(context, inp) for inp in converted.get('inputs', []) ], 'outputs': [ OutputParameter.from_dict(context, inp) for inp in converted.get('outputs', []) ] }) return cls(**kwargs)
class CommandLineTool(Process): WORKING_DIR = '/work' def __init__( self, process_id, inputs, outputs, requirements, hints, label, description, base_command, arguments=None, stdin=None, stdout=None): super(CommandLineTool, self).__init__( process_id, inputs, outputs, requirements=requirements, hints=hints, label=label, description=description ) self.base_command = base_command self.arguments = arguments or [] self.stdin = stdin self.stdout = stdout self.mappings = {} self.cli_job = None self._command_line = None self.container = next( (r for r in self.requirements if hasattr(r, 'run')), next((r for r in self.hints if hasattr(r, 'run')), None) ) def run(self, job, job_dir=None): job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.container: self.ensure_files(job, job_dir) abspath_job = Job( job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context ) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.container.run(cmd_line, job_dir) result_path = os.path.abspath(job_dir) + '/result.cwl.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(outputs, res) outputs = {o.id: construct_files(outputs.get(o.id), o.validator) for o in job.app.outputs} self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs def command_line(self, job, job_dir=None): self.remap_paths(job.inputs, job_dir) self._command_line = self.cli_job.cmd_line() return self._command_line def install(self, *args, **kwargs): if self.container: self.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.container: self.container.ensure_files(job, job_dir) def remap_paths(self, inputs, job_dir): if self.container: files = collect_files(inputs) flatened = flatten_files(files) paths = [os.path.dirname(f.path) for f in flatened] + [job_dir] prefixes = collect_prefixes(paths) self.mappings = self.container.get_mapping(prefixes) for file in files: file.remap(self.mappings) def unmap_paths(self, outputs): files = collect_files(outputs) for file in files: file.remap({v: k for k, v in six.iteritems(self.mappings)}) def to_dict(self, context=None): d = super(CommandLineTool, self).to_dict(context) d.update({ 'class': 'CommandLineTool', 'baseCommand': self.base_command, 'arguments': self.arguments, 'stdin': self.stdin, 'stdout': self.stdout }) return d @classmethod def from_dict(cls, context, d): converted = {k: context.from_dict(v) for k, v in six.iteritems(d)} kwargs = Process.kwarg_dict(converted) kwargs.update({ 'base_command': converted['baseCommand'], 'arguments': converted.get('arguments'), 'stdin': converted.get('stdin'), 'stdout': converted.get('stdout'), 'inputs': [InputParameter.from_dict(context, inp) for inp in converted.get('inputs', [])], 'outputs': [OutputParameter.from_dict(context, inp) for inp in converted.get('outputs', [])] }) return cls(**kwargs)
def run(self, job, job_dir=None): self.load_input_content(job) job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' if not os.path.exists(job_dir): os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) eval = ValueResolver(job) cfr = self.get_requirement_or_hint(CreateFileRequirement) if cfr: cfr.create_files(job_dir, eval) env = None evr = self.get_requirement_or_hint(EnvVarRequirement) if evr: env = evr.var_map(eval) self.ensure_files(job, job_dir) self.install(job=job) abspath_job = Job( job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context ) cmd_line = self.command_line(job, job_dir) log.info("Running: %s" % cmd_line) self.job_dump(job, job_dir) if self.container: self.container.run(cmd_line, job_dir, env) else: ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir) if ret != 0: raise RabixError("Command failed with exit status %s" % ret) result_path = os.path.abspath(job_dir) + '/cwl.output.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(job.context.to_primitive(outputs), res) self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs
class CommandLineTool(Process): WORKING_DIR = '/work' def __init__( self, process_id, inputs, outputs, requirements, hints, label, description, base_command, arguments=None, stdin=None, stdout=None): super(CommandLineTool, self).__init__( process_id, inputs, outputs, requirements=requirements, hints=hints, label=label, description=description ) self.base_command = base_command self.arguments = [a if isinstance(a, dict) else {'valueFrom': a} for a in (arguments or [])] self.stdin = stdin self.stdout = stdout self.mappings = {} self.cli_job = None self._command_line = None self.container = next( (r for r in self.requirements if hasattr(r, 'run')), next((r for r in self.hints if hasattr(r, 'run')), None) ) def run(self, job, job_dir=None): self.load_input_content(job) job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' if not os.path.exists(job_dir): os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) eval = ValueResolver(job) cfr = self.get_requirement_or_hint(CreateFileRequirement) if cfr: cfr.create_files(job_dir, eval) env = None evr = self.get_requirement_or_hint(EnvVarRequirement) if evr: env = evr.var_map(eval) self.ensure_files(job, job_dir) self.install(job=job) abspath_job = Job( job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context ) cmd_line = self.command_line(job, job_dir) log.info("Running: %s" % cmd_line) self.job_dump(job, job_dir) if self.container: self.container.run(cmd_line, job_dir, env) else: ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir) if ret != 0: raise RabixError("Command failed with exit status %s" % ret) result_path = os.path.abspath(job_dir) + '/cwl.output.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(job.context.to_primitive(outputs), res) self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs def command_line(self, job, job_dir=None): self.remap_paths(job.inputs, job_dir) self._command_line = self.cli_job.cmd_line() return self._command_line def install(self, *args, **kwargs): if self.container: self.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.container: self.container.ensure_files(job, job_dir) def remap_paths(self, inputs, job_dir): if self.container: files = collect_files(inputs) flatened = flatten_files(files) paths = [os.path.dirname(f.path) for f in flatened] + [job_dir] prefixes = collect_prefixes(paths) self.mappings = self.container.get_mapping(prefixes) for file in files: file.remap(self.mappings) def unmap_paths(self, outputs): files = collect_files(outputs) for file in files: file.remap({v: k for k, v in six.iteritems(self.mappings)}) def to_dict(self, context=None): d = super(CommandLineTool, self).to_dict(context) d.update({ 'class': 'CommandLineTool', 'baseCommand': self.base_command, 'arguments': self.arguments, 'stdin': self.stdin, 'stdout': self.stdout }) return d @classmethod def from_dict(cls, context, d): converted = {k: context.from_dict(v) for k, v in six.iteritems(d)} kwargs = Process.kwarg_dict(converted) kwargs.update({ 'base_command': converted['baseCommand'], 'arguments': converted.get('arguments'), 'stdin': converted.get('stdin'), 'stdout': converted.get('stdout'), 'inputs': [InputParameter.from_dict(context, inp) for inp in converted.get('inputs', [])], 'outputs': [OutputParameter.from_dict(context, inp) for inp in converted.get('outputs', [])] }) return cls(**kwargs)
class CliApp(App): WORKING_DIR = '/work' def __init__(self, app_id, inputs, outputs, app_description=None, annotations=None, platform_features=None, adapter=None, software_description=None, requirements=None): super(CliApp, self).__init__( app_id, inputs, outputs, app_description=app_description, annotations=annotations, platform_features=platform_features ) self.adapter = adapter self.software_description = software_description self.requirements = requirements self.mappings = {} self.cli_job = None self._command_line = None def run(self, job, job_dir=None): job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.requirements.container: self.ensure_files(job, job_dir) abspath_job = copy.deepcopy(job) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.requirements.container.run(cmd_line, job_dir) result_path = os.path.abspath(job_dir) + '/result.cwl.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs( os.path.abspath(job_dir), abspath_job) json.dump(outputs, res) outputs = job.app.construct_outputs(outputs) self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs def command_line(self, job, job_dir=None): self.remap_paths(job.inputs, job_dir) self._command_line = self.cli_job.cmd_line() return self._command_line def install(self, *args, **kwargs): if self.requirements and self.requirements.container: self.requirements.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.requirements and self.requirements.container: self.requirements.container.ensure_files(job, job_dir) def remap_paths(self, inputs, job_dir): if self.requirements and self.requirements.container: files = collect_files(inputs) flatened = flatten_files(files) paths = [os.path.dirname(f.path) for f in flatened] + [job_dir] prefixes = collect_prefixes(paths) self.mappings = self.requirements.container.get_mapping(prefixes) for file in files: file.remap(self.mappings) def unmap_paths(self, outputs): files = collect_files(outputs) for file in files: file.remap({v: k for k, v in six.iteritems(self.mappings)}) def to_dict(self, context=None): d = super(CliApp, self).to_dict(context) d.update({ "@type": "CommandLine", 'name': self.id, 'adapter': self.adapter, 'annotations': self.annotations, 'platform_features': self.platform_features, 'inputs': self.inputs.schema, 'outputs': self.outputs.schema, 'requirements': self.requirements.to_dict(context) }) return d @classmethod def from_dict(cls, context, d): return cls(d.get('@id', d.get('name')), context.from_dict(d['inputs']), context.from_dict(d.get('outputs', {})), app_description=d.get('appDescription'), annotations=d.get('annotations'), platform_features=d.get('platform_features'), adapter=context.from_dict(d.get('adapter')), software_description=d.get('softwareDescription'), requirements=Requirements.from_dict( context, d.get('requirements', {})))
def run(self, job, job_dir=None): self.load_input_content(job) job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' if not os.path.exists(job_dir): os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) eval = ValueResolver(job) cfr = self.get_requirement_or_hint(CreateFileRequirement) if cfr: cfr.create_files(job_dir, eval) env = None evr = self.get_requirement_or_hint(EnvVarRequirement) if evr: env = evr.var_map(eval) self.ensure_files(job, job_dir) self.install(job=job) abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context) cmd_line = self.command_line(job, job_dir) log.info("Running: %s" % cmd_line) self.job_dump(job, job_dir) if self.container: self.container.run(cmd_line, job_dir, env) else: ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir) if ret != 0: raise RabixError("Command failed with exit status %s" % ret) result_path = os.path.abspath(job_dir) + '/cwl.output.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs(os.path.abspath(job_dir), abspath_job) json.dump(job.context.to_primitive(outputs), res) self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs
class CommandLineTool(Process): WORKING_DIR = '/work' def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, base_command, arguments=None, stdin=None, stdout=None): super(CommandLineTool, self).__init__(process_id, inputs, outputs, requirements=requirements, hints=hints, label=label, description=description) self.base_command = base_command self.arguments = [ a if isinstance(a, dict) else { 'valueFrom': a } for a in (arguments or []) ] self.stdin = stdin self.stdout = stdout self.mappings = {} self.cli_job = None self._command_line = None self.container = next( (r for r in self.requirements if hasattr(r, 'run')), next((r for r in self.hints if hasattr(r, 'run')), None)) def run(self, job, job_dir=None): self.load_input_content(job) job_dir = os.path.abspath(job_dir or job.id) if not job_dir.endswith('/'): job_dir += '/' if not os.path.exists(job_dir): os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) eval = ValueResolver(job) cfr = self.get_requirement_or_hint(CreateFileRequirement) if cfr: cfr.create_files(job_dir, eval) env = None evr = self.get_requirement_or_hint(EnvVarRequirement) if evr: env = evr.var_map(eval) self.ensure_files(job, job_dir) self.install(job=job) abspath_job = Job(job.id, job.app, copy.deepcopy(job.inputs), job.allocated_resources, job.context) cmd_line = self.command_line(job, job_dir) log.info("Running: %s" % cmd_line) self.job_dump(job, job_dir) if self.container: self.container.run(cmd_line, job_dir, env) else: ret = subprocess.call(['bash', '-c', cmd_line], cwd=job_dir) if ret != 0: raise RabixError("Command failed with exit status %s" % ret) result_path = os.path.abspath(job_dir) + '/cwl.output.json' if os.path.exists(result_path): with open(result_path, 'r') as res: outputs = json.load(res) else: with open(result_path, 'w') as res: outputs = self.cli_job.get_outputs(os.path.abspath(job_dir), abspath_job) json.dump(job.context.to_primitive(outputs), res) self.unmap_paths(outputs) def write_rbx(f): if isinstance(f, File): with open(f.path + '.rbx.json', 'w') as rbx: json.dump(f.to_dict(), rbx) map_rec_collection(write_rbx, outputs) return outputs def command_line(self, job, job_dir=None): self.remap_paths(job.inputs, job_dir) self._command_line = self.cli_job.cmd_line() return self._command_line def install(self, *args, **kwargs): if self.container: self.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.container: self.container.ensure_files(job, job_dir) def remap_paths(self, inputs, job_dir): if self.container: files = collect_files(inputs) flatened = flatten_files(files) paths = [os.path.dirname(f.path) for f in flatened] + [job_dir] prefixes = collect_prefixes(paths) self.mappings = self.container.get_mapping(prefixes) for file in files: file.remap(self.mappings) def unmap_paths(self, outputs): files = collect_files(outputs) for file in files: file.remap({v: k for k, v in six.iteritems(self.mappings)}) def to_dict(self, context=None): d = super(CommandLineTool, self).to_dict(context) d.update({ 'class': 'CommandLineTool', 'baseCommand': self.base_command, 'arguments': self.arguments, 'stdin': self.stdin, 'stdout': self.stdout }) return d @classmethod def from_dict(cls, context, d): converted = {k: context.from_dict(v) for k, v in six.iteritems(d)} kwargs = Process.kwarg_dict(converted) kwargs.update({ 'base_command': converted['baseCommand'], 'arguments': converted.get('arguments'), 'stdin': converted.get('stdin'), 'stdout': converted.get('stdout'), 'inputs': [ InputParameter.from_dict(context, inp) for inp in converted.get('inputs', []) ], 'outputs': [ OutputParameter.from_dict(context, inp) for inp in converted.get('outputs', []) ] }) return cls(**kwargs)
class CliApp(App): WORKING_DIR = '/work' def __init__(self, app_id, inputs, outputs, app_description=None, annotations=None, platform_features=None, adapter=None, software_description=None, requirements=None): super(CliApp, self).__init__(app_id, inputs, outputs, app_description=app_description, annotations=annotations, platform_features=platform_features) self.adapter = adapter self.software_description = software_description self.requirements = requirements self.cli_job = None self._command_line = None def run(self, job, job_dir=None): job_dir = job_dir or job.id os.mkdir(job_dir) os.chmod(job_dir, os.stat(job_dir).st_mode | stat.S_IROTH | stat.S_IWOTH) self.cli_job = CLIJob(job) if self.requirements.container: self.ensure_files(job, job_dir) abspath_job = copy.deepcopy(job) self.install(job=job) cmd_line = self.command_line(job, job_dir) self.job_dump(job, job_dir) self.requirements.container.run(cmd_line) with open(os.path.abspath(job_dir) + '/result.cwl.json', 'w') as f: outputs = self.cli_job.get_outputs(os.path.abspath(job_dir), abspath_job) json.dump(outputs, f) for k, v in six.iteritems(outputs): if isinstance(v, list): for f in v: with open(f['path'] + '.rbx.json', 'w') as rx: json.dump(f, rx) elif isinstance(v, dict): with open(v['path'] + '.rbx.json', 'w') as rx: json.dump(v, rx) return outputs def command_line(self, job, job_dir=None): if not self._command_line: self.set_config(job=job, job_dir=job_dir) self._command_line = self.cli_job.cmd_line() # print(self._command_line) return self._command_line def install(self, *args, **kwargs): if self.requirements and self.requirements.container: self.requirements.container.install(*args, **kwargs) def ensure_files(self, job, job_dir): if self.requirements and self.requirements.container: self.requirements.container.ensure_files(job, job_dir) def set_config(self, *args, **kwargs): if self.requirements and self.requirements.container: self.requirements.container.set_config(*args, **kwargs) def to_dict(self, context=None): d = super(CliApp, self).to_dict(context) d.update({ "@type": "CommandLine", 'name': self.id, 'adapter': self.adapter, 'annotations': self.annotations, 'platform_features': self.platform_features, 'inputs': self.inputs.schema, 'outputs': self.outputs.schema, 'requirements': self.requirements.to_dict(context) }) return d @classmethod def from_dict(cls, context, d): return cls(d.get('@id', d.get('name')), context.from_dict(d['inputs']), context.from_dict(d.get('outputs', {})), app_description=d.get('appDescription'), annotations=d.get('annotations'), platform_features=d.get('platform_features'), adapter=context.from_dict(d.get('adapter')), software_description=d.get('softwareDescription'), requirements=Requirements.from_dict( context, d.get('requirements', {})))