def test_nested_pipes_stream_setup_stream_multiplex(): tool = jip.find('joined_pipeline') assert tool is not None p = jip.Pipeline() p.run(tool, input=["Makefile", "README.rst", "setup.py"], output="${input}.dat") p.expand(validate=False) # 2 nodes 1 edge assert len(p) == 6 assert len(p.edges) == 3 t1_0 = p.get("TestJob1.0") t2_0 = p.get("TestJob2.0") assert t1_0.has_outgoing(t2_0, ('output', 'input'), True) t1_1 = p.get("TestJob1.1") t2_1 = p.get("TestJob2.1") assert t1_1.has_outgoing(t2_1, ('output', 'input'), True) t1_2 = p.get("TestJob1.2") t2_2 = p.get("TestJob2.2") assert t1_2.has_outgoing(t2_2, ('output', 'input'), True) # test option values cwd = os.getcwd() join = os.path.join assert t1_0.input == join(cwd, 'Makefile') assert t1_1.input == join(cwd, 'README.rst') assert t1_2.input == join(cwd, 'setup.py') assert t2_0.output == join(cwd, 'Makefile.dat') assert t2_1.output == join(cwd, 'README.rst.dat') assert t2_2.output == join(cwd, 'setup.py.dat')
def test_tool_class_decorator_options_for_functions(): data = [] validated = [] @jip.pytool() class mytool(object): """ usage: tool <name> """ def validate(self): validated.append(True) def run(self): # in call implementation, we are not a tool instance assert isinstance(self, mytool) # but we have the helpers assert hasattr(self, 'args') assert hasattr(self, 'options') assert hasattr(self, 'tool_instance') assert hasattr(self, 'check_file') assert hasattr(self, 'ensure') assert hasattr(self, 'validation_error') data.append(self.options['name'].get()) t = find('mytool') assert t assert t.options['name'] is not None t.options['name'] = 'data.txt' t.validate() t.run() assert data[0] == 'data.txt' assert validated[0]
def test_bash_tool_options(): bash = find("bash") assert bash.options is not None assert len(bash.options) == 5 assert bash.options['cmd'] is not None assert bash.options['input'] is not None assert bash.options['output'] is not None
def testPipelineStructure(self): # load the pipeline tool = jip.find("examples/bwa/pileup.jip") assert tool is not None # create a pipeline p = jip.Pipeline() # create a new pipeline node and configure id p.run(tool, input="setup.py", reference="Makefile", output="out.txt") # expand the pipeline such that the internal pipeline is resolved p.expand(validate=False) # after expansion with this setuo, the pipeline should have 7 nodes assert len(p) == 7 # the graph should consist of 6 edges assert len(p.edges) == 6 # get out the nodes. we have to use indexes here # because the names might have changed after expansion ref = p.get("ref") align = p.get("align") sam = p.get("sam") bam = p.get("bam") dups = p.get("dups") index = p.get("index") pileup = p.get("pileup") # check the connections assert not ref.has_incoming() assert align.has_incoming(ref) assert sam.has_incoming(align) assert bam.has_incoming(sam) assert dups.has_incoming(bam) assert index.has_incoming(dups) assert pileup.has_incoming(index) assert not pileup.has_outgoing()
def test_tool_hidden_option(): @jip.tool("test") class TestTool(object): """\Test tool Usage: test [-a <opt_a>] [-b <opt_b>] [-c] -i <input> -o <output> Inputs: -i, --input <input> The input Outputs: -o, --output <output> The output Options: -a <opt_a> Option a -b <opt_b> Option b -c Options c (a flag) """ def setup(self): self.opts['output'].hidden = True def get_command(self): return '${options()} ${output|arg(">")}' tool = find('test') tool.parse_args(['-i', 'input.txt', '-a', 'A', '-c', '-o', 'output.txt']) assert tool.get_command() == ('bash', '-a A -c -i input.txt -o output.txt >output.txt')
def test_tool_decorator_options_for_functions(): data = [] @jip.pytool() def mytool(self): """ usage: tool -i <input> options: -i, --input <input> The input """ assert isinstance(self, jip.tools.Tool) # but we have the helpers assert hasattr(self, 'args') assert hasattr(self, 'options') #assert hasattr(self, 'tool_instance') assert hasattr(self, 'check_file') assert hasattr(self, 'ensure') assert hasattr(self, 'validation_error') data.append(self.options['input'].get()) t = find('mytool') assert t assert t.options['input'] is not None t.options['input'] = 'data.txt' t.run() assert data[0] == 'data.txt'
def test_tool_decorator_delegate(): @jip.tool("test_delegates") class MyTool(object): """\ usage: delegate [-i <input>] """ def validate(self): return "delegated_validate:%s" % self.options['i'].get() def is_done(self): return "delegated_is_done:%s" % self.options['i'].get() def pipeline(self): return "delegated_pipeline:%s" % self.options['i'].get() def cleanup(self): return "delegated_cleanup:%s" % self.options['i'].get() def help(self): return "delegated_help:%s" % self.options['i'].get() def get_command(self): return 'inter', "delegated_cmd:%s" % self.options['i'].get() test_tool = find("test_delegates") test_tool.parse_args(["-i", "infile.txt"]) assert test_tool is not None assert test_tool.validate() == "delegated_validate:infile.txt" assert test_tool.is_done() == "delegated_is_done:infile.txt" assert test_tool.pipeline() == "delegated_pipeline:infile.txt" assert test_tool.cleanup() == "delegated_cleanup:infile.txt" assert test_tool.help() == "delegated_help:infile.txt" assert test_tool.get_command() == ('inter', "delegated_cmd:infile.txt")
def testPipelineStructureMultiplexed(self): # load the pipeline tool = jip.find("examples/bwa/pileup.jip") assert tool is not None # create a pipeline p = jip.Pipeline() # create a new pipeline node and configure id p.run(tool, input=["setup.py", "README.rst"], reference="Makefile", output="${input|ext}_out.txt") # expand the pipeline such that the internal pipeline is resolved # this will also validate all nodes and raise an exception # if one of the nodes validations failed p.expand(validate=False) # after expansion with this setuo, the pipeline should have 7 nodes assert len(p) == 13 # the graph should consist of 6 edges assert len(p.edges) == 12 # get out the nodes. we have to use indexes here # because the names might have changed after expansion ref = p.get("ref") align = p.get("align.0") sam = p.get("sam.0") bam = p.get("bam.0") dups = p.get("dups.0") index = p.get("index.0") pileup = p.get("pileup.0") # check the connections assert not ref.has_incoming() assert align.has_incoming(ref) assert sam.has_incoming(align) assert bam.has_incoming(sam) assert dups.has_incoming(bam) assert index.has_incoming(dups) assert pileup.has_incoming(index) assert not pileup.has_outgoing() # test second set ref = p.get("ref") align = p.get("align.1") sam = p.get("sam.1") bam = p.get("bam.1") dups = p.get("dups.1") index = p.get("index.1") pileup = p.get("pileup.1") # check the connections assert not ref.has_incoming() assert align.has_incoming(ref) assert sam.has_incoming(align) assert bam.has_incoming(sam) assert dups.has_incoming(bam) assert index.has_incoming(dups) assert pileup.has_incoming(index) assert not pileup.has_outgoing()
def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError, e: print >>sys.stderr, str(e) sys.exit(1)
def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError, e: print >> sys.stderr, str(e) sys.exit(1)
def test_tool_setup_called_on_init(): @jip.tool() class setup_tool(object): def init(self): self.add_output('output') def get_comand(self): return "true" t = jip.find('setup_tool') assert t.options['output'] is not None
def test_nested_pipes_stream_setup_intermediate(): tool = jip.find('joined_pipeline') assert tool is not None p = jip.Pipeline() p.run(tool, input="Makefile", output="out.txt", inter="inter.out") p.expand() # 2 nodes 1 edge assert len(p) == 2 assert len(p.edges) == 1 t1 = p.get("TestJob1") t2 = p.get("TestJob2") assert t1.has_outgoing(t2, ('output', 'input'), False)
def test_nested_pipes_stream_setup_stream_jobs(): tool = jip.find('joined_pipeline') assert tool is not None p = jip.Pipeline() p.run(tool, input="Makefile", output="out.txt") jobs = jip.create_jobs(p) groups = jip.create_groups(jobs) cwd = os.getcwd() join = os.path.join assert len(groups) == 1 assert len(jobs) == 2 assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile') assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
def test_render_boolean_option(): @jip.tool("simple") class SimpleTool(object): """\ Test tool Usage: tool [-t] Options: -t A bool option """ pass tool = jip.find("simple") assert render_template("${t|arg}", tool=tool, test="1") == "" tool.parse_args(['-t']) assert render_template("${t|arg}", tool=tool, test="1") == "-t"
def test_tool_decorator_options_for_templates(): @jip.tool() def mytool(): """ usage: tool -i <input> options: -i, --input <input> The input """ return "run on ${input}" t = find('mytool') assert t assert t.options['input'] is not None t.options['input'] = 'data.txt' assert t.get_command()[1] == 'run on data.txt'
def test_render_value_option(): @jip.tool("simple") class SimpleTool(object): """\ Test tool Usage: tool [-t <in>] Options: -t <in> A value option """ pass tool = jip.find("simple") tool.parse_args(["-t", "infile"]) assert render_template("${t|arg}", tool=tool, test="1") == "-t infile" assert render_template("${t|arg('-o ')}", tool=tool, test="1") == "-o infile"
def test_node_options_with_assignment(): tool = jip.find('joined_pipeline') assert tool is not None p = jip.Pipeline() n = p.run(tool) n.input = "Makefile" n.output = "out.txt" n.inter = "inter.out" assert n._tool.options['input'].get() == "Makefile" p.expand() # 2 nodes 1 edge assert len(p) == 2 assert len(p.edges) == 1 t1 = p.get("TestJob1") t2 = p.get("TestJob2") assert t1.has_outgoing(t2, ('output', 'input'), False)
def tool(self): """Get the tool instance that is associated with this job. If the tool is not set, it will be loaded using the :py:func:`jip.find` function """ if not self._tool: try: from jip import find self._tool = find(self.tool_name if self.path is None else self.path) for opt in self.configuration: if opt.name in self._tool.options: self._tool.options[opt.name]._value = opt._value else: self._tool.options[opt.name] = opt except: log.error("Unable to reload tool: %s", self.tool_name, exc_info=True) return self._tool
def test_tool_property_names(): @jip.pytool() class mytool(object): """ usage: mytool --input --no-hash """ def validate(self): assert hasattr(self, 'input') assert hasattr(self, 'no_hash') def run(self): pass t = find('mytool') assert t.options["input"] is not None assert t.options["no_hash"] is not None t.validate()
def test_tool_decorator_delegate_validate_to_external_method(): def my_validate(self): return "delegated_validate:%s" % self.options['i'].get() def my_is_done(self): return "delegated_is_done:%s" % self.options['i'].get() def my_pipeline(self): return "delegated_pipeline:%s" % self.options['i'].get() def my_cleanup(self): return "delegated_cleanup:%s" % self.options['i'].get() def my_help(self): return "delegated_help:%s" % self.options['i'].get() def my_get_command(self): return 'inter', "delegated_cmd:%s" % self.options['i'].get() @jip.tool("test_delegates", validate=my_validate, pipeline=my_pipeline, get_command=my_get_command, cleanup=my_cleanup, is_done=my_is_done, help=my_help) class MyTool(object): """\ usage: delegate [-i <input>] """ test_tool = find("test_delegates") test_tool.parse_args(["-i", "infile.txt"]) assert test_tool is not None assert test_tool.validate() == "delegated_validate:infile.txt" assert test_tool.is_done() == "delegated_is_done:infile.txt" assert test_tool.pipeline() == "delegated_pipeline:infile.txt" assert test_tool.cleanup() == "delegated_cleanup:infile.txt" assert test_tool.help() == "delegated_help:infile.txt" assert test_tool.get_command() == ('inter', "delegated_cmd:infile.txt")
def test_node_options_with_assignment_jobs(): tool = jip.find('joined_pipeline') assert tool is not None p = jip.Pipeline() n = p.run(tool) n.input = "Makefile" n.output = "out.txt" n.inter = "inter.out" jobs = jip.create_jobs(p) groups = jip.create_groups(jobs) cwd = os.getcwd() join = os.path.join assert len(groups) == 2 assert len(jobs) == 2 assert jobs[0].configuration['input'].get() == join(cwd, 'Makefile') assert jobs[0].configuration['output'].get() == join(cwd, 'inter.out') assert jobs[1].configuration['input'].get() == join(cwd, 'inter.out') assert jobs[1].configuration['output'].get() == join(cwd, 'out.txt')
def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError as e: print(str(e), file=sys.stderr) sys.exit(1) # disable required checks jip.options._check_required = False profile = jip.profiles.get(name='default') jobs = jip.jobs.create_jobs(script, args=script_args, profile=profile) specs = {} default_env = os.environ env_exckludes = [ "JIP_MODULES", "JIP_LOGLEVEL", "JIP_PATH", "JIP_DB_LOGLEVEL" ] for j in jobs: job_env = {} for k, v in iteritems(j.env): if not k in env_exckludes and v and v != default_env.get(k, None): job_env[k] = v spec = sorted_dict({ "threads": j.threads, "mem": j.max_memory, "queue": j.queue, "priority": j.priority, "time": j.max_time, "account": j.account, "extra": j.extra, "env": job_env }) specs[j.name] = spec print(json.dumps({"jobs": specs}, indent=4, sort_keys=False))
def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError as e: print(str(e), file=sys.stderr) sys.exit(1) # load profile profile = jip.profiles.get( name='default' if not args['--profile'] else args['--profile']) profile.tool_name = script.name if args['--spec']: spec_prof = jip.profiles.Profile.from_file(args['--spec']) spec_prof.update(profile) profile = spec_prof profile.load_args(args) log.info("Profile: %s", profile) if args['--dry'] or args['--show']: # we handle --dry and --show separatly, # create the jobs and call the show commands jobs = jip.jobs.create_jobs(script, args=script_args, profile=profile) error = None try: jip.jobs.check_output_files(jobs) jip.jobs.check_queued_jobs(jobs) except Exception as err: error = err if args['--dry']: show_dry(jobs, options=script.options, profiles=True) if args['--show']: show_commands(jobs) if error: print("%s\n" % colorize("Validation error!", RED), file=sys.stderr) print(str(error), file=sys.stderr) sys.exit(1) return ##################################################### # prepare jobs for submission ##################################################### force = args['--force'] jobs = jip.jobs.create_jobs(script, args=script_args, keep=args['--keep'], profile=profile, profiler=args['--with-profiler']) if len(jobs) == 0: return if args['--hold']: ##################################################### # Only save the jobs and let them stay on hold ##################################################### jip.db.save(jobs) print("Jobs stored and put on hold") else: try: ##################################################### # Iterate the executions and submit ##################################################### for exe in jip.jobs.create_executions(jobs, save=True, check_outputs=not force, check_queued=not force): if exe.completed and not force: print(colorize("Skipping %s" % exe.name, YELLOW)) else: if jip.jobs.submit_job(exe.job, force=force): print("Submitted %s with remote id %s" % (exe.job.id, exe.job.job_id)) except Exception as err: log.debug("Submission error: %s", err, exc_info=True) print(colorize("Error while submitting job:", RED), colorize(str(err), RED), file=sys.stderr) ################################################## # delete all submitted jobs ################################################## jip.jobs.delete(jobs, clean_logs=True)
def main(): args = parse_args(__doc__, options_first=True) print "Tools scripts" print "-------------" print "Please note that there might be more. Here, we search only for" print "files with the .jip extension!" print "" print "Search paths:" print "Current directory: %s" % getcwd() print "Jip configuration: %s" % jip.config.get("jip_path", "") print "JIP_PATH variable: %s" % getenv("JIP_PATH", "") print "" rows = [] for name, path in jip.scanner.scan_files().iteritems(): rows.append((name, path)) print render_table(["Name", "Path"], rows) print "" print "Tools implemented in Python modules" print "-----------------------------------" print "The modules must be available in PYTHONPATH and must be specified" print "in the jip configuration or in the JIP_MODULES environment" print "variable. Please note that pipeline scripts that contain" print "python blocks are allowed to load modules that contain tool" print "implementation. These tools might not be found by this scan!" print "" print "Jip configuration: %s" % jip.config.get("jip_modules", "") print "JIP_MODULES variable: %s" % getenv("JIP_MODULES", "") print "" rows = [] jip.scanner.scan_modules() for name, cls in jip.scanner.registry.iteritems(): help = cls.help() description = "-" if help is not None: description = help.split("\n")[0] if len(description) > 60: description = "%s ..." % description[:46] rows.append((name, description)) print render_table(["Tool", "Description"], rows) print "" print "All Tools detected" print "------------------" print "" covered = set([]) rows = [] for name, p in jip.scanner.scan().iteritems(): if name in covered: continue covered.add(name) cls = jip.find(name) help = cls.help() description = "-" if help is not None: description = help.split("\n")[0] if len(description) > 60: description = "%s ..." % description[:46] rows.append((cls.name, description)) print render_table(["Tool", "Description"], rows)
def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file, is_pipeline=args['--pipeline']) except LookupError as e: print(str(e), file=sys.stderr) sys.exit(1) if args['--dry'] or args['--show']: dry(script, script_args, dry=args['--dry'], show=args['--show']) return keep = args['--keep'] force = args['--force'] profiler = args['--with-profiler'] silent = not args['--status'] try: profile = jip.profiles.Profile() if args['--spec']: spec_prof = jip.profiles.Profile.from_file(args['--spec']) spec_prof.update(profile) profile = spec_prof profile.load_args(args) jobs = jip.jobs.create_jobs(script, args=script_args, keep=keep, profile=profile) # assign job ids for i, j in enumerate(jobs): j.id = i + 1 for exe in jip.jobs.create_executions(jobs): if exe.completed and not force: if not silent: print(colorize("Skipping", YELLOW), exe.name, file=sys.stderr) else: if not silent: print(colorize("Running", YELLOW), " {name:30} ".format(name=colorize(exe.name, BLUE)), file=sys.stderr) sys.stderr.flush() start = datetime.now() success = jip.jobs.run_job(exe.job, profiler=profiler) end = timedelta(seconds=(datetime.now() - start).seconds) if success: if not silent: print(colorize(exe.job.state, GREEN), "[%s]" % end, file=sys.stderr) else: if not silent: print(colorize(exe.job.state, RED), file=sys.stderr) sys.exit(1) except (jip.ValidationError, jip.ParserException) as va: print(str(va), file=sys.stderr) sys.exit(1) except Exception as va: raise