def start(self): logger.info("Calling `start`") self._setup_options() self.setup_options() self.options = self.parser.parse_args() self.args = self.options.files self._print_version() # Check to make sure some executors have been specified if we are # actually going to run: if self.options.execute: noExecSpecified(self.options.num_exec) self._setup_pipeline(self.options) self._setup_directories() self.appName = self.setup_appName() self.setup_logger() # TODO this doesn't capture environment variables # or contents of any config file so isn't really complete self.reconstructCommand() pbs_submit = self.options.queue_type == "pbs" \ and not self.options.local # --create-graph causes the pipeline to be constructed # both at PBS submit time and on the grid; this may be an extremely # expensive duplication if (self.options.execute and not pbs_submit) or self.options.create_graph: logger.debug("Calling `run`") self.run() logger.debug("Calling `initialize`") self.pipeline.initialize() self.pipeline.printStages(self.options.pipeline_name) if self.options.create_graph: logger.debug("Writing dot file...") nx.write_dot(self.pipeline.G, str(self.options.pipeline_name) + "_labeled-tree.dot") logger.debug("Done.") if not self.options.execute: print("Not executing the command (--no-execute is specified).\nDone.") return if pbs_submit: roq = runOnQueueingSystem(self.options, sys.argv) roq.createAndSubmitPbsScripts() logger.info("Finished submitting PBS job scripts...quitting") return #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified) # if use_ns is specified, Pyro NameServer must be started. logger.info("Starting pipeline daemon...") pipelineDaemon(self.pipeline, self.options, sys.argv[0]) logger.info("Server has stopped. Quitting...")
def test_roq_init_defaults(self, setupopts): """Makes sure that pbs related defaults are correctly initialized""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") assert roq.queue == "pbs" assert roq.arguments == None assert roq.jobName == "pydpiper" assert roq.time == "2:00:00:00"
def test_roq_init_defaults(self, setupopts): """Makes sure that pbs related defaults are correctly initialized""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") assert roq.queue == "pbs" assert roq.arguments == None assert roq.jobName == "pydpiper" assert roq.time == "2:00:00:00"
def test_create_job_0exec(self, setupopts): """Test defaults and file creation when no executors are specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 0: pytest.skip("This test will not run if --num-executors is specified") roq.createPbsScripts() assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True
def start(self): logger.info("Calling `start`") self._setup_options() self.setup_options() self.options = self.parser.parse_args() self.args = self.options.files self._print_version() #Check to make sure some executors have been specified. noExecSpecified(self.options.num_exec) self._setup_pipeline(self.options) self._setup_directories() self.appName = self.setup_appName() self.setup_logger() # NB this doesn't capture environment variables # or contents of any config file so isn't really complete self.reconstructCommand() pbs_submit = (self.options.queue == "pbs" or \ self.options.queue_type == "pbs") \ and not self.options.local if (self.options.execute and not pbs_submit) or self.options.create_graph: logger.debug("Calling `run`") self.run() logger.debug("Calling `initialize`") self.pipeline.initialize() self.pipeline.printStages(self.options.pipeline_name) if self.options.create_graph: logger.debug("Writing dot file...") nx.write_dot(self.pipeline.G, "labeled-tree.dot") logger.debug("Done.") if not self.options.execute: print "Not executing the command (--no-execute is specified).\nDone." return if pbs_submit: roq = runOnQueueingSystem(self.options, sys.argv) roq.createAndSubmitPbsScripts() logger.info("Finished submitting PBS job scripts...quitting") return #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified) # if use_ns is specified, Pyro NameServer must be started. logger.info("Starting pipeline daemon...") pipelineDaemon(self.pipeline, self.options, sys.argv[0]) logger.info("Server has stopped. Quitting...")
def test_create_job_0exec(self, setupopts): """Test defaults and file creation when no executors are specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 0: pytest.skip( "This test will not run if --num-executors is specified") roq.createPbsScripts() assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True
def start(self): self._setup_options() self.setup_options() self.options, self.args = self.parser.parse_args() self._print_version() #Check to make sure some executors have been specified. noExecSpecified(self.options.num_exec) self._setup_pipeline() self._setup_directories() self.appName = self.setup_appName() self.setup_logger() if self.options.queue=="pbs": roq = runOnQueueingSystem(self.options, sys.argv) roq.createPbsScripts() return if self.options.restart: print "\nThe restart option is deprecated (pipelines are not pickled anymore, because it takes too much time). Will restart based on which files exists already\n" #logger.info("Restarting pipeline from pickled files.") #self.pipeline.restart() self.reconstructCommand() self.run() self.pipeline.initialize() self.pipeline.printStages(self.appName) else: self.reconstructCommand() self.run() self.pipeline.initialize() self.pipeline.printStages(self.appName) if self.options.create_graph: logger.debug("Writing dot file...") nx.write_dot(self.pipeline.G, "labeled-tree.dot") logger.debug("Done.") if not self.options.execute: print "Not executing the command (--no-execute is specified).\nDone." return #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified) # if use_ns is specified, Pyro NameServer must be started. logger.info("Starting pipeline daemon...") pipelineDaemon(self.pipeline, self.options, sys.argv[0]) logger.info("Server has stopped. Quitting...")
def start(self): self._setup_options() self.setup_options() self.options, self.args = self.parser.parse_args() self._print_version() #Check to make sure some executors have been specified. noExecSpecified(self.options.num_exec) self._setup_pipeline() self._setup_directories() self.appName = self.setup_appName() self.setup_logger() if self.options.queue == "pbs": roq = runOnQueueingSystem(self.options, sys.argv) roq.createPbsScripts() return if self.options.restart: logger.info("Restarting pipeline from pickled files.") self.pipeline.restart() self.pipeline.initialize() self.pipeline.printStages(self.appName) else: self.reconstructCommand() self.run() self.pipeline.initialize() self.pipeline.printStages(self.appName) if self.options.create_graph: logger.debug("Writing dot file...") nx.write_dot(self.pipeline.G, "labeled-tree.dot") logger.debug("Done.") if not self.options.execute: print "Not executing the command (--no-execute is specified).\nDone." return #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified) # if use_ns is specified, Pyro NameServer must be started. logger.info("Starting pipeline daemon...") pipelineDaemon(self.pipeline, self.options, sys.argv[0]) logger.info("Server has stopped. Quitting...")
def test_create_job_1exec(self, setupopts): """This test requires specifying --num-executors=1 on command line Verifies defaults and file creation when only one executor is specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createPbsScripts() assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True callsExec = False jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True assert callsExec == True
def test_create_job_1exec(self, setupopts): """This test requires specifying --num-executors=1 on command line Verifies defaults and file creation when only one executor is specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createPbsScripts() assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True callsExec = False jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True assert callsExec == True
def test_create_job_multiproc(self, setupopts): """This test requires specifying --proc=24 on command line Verifies defaults and file creation when multiple nodes are needed""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.ppn != 8: pytest.skip("specify --ppn=8 to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") if roq.proc != 24: pytest.skip("specify --proc==24 to continue with this test") roq.createPbsScripts() correctNodesPpn = False jobFileString = open(roq.jobFileName).read() if "nodes=3:ppn=8" in jobFileString: correctNodesPpn = True assert correctNodesPpn == True
def test_create_job_multiproc(self, setupopts): """This test requires specifying --proc=24 on command line Verifies defaults and file creation when multiple nodes are needed""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.ppn != 8: pytest.skip("specify --ppn=8 to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") if roq.proc != 24: pytest.skip("specify --proc==24 to continue with this test") roq.createPbsScripts() correctNodesPpn = False jobFileString = open(roq.jobFileName).read() if "nodes=3:ppn=8" in jobFileString: correctNodesPpn = True assert correctNodesPpn == True
def start(self): self._setup_options() self.setup_options() self.options, self.args = self.parser.parse_args() self._setup_pipeline() self._setup_directories() self.appName = self.setup_appName() self.setup_logger() if self.options.queue=="pbs": roq = runOnQueueingSystem(self.options, sys.argv) roq.createPbsScripts() return if self.options.restart: logger.info("Restarting pipeline from pickled files.") self.pipeline.restart() self.pipeline.initialize() self.pipeline.printStages(self.appName) else: self.reconstructCommand() self.run() self.pipeline.initialize() self.pipeline.printStages(self.appName) if self.options.create_graph: logger.debug("Writing dot file...") nx.write_dot(self.pipeline.G, "labeled-tree.dot") logger.debug("Done.") if not self.options.execute: print "Not executing the command (--no-execute is specified).\nDone." return #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified) # if use_ns is specified, Pyro NameServer must be started. logger.info("Starting pipeline daemon...") pipelineDaemon(self.pipeline, self.options, sys.argv[0]) logger.info("Server has stopped. Quitting...")
def test_create_job_with_args(self, setupopts): """This test verifies the appropriate script creation for a sample set of arguments.""" allOptions = setupopts.returnAllOptions() sampleArgs = setupopts.returnSampleArgs() roq = runOnQueueingSystem(allOptions, sysArgs=sampleArgs) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createPbsScripts() assert roq.jobName == "TestProgName.py" mncFilesIncluded = False sleepIncluded = False jobFileString = open(roq.jobFileName).read() if "img_A.mnc" and "img_B.mnc" in jobFileString: mncFilesIncluded = True if "sleep 60" in jobFileString: sleepIncluded = True assert mncFilesIncluded == True assert sleepIncluded == True
def test_create_job_with_args(self, setupopts): """This test verifies the appropriate script creation for a sample set of arguments.""" allOptions = setupopts.returnAllOptions() sampleArgs = setupopts.returnSampleArgs() roq = runOnQueueingSystem(allOptions, sysArgs=sampleArgs) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createPbsScripts() assert roq.jobName == "TestProgName.py" mncFilesIncluded = False sleepIncluded = False jobFileString = open(roq.jobFileName).read() if "img_A.mnc" and "img_B.mnc" in jobFileString: mncFilesIncluded = True if "sleep 60" in jobFileString: sleepIncluded = True assert mncFilesIncluded == True assert sleepIncluded == True
def test_create_executor_only(self, setupopts): allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createExecutorJobFile(1) assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True correctFileName = False callsExec = False correctName = False if "pydpiper-executor-1" in roq.jobFileName: correctFileName = True jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True if "pydpiper-executor" in jobFileString: correctName = True assert correctFileName == True assert callsExec == True assert correctName == True
def test_create_executor_only(self, setupopts): allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") roq.createExecutorJobFile(1) assert roq.jobName == "pydpiper" assert isfile(roq.jobFileName) == True correctFileName = False callsExec = False correctName = False if "pydpiper-executor-1" in roq.jobFileName: correctFileName = True jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True if "pydpiper-executor" in jobFileString: correctName = True assert correctFileName == True assert callsExec == True assert correctName == True
def test_create_job_newppn(self, setupopts): """This test requires specifying --ppn=10, --num-executors=1 and --proc=24 Verifies appropriate node selection""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.ppn != 10: pytest.skip("specify --ppn=10 to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") if roq.proc != 24: pytest.skip("specify --proc==24 to continue with this test") roq.createPbsScripts() roq.createPbsScripts() correctPpn = False correctProc = False jobFileString = open(roq.jobFileName).read() if "nodes=2:ppn=10" in jobFileString: correctPpn = True if "--proc=24" in jobFileString: correctProc = True assert correctPpn == True assert correctProc == True
def test_create_job_newppn(self, setupopts): """This test requires specifying --ppn=10, --num-executors=1 and --proc=24 Verifies appropriate node selection""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.ppn != 10: pytest.skip("specify --ppn=10 to continue with this test") if roq.numexec != 1: pytest.skip("specify --num-executors=1 to continue with this test") if roq.proc != 24: pytest.skip("specify --proc==24 to continue with this test") roq.createPbsScripts() roq.createPbsScripts() correctPpn = False correctProc = False jobFileString = open(roq.jobFileName).read() if "nodes=2:ppn=10" in jobFileString: correctPpn=True if "--proc=24" in jobFileString: correctProc = True assert correctPpn == True assert correctProc == True
def test_create_job_3execs(self, setupopts): """This test requires specifying --num-executors=3 on command line Verifies defaults and file creation when three executors are specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 3: pytest.skip("specify --num-executors=3 to continue with this test") roq.createPbsScripts() # The following assert statements apply to the final .job file created correctFileName = False callsExec = False correctName = False if "pydpiper-executor-2" in roq.jobFileName: correctFileName = True jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True if "pydpiper-executor" in jobFileString: correctName = True assert correctFileName == True assert callsExec == True assert correctName == True
def test_create_job_3execs(self, setupopts): """This test requires specifying --num-executors=3 on command line Verifies defaults and file creation when three executors are specified""" allOptions = setupopts.returnAllOptions() roq = runOnQueueingSystem(allOptions) if roq.queue is None: pytest.skip("specify --queue=pbs to continue with this test") if roq.numexec != 3: pytest.skip("specify --num-executors=3 to continue with this test") roq.createPbsScripts() # The following assert statements apply to the final .job file created correctFileName = False callsExec = False correctName = False if "pydpiper-executor-2" in roq.jobFileName: correctFileName = True jobFileString = open(roq.jobFileName).read() if "pipeline_executor.py" in jobFileString: callsExec = True if "pydpiper-executor" in jobFileString: correctName = True assert correctFileName == True assert callsExec == True assert correctName == True
if __name__ == "__main__": usage = "%prog [options]" description = "pipeline executor" # command line option handling parser = OptionParser(usage=usage, description=description) addExecutorOptionGroup(parser) (options,args) = parser.parse_args() #Check to make sure some executors have been specified. noExecSpecified(options.num_exec) if options.queue=="pbs": roq = q.runOnQueueingSystem(options) for i in range(options.num_exec): roq.createExecutorJobFile(i) elif options.queue=="sge": for i in range(options.num_exec): pe = pipelineExecutor(options) pe.submitToQueue() else: for i in range(options.num_exec): pe = pipelineExecutor(options) process = Process(target=launchExecutor, args=(pe,)) process.start() process.join()
# when constructing the executor shell command options = parser.parse_known_args()[0] #Check to make sure some executors have been specified. noExecSpecified(options.num_exec) def local_launch(options): pe = pipelineExecutor(options) # executors don't use any shared-memory constructs, so OK to copy ps = [ Process(target=launchExecutor, args=(pe, )) for _ in range(options.num_exec) ] for p in ps: p.start() for p in ps: p.join() if options.local: local_launch(options) elif options.queue == "pbs" or options.queue_type == "pbs": roq = q.runOnQueueingSystem(options) for i in range(options.num_exec): roq.createAndSubmitExecutorJobFile(i) elif options.queue == "sge" or options.queue_type == "sge": for i in range(options.num_exec): pe = pipelineExecutor(options) pe.submitToQueue() else: local_launch(options)
# Alternately, we could keep a copy of the executor parser around # when constructing the executor shell command options = parser.parse_known_args()[0] #Check to make sure some executors have been specified. noExecSpecified(options.num_exec) def local_launch(options): pe = pipelineExecutor(options) # executors don't use any shared-memory constructs, so OK to copy ps = [Process(target=launchExecutor, args=(pe,)) for _ in range(options.num_exec)] for p in ps: p.start() for p in ps: p.join() if options.local: local_launch(options) elif options.queue_type == "pbs": roq = q.runOnQueueingSystem(options, sysArgs=sys.argv) for i in range(options.num_exec): roq.createAndSubmitExecutorJobFile(i, after=None, time=q.timestr_to_secs(options.time)) elif options.queue_type == "sge": for i in range(options.num_exec): pe = pipelineExecutor(options) pe.submitToQueue() else: local_launch(options)