def start(self):
logger.info("Calling `start`")
self._setup_options()
self.setup_options()
self.options = self.parser.parse_args()
self.args = self.options.files
self._print_version()
# Check to make sure some executors have been specified if we are
# actually going to run:
if self.options.execute:
noExecSpecified(self.options.num_exec)
self._setup_pipeline(self.options)
self._setup_directories()
self.appName = self.setup_appName()
self.setup_logger()
# TODO this doesn't capture environment variables
# or contents of any config file so isn't really complete
self.reconstructCommand()
pbs_submit = self.options.queue_type == "pbs" \
and not self.options.local
# --create-graph causes the pipeline to be constructed
# both at PBS submit time and on the grid; this may be an extremely
# expensive duplication
if (self.options.execute and not pbs_submit) or self.options.create_graph:
logger.debug("Calling `run`")
self.run()
logger.debug("Calling `initialize`")
self.pipeline.initialize()
self.pipeline.printStages(self.options.pipeline_name)
if self.options.create_graph:
logger.debug("Writing dot file...")
nx.write_dot(self.pipeline.G, str(self.options.pipeline_name) + "_labeled-tree.dot")
logger.debug("Done.")
if not self.options.execute:
print("Not executing the command (--no-execute is specified).\nDone.")
return
if pbs_submit:
roq = runOnQueueingSystem(self.options, sys.argv)
roq.createAndSubmitPbsScripts()
logger.info("Finished submitting PBS job scripts...quitting")
return
#pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
# if use_ns is specified, Pyro NameServer must be started.
logger.info("Starting pipeline daemon...")
pipelineDaemon(self.pipeline, self.options, sys.argv[0])
logger.info("Server has stopped. Quitting...")
def test_roq_init_defaults(self, setupopts):
"""Makes sure that pbs related defaults are correctly initialized"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
assert roq.queue == "pbs"
assert roq.arguments == None
assert roq.jobName == "pydpiper"
assert roq.time == "2:00:00:00"
def test_roq_init_defaults(self, setupopts):
"""Makes sure that pbs related defaults are correctly initialized"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
assert roq.queue == "pbs"
assert roq.arguments == None
assert roq.jobName == "pydpiper"
assert roq.time == "2:00:00:00"
def test_create_job_0exec(self, setupopts):
"""Test defaults and file creation when no executors are specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 0:
pytest.skip("This test will not run if --num-executors is specified")
roq.createPbsScripts()
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
def start(self):
logger.info("Calling `start`")
self._setup_options()
self.setup_options()
self.options = self.parser.parse_args()
self.args = self.options.files
self._print_version()
#Check to make sure some executors have been specified.
noExecSpecified(self.options.num_exec)
self._setup_pipeline(self.options)
self._setup_directories()
self.appName = self.setup_appName()
self.setup_logger()
# NB this doesn't capture environment variables
# or contents of any config file so isn't really complete
self.reconstructCommand()
pbs_submit = (self.options.queue == "pbs" or \
self.options.queue_type == "pbs") \
and not self.options.local
if (self.options.execute
and not pbs_submit) or self.options.create_graph:
logger.debug("Calling `run`")
self.run()
logger.debug("Calling `initialize`")
self.pipeline.initialize()
self.pipeline.printStages(self.options.pipeline_name)
if self.options.create_graph:
logger.debug("Writing dot file...")
nx.write_dot(self.pipeline.G, "labeled-tree.dot")
logger.debug("Done.")
if not self.options.execute:
print "Not executing the command (--no-execute is specified).\nDone."
return
if pbs_submit:
roq = runOnQueueingSystem(self.options, sys.argv)
roq.createAndSubmitPbsScripts()
logger.info("Finished submitting PBS job scripts...quitting")
return
#pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
# if use_ns is specified, Pyro NameServer must be started.
logger.info("Starting pipeline daemon...")
pipelineDaemon(self.pipeline, self.options, sys.argv[0])
logger.info("Server has stopped. Quitting...")
def test_create_job_0exec(self, setupopts):
"""Test defaults and file creation when no executors are specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 0:
pytest.skip(
"This test will not run if --num-executors is specified")
roq.createPbsScripts()
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
def start(self):
self._setup_options()
self.setup_options()
self.options, self.args = self.parser.parse_args()
self._print_version()
#Check to make sure some executors have been specified.
noExecSpecified(self.options.num_exec)
self._setup_pipeline()
self._setup_directories()
self.appName = self.setup_appName()
self.setup_logger()
if self.options.queue=="pbs":
roq = runOnQueueingSystem(self.options, sys.argv)
roq.createPbsScripts()
return
if self.options.restart:
print "\nThe restart option is deprecated (pipelines are not pickled anymore, because it takes too much time). Will restart based on which files exists already\n"
#logger.info("Restarting pipeline from pickled files.")
#self.pipeline.restart()
self.reconstructCommand()
self.run()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
else:
self.reconstructCommand()
self.run()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
if self.options.create_graph:
logger.debug("Writing dot file...")
nx.write_dot(self.pipeline.G, "labeled-tree.dot")
logger.debug("Done.")
if not self.options.execute:
print "Not executing the command (--no-execute is specified).\nDone."
return
#pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
# if use_ns is specified, Pyro NameServer must be started.
logger.info("Starting pipeline daemon...")
pipelineDaemon(self.pipeline, self.options, sys.argv[0])
logger.info("Server has stopped. Quitting...")
def start(self):
self._setup_options()
self.setup_options()
self.options, self.args = self.parser.parse_args()
self._print_version()
#Check to make sure some executors have been specified.
noExecSpecified(self.options.num_exec)
self._setup_pipeline()
self._setup_directories()
self.appName = self.setup_appName()
self.setup_logger()
if self.options.queue == "pbs":
roq = runOnQueueingSystem(self.options, sys.argv)
roq.createPbsScripts()
return
if self.options.restart:
logger.info("Restarting pipeline from pickled files.")
self.pipeline.restart()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
else:
self.reconstructCommand()
self.run()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
if self.options.create_graph:
logger.debug("Writing dot file...")
nx.write_dot(self.pipeline.G, "labeled-tree.dot")
logger.debug("Done.")
if not self.options.execute:
print "Not executing the command (--no-execute is specified).\nDone."
return
#pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
# if use_ns is specified, Pyro NameServer must be started.
logger.info("Starting pipeline daemon...")
pipelineDaemon(self.pipeline, self.options, sys.argv[0])
logger.info("Server has stopped. Quitting...")
def test_create_job_1exec(self, setupopts):
"""This test requires specifying --num-executors=1 on command line
Verifies defaults and file creation when only one executor is specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createPbsScripts()
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
callsExec = False
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
assert callsExec == True
def test_create_job_1exec(self, setupopts):
"""This test requires specifying --num-executors=1 on command line
Verifies defaults and file creation when only one executor is specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createPbsScripts()
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
callsExec = False
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
assert callsExec == True
def test_create_job_multiproc(self, setupopts):
"""This test requires specifying --proc=24 on command line
Verifies defaults and file creation when multiple nodes are needed"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.ppn != 8:
pytest.skip("specify --ppn=8 to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
if roq.proc != 24:
pytest.skip("specify --proc==24 to continue with this test")
roq.createPbsScripts()
correctNodesPpn = False
jobFileString = open(roq.jobFileName).read()
if "nodes=3:ppn=8" in jobFileString:
correctNodesPpn = True
assert correctNodesPpn == True
def test_create_job_multiproc(self, setupopts):
"""This test requires specifying --proc=24 on command line
Verifies defaults and file creation when multiple nodes are needed"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.ppn != 8:
pytest.skip("specify --ppn=8 to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
if roq.proc != 24:
pytest.skip("specify --proc==24 to continue with this test")
roq.createPbsScripts()
correctNodesPpn = False
jobFileString = open(roq.jobFileName).read()
if "nodes=3:ppn=8" in jobFileString:
correctNodesPpn = True
assert correctNodesPpn == True
def start(self):
self._setup_options()
self.setup_options()
self.options, self.args = self.parser.parse_args()
self._setup_pipeline()
self._setup_directories()
self.appName = self.setup_appName()
self.setup_logger()
if self.options.queue=="pbs":
roq = runOnQueueingSystem(self.options, sys.argv)
roq.createPbsScripts()
return
if self.options.restart:
logger.info("Restarting pipeline from pickled files.")
self.pipeline.restart()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
else:
self.reconstructCommand()
self.run()
self.pipeline.initialize()
self.pipeline.printStages(self.appName)
if self.options.create_graph:
logger.debug("Writing dot file...")
nx.write_dot(self.pipeline.G, "labeled-tree.dot")
logger.debug("Done.")
if not self.options.execute:
print "Not executing the command (--no-execute is specified).\nDone."
return
#pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
# if use_ns is specified, Pyro NameServer must be started.
logger.info("Starting pipeline daemon...")
pipelineDaemon(self.pipeline, self.options, sys.argv[0])
logger.info("Server has stopped. Quitting...")
def test_create_job_with_args(self, setupopts):
"""This test verifies the appropriate script creation for a sample
set of arguments."""
allOptions = setupopts.returnAllOptions()
sampleArgs = setupopts.returnSampleArgs()
roq = runOnQueueingSystem(allOptions, sysArgs=sampleArgs)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createPbsScripts()
assert roq.jobName == "TestProgName.py"
mncFilesIncluded = False
sleepIncluded = False
jobFileString = open(roq.jobFileName).read()
if "img_A.mnc" and "img_B.mnc" in jobFileString:
mncFilesIncluded = True
if "sleep 60" in jobFileString:
sleepIncluded = True
assert mncFilesIncluded == True
assert sleepIncluded == True
def test_create_job_with_args(self, setupopts):
"""This test verifies the appropriate script creation for a sample
set of arguments."""
allOptions = setupopts.returnAllOptions()
sampleArgs = setupopts.returnSampleArgs()
roq = runOnQueueingSystem(allOptions, sysArgs=sampleArgs)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createPbsScripts()
assert roq.jobName == "TestProgName.py"
mncFilesIncluded = False
sleepIncluded = False
jobFileString = open(roq.jobFileName).read()
if "img_A.mnc" and "img_B.mnc" in jobFileString:
mncFilesIncluded = True
if "sleep 60" in jobFileString:
sleepIncluded = True
assert mncFilesIncluded == True
assert sleepIncluded == True
def test_create_executor_only(self, setupopts):
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createExecutorJobFile(1)
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
correctFileName = False
callsExec = False
correctName = False
if "pydpiper-executor-1" in roq.jobFileName:
correctFileName = True
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
if "pydpiper-executor" in jobFileString:
correctName = True
assert correctFileName == True
assert callsExec == True
assert correctName == True
def test_create_executor_only(self, setupopts):
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
roq.createExecutorJobFile(1)
assert roq.jobName == "pydpiper"
assert isfile(roq.jobFileName) == True
correctFileName = False
callsExec = False
correctName = False
if "pydpiper-executor-1" in roq.jobFileName:
correctFileName = True
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
if "pydpiper-executor" in jobFileString:
correctName = True
assert correctFileName == True
assert callsExec == True
assert correctName == True
def test_create_job_newppn(self, setupopts):
"""This test requires specifying --ppn=10, --num-executors=1 and --proc=24
Verifies appropriate node selection"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.ppn != 10:
pytest.skip("specify --ppn=10 to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
if roq.proc != 24:
pytest.skip("specify --proc==24 to continue with this test")
roq.createPbsScripts()
roq.createPbsScripts()
correctPpn = False
correctProc = False
jobFileString = open(roq.jobFileName).read()
if "nodes=2:ppn=10" in jobFileString:
correctPpn = True
if "--proc=24" in jobFileString:
correctProc = True
assert correctPpn == True
assert correctProc == True
def test_create_job_newppn(self, setupopts):
"""This test requires specifying --ppn=10, --num-executors=1 and --proc=24
Verifies appropriate node selection"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.ppn != 10:
pytest.skip("specify --ppn=10 to continue with this test")
if roq.numexec != 1:
pytest.skip("specify --num-executors=1 to continue with this test")
if roq.proc != 24:
pytest.skip("specify --proc==24 to continue with this test")
roq.createPbsScripts()
roq.createPbsScripts()
correctPpn = False
correctProc = False
jobFileString = open(roq.jobFileName).read()
if "nodes=2:ppn=10" in jobFileString:
correctPpn=True
if "--proc=24" in jobFileString:
correctProc = True
assert correctPpn == True
assert correctProc == True
def test_create_job_3execs(self, setupopts):
"""This test requires specifying --num-executors=3 on command line
Verifies defaults and file creation when three executors are specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 3:
pytest.skip("specify --num-executors=3 to continue with this test")
roq.createPbsScripts()
# The following assert statements apply to the final .job file created
correctFileName = False
callsExec = False
correctName = False
if "pydpiper-executor-2" in roq.jobFileName:
correctFileName = True
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
if "pydpiper-executor" in jobFileString:
correctName = True
assert correctFileName == True
assert callsExec == True
assert correctName == True
def test_create_job_3execs(self, setupopts):
"""This test requires specifying --num-executors=3 on command line
Verifies defaults and file creation when three executors are specified"""
allOptions = setupopts.returnAllOptions()
roq = runOnQueueingSystem(allOptions)
if roq.queue is None:
pytest.skip("specify --queue=pbs to continue with this test")
if roq.numexec != 3:
pytest.skip("specify --num-executors=3 to continue with this test")
roq.createPbsScripts()
# The following assert statements apply to the final .job file created
correctFileName = False
callsExec = False
correctName = False
if "pydpiper-executor-2" in roq.jobFileName:
correctFileName = True
jobFileString = open(roq.jobFileName).read()
if "pipeline_executor.py" in jobFileString:
callsExec = True
if "pydpiper-executor" in jobFileString:
correctName = True
assert correctFileName == True
assert callsExec == True
assert correctName == True
if __name__ == "__main__":
usage = "%prog [options]"
description = "pipeline executor"
# command line option handling
parser = OptionParser(usage=usage, description=description)
addExecutorOptionGroup(parser)
(options,args) = parser.parse_args()
#Check to make sure some executors have been specified.
noExecSpecified(options.num_exec)
if options.queue=="pbs":
roq = q.runOnQueueingSystem(options)
for i in range(options.num_exec):
roq.createExecutorJobFile(i)
elif options.queue=="sge":
for i in range(options.num_exec):
pe = pipelineExecutor(options)
pe.submitToQueue()
else:
for i in range(options.num_exec):
pe = pipelineExecutor(options)
process = Process(target=launchExecutor, args=(pe,))
process.start()
process.join()
# when constructing the executor shell command
options = parser.parse_known_args()[0]
#Check to make sure some executors have been specified.
noExecSpecified(options.num_exec)
def local_launch(options):
pe = pipelineExecutor(options)
# executors don't use any shared-memory constructs, so OK to copy
ps = [
Process(target=launchExecutor, args=(pe, ))
for _ in range(options.num_exec)
]
for p in ps:
p.start()
for p in ps:
p.join()
if options.local:
local_launch(options)
elif options.queue == "pbs" or options.queue_type == "pbs":
roq = q.runOnQueueingSystem(options)
for i in range(options.num_exec):
roq.createAndSubmitExecutorJobFile(i)
elif options.queue == "sge" or options.queue_type == "sge":
for i in range(options.num_exec):
pe = pipelineExecutor(options)
pe.submitToQueue()
else:
local_launch(options)
# Alternately, we could keep a copy of the executor parser around
# when constructing the executor shell command
options = parser.parse_known_args()[0]
#Check to make sure some executors have been specified.
noExecSpecified(options.num_exec)
def local_launch(options):
pe = pipelineExecutor(options)
# executors don't use any shared-memory constructs, so OK to copy
ps = [Process(target=launchExecutor, args=(pe,))
for _ in range(options.num_exec)]
for p in ps:
p.start()
for p in ps:
p.join()
if options.local:
local_launch(options)
elif options.queue_type == "pbs":
roq = q.runOnQueueingSystem(options, sysArgs=sys.argv)
for i in range(options.num_exec):
roq.createAndSubmitExecutorJobFile(i, after=None,
time=q.timestr_to_secs(options.time))
elif options.queue_type == "sge":
for i in range(options.num_exec):
pe = pipelineExecutor(options)
pe.submitToQueue()
else:
local_launch(options)
