def main():
"""Restarts a toil.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.toil = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib:
print "There is no root batchjob in the toil from which to start, exiting"
sys.exit(0)
return mainLoop(config, batchSystem, jobStore, jobStore.load(config.attrib["rootJob"]))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean(Job._loadRootJob(jobStore))
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def setUp(self):
super( JobWrapperTest, self ).setUp( )
self.jobStorePath = self._getTestJobStorePath()
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args(args=[self.jobStorePath])
self.contextManager = setupToil(options)
config, batchSystem, self.jobStore = self.contextManager.__enter__()
def setUp(self):
super(JobWrapperTest, self).setUp()
self.jobStorePath = self._getTestJobStorePath()
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args(args=[self.jobStorePath])
self.contextManager = setupToil(options)
config, batchSystem, self.jobStore = self.contextManager.__enter__()
def setUp(self):
super(JobTest, self).setUp()
self.testToil = os.path.join(os.getcwd(), "testJobDir")
parser = OptionParser()
Job.Runner.addToilOptions(parser)
options, args = parser.parse_args()
options.toil = self.testToil
self.contextManager = setupToil(options)
config, batchSystem, jobStore = self.contextManager.__enter__()
self.jobStore = jobStore
def setUp(self):
super( JobTest, self ).setUp( )
self.testToil = os.path.join(os.getcwd(), "testJobDir")
parser = OptionParser()
Job.Runner.addToilOptions(parser)
options, args = parser.parse_args()
options.toil = self.testToil
self.contextManager = setupToil(options)
config, batchSystem, jobStore = self.contextManager.__enter__()
self.jobStore = jobStore
def startToil(job, options):
"""
Runs the toil using the given options (see Job.Runner.getDefaultOptions
and Job.Runner.addToilOptions) starting with this job.
Raises an exception if the given toil already exists.
"""
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib: #No jobs have yet been run
# Setup the first batchjob.
rootJob = job._serialiseFirstJob(jobStore)
else:
rootJob = jobStore.load(config.attrib["rootJob"])
return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options):
"""
Runs the toil using the given options (see Job.Runner.getDefaultOptions
and Job.Runner.addToilOptions) starting with this job.
Raises an exception if the given toil already exists.
"""
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib: #No jobs have yet been run
# Setup the first batchjob.
rootJob = job._serialiseFirstJob(jobStore)
else:
rootJob = jobStore.load(config.attrib["rootJob"])
return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options):
"""
Runs the toil workflow using the given options
(see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions)
starting with this job.
:raises: toil.leader.FailedJobsException if at the end of function their remain
failed jobs
"""
setLoggingFromOptions(options)
with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
if options.restart:
jobStore.clean() #This cleans up any half written jobs after a restart
rootJob = job._loadRootJob(jobStore)
else:
#Setup the first wrapper.
rootJob = job._serialiseFirstJob(jobStore)
return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options):
"""
Runs the toil workflow using the given options
(see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions)
starting with this job.
:raises: toil.leader.FailedJobsException if at the end of function their remain
failed jobs
"""
setLoggingFromOptions(options)
with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
if options.restart:
jobStore.clean() #This cleans up any half written jobs after a restart
rootJob = job._loadRootJob(jobStore)
else:
#Setup the first wrapper.
rootJob = job._serialiseFirstJob(jobStore)
return mainLoop(config, batchSystem, jobStore, rootJob)
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument(
"jobStore",
type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {
jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()
}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config,
batchSystem,
jobStore,
Job._loadRootJob(jobStore),
jobCache=jobCache)
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore), jobCache=jobCache)
def cleanup(options):
"""
Removes the jobStore backing the toil.
"""
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.deleteJobStore()
def cleanup(options):
"""
Removes the jobStore backing the toil.
"""
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.deleteJobStore()