def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean(Job._loadRootJob(jobStore))
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.toil = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib:
print "There is no root batchjob in the toil from which to start, exiting"
sys.exit(0)
return mainLoop(config, batchSystem, jobStore, jobStore.load(config.attrib["rootJob"]))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument(
"jobStore",
type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {
jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()
}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config,
batchSystem,
jobStore,
Job._loadRootJob(jobStore),
jobCache=jobCache)
def _runMainLoop(self, rootJob):
"""
Runs the main loop with the given job.
:param toil.job.Job rootJob: The root job for the workflow.
:rtype: Any
"""
with RealtimeLogger(self._batchSystem, level=self.options.logLevel if self.options.realTimeLogging else None):
# FIXME: common should not import from leader
from toil.leader import mainLoop
return mainLoop(config=self.config,
batchSystem=self._batchSystem,
provisioner=None,
jobStore=self._jobStore,
rootJobWrapper=rootJob,
jobCache=self._jobCache)
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore), jobCache=jobCache)
def startToil(job, options):
"""
Runs the toil using the given options (see Job.Runner.getDefaultOptions
and Job.Runner.addToilOptions) starting with this job.
Raises an exception if the given toil already exists.
"""
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib: #No jobs have yet been run
# Setup the first batchjob.
rootJob = job._serialiseFirstJob(jobStore)
else:
rootJob = jobStore.load(config.attrib["rootJob"])
return mainLoop(config, batchSystem, jobStore, rootJob)
def _runMainLoop(self, rootJob):
"""
Runs the main loop with the given job.
:param toil.job.Job rootJob: The root job for the workflow.
:rtype: Any
"""
with RealtimeLogger(self._batchSystem,
level=self.options.logLevel if self.options.realTimeLogging else None):
# FIXME: common should not import from leader
from toil.leader import mainLoop
return mainLoop(config=self.config,
batchSystem=self._batchSystem,
provisioner=None,
jobStore=self._jobStore,
rootJobWrapper=rootJob,
jobCache=self._jobCache)
def startToil(job, options):
"""
Runs the toil using the given options (see Job.Runner.getDefaultOptions
and Job.Runner.addToilOptions) starting with this job.
Raises an exception if the given toil already exists.
"""
setLoggingFromOptions(options)
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
if "rootJob" not in config.attrib: #No jobs have yet been run
# Setup the first batchjob.
rootJob = job._serialiseFirstJob(jobStore)
else:
rootJob = jobStore.load(config.attrib["rootJob"])
return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options):
"""
Runs the toil workflow using the given options
(see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions)
starting with this job.
:raises: toil.leader.FailedJobsException if at the end of function their remain
failed jobs
"""
setLoggingFromOptions(options)
with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
if options.restart:
jobStore.clean() #This cleans up any half written jobs after a restart
rootJob = job._loadRootJob(jobStore)
else:
#Setup the first wrapper.
rootJob = job._serialiseFirstJob(jobStore)
return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options):
"""
Runs the toil workflow using the given options
(see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions)
starting with this job.
:raises: toil.leader.FailedJobsException if at the end of function their remain
failed jobs
"""
setLoggingFromOptions(options)
with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
if options.restart:
jobStore.clean() #This cleans up any half written jobs after a restart
rootJob = job._loadRootJob(jobStore)
else:
#Setup the first wrapper.
rootJob = job._serialiseFirstJob(jobStore)
return mainLoop(config, batchSystem, jobStore, rootJob)