def main(): """Restarts a toil. """ ########################################## #Construct the arguments. ########################################## parser = OptionParser() addOptions(parser) options, args = parser.parse_args() if len(args) != 0: parser.error("Unrecognised input arguments: %s" % " ".join(args)) if len(sys.argv) == 1: parser.print_help() sys.exit(0) assert len(args) <= 1 #Only toil may be specified as argument if len(args) == 1: #Allow toil directory as arg options.toil = args[0] ########################################## #Now run the toil construction/leader ########################################## setLoggingFromOptions(options) with setupToil(options) as (config, batchSystem, jobStore): jobStore.clean() if "rootJob" not in config.attrib: print "There is no root batchjob in the toil from which to start, exiting" sys.exit(0) return mainLoop(config, batchSystem, jobStore, jobStore.load(config.attrib["rootJob"]))
def main(): """Restarts a toil workflow. """ ########################################## #Construct the arguments. ########################################## parser = OptionParser() addOptions(parser) options, args = parser.parse_args() if len(args) != 0: parser.error("Unrecognised input arguments: %s" % " ".join(args)) if len(sys.argv) == 1: parser.print_help() sys.exit(0) assert len(args) <= 1 #Only toil may be specified as argument if len(args) == 1: #Allow toil directory as arg options.jobStore = args[0] ########################################## #Now run the toil construction/leader ########################################## setLoggingFromOptions(options) options.restart = True with setupToil(options) as (config, batchSystem, jobStore): jobStore.clean() mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main(): """Restarts a toil workflow. """ ########################################## #Construct the arguments. ########################################## parser = getBasicOptionParser() parser.add_argument("--version", action='version', version=version) parser.add_argument("jobStore", type=str, help=("Store in which to place job management files \ and the global accessed temporary files" "(If this is a file path this needs to be globally accessible " "by all machines running jobs).\n" "If the store already exists and restart is false an" " ExistingJobStoreException exception will be thrown.")) options = parseBasicOptions(parser) ########################################## #Now run the toil construction/leader ########################################## setLoggingFromOptions(options) options.restart = True with setupToil(options) as (config, batchSystem, jobStore): jobStore.clean(Job._loadRootJob(jobStore)) mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def setUp(self): super( JobWrapperTest, self ).setUp( ) self.jobStorePath = self._getTestJobStorePath() parser = ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args(args=[self.jobStorePath]) self.contextManager = setupToil(options) config, batchSystem, self.jobStore = self.contextManager.__enter__()
def setUp(self): super(JobWrapperTest, self).setUp() self.jobStorePath = self._getTestJobStorePath() parser = ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args(args=[self.jobStorePath]) self.contextManager = setupToil(options) config, batchSystem, self.jobStore = self.contextManager.__enter__()
def setUp(self): super(JobTest, self).setUp() self.testToil = os.path.join(os.getcwd(), "testJobDir") parser = OptionParser() Job.Runner.addToilOptions(parser) options, args = parser.parse_args() options.toil = self.testToil self.contextManager = setupToil(options) config, batchSystem, jobStore = self.contextManager.__enter__() self.jobStore = jobStore
def setUp(self): super( JobTest, self ).setUp( ) self.testToil = os.path.join(os.getcwd(), "testJobDir") parser = OptionParser() Job.Runner.addToilOptions(parser) options, args = parser.parse_args() options.toil = self.testToil self.contextManager = setupToil(options) config, batchSystem, jobStore = self.contextManager.__enter__() self.jobStore = jobStore
def startToil(job, options): """ Runs the toil using the given options (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) starting with this job. Raises an exception if the given toil already exists. """ setLoggingFromOptions(options) with setupToil(options) as (config, batchSystem, jobStore): jobStore.clean() if "rootJob" not in config.attrib: #No jobs have yet been run # Setup the first batchjob. rootJob = job._serialiseFirstJob(jobStore) else: rootJob = jobStore.load(config.attrib["rootJob"]) return mainLoop(config, batchSystem, jobStore, rootJob)
def startToil(job, options): """ Runs the toil workflow using the given options (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) starting with this job. :raises: toil.leader.FailedJobsException if at the end of function their remain failed jobs """ setLoggingFromOptions(options) with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore): if options.restart: jobStore.clean() #This cleans up any half written jobs after a restart rootJob = job._loadRootJob(jobStore) else: #Setup the first wrapper. rootJob = job._serialiseFirstJob(jobStore) return mainLoop(config, batchSystem, jobStore, rootJob)
def main(): """Restarts a toil workflow. """ ########################################## #Construct the arguments. ########################################## parser = getBasicOptionParser() parser.add_argument("--version", action='version', version=version) parser.add_argument( "jobStore", type=str, help=("Store in which to place job management files \ and the global accessed temporary files" "(If this is a file path this needs to be globally accessible " "by all machines running jobs).\n" "If the store already exists and restart is false an" " ExistingJobStoreException exception will be thrown.")) options = parseBasicOptions(parser) ########################################## #Now run the toil construction/leader ########################################## setLoggingFromOptions(options) options.restart = True with setupToil(options) as (config, batchSystem, jobStore): # Load the whole jobstore into memory in a batch logger.info("Downloading entire JobStore") jobCache = { jobWrapper.jobStoreID: jobWrapper for jobWrapper in jobStore.jobs() } logger.info("{} jobs downloaded.".format(len(jobCache))) jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache) mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore), jobCache=jobCache)
def main(): """Restarts a toil workflow. """ ########################################## #Construct the arguments. ########################################## parser = getBasicOptionParser() parser.add_argument("--version", action='version', version=version) parser.add_argument("jobStore", type=str, help=("Store in which to place job management files \ and the global accessed temporary files" "(If this is a file path this needs to be globally accessible " "by all machines running jobs).\n" "If the store already exists and restart is false an" " ExistingJobStoreException exception will be thrown.")) options = parseBasicOptions(parser) ########################################## #Now run the toil construction/leader ########################################## setLoggingFromOptions(options) options.restart = True with setupToil(options) as (config, batchSystem, jobStore): # Load the whole jobstore into memory in a batch logger.info("Downloading entire JobStore") jobCache = {jobWrapper.jobStoreID: jobWrapper for jobWrapper in jobStore.jobs()} logger.info("{} jobs downloaded.".format(len(jobCache))) jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache) mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore), jobCache=jobCache)
def cleanup(options): """ Removes the jobStore backing the toil. """ with setupToil(options) as (config, batchSystem, jobStore): jobStore.deleteJobStore()