def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean(Job._loadRootJob(jobStore))
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = OptionParser()
addOptions(parser)
options, args = parser.parse_args()
if len(args) != 0:
parser.error("Unrecognised input arguments: %s" % " ".join(args))
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
jobStore.clean()
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument(
"jobStore",
type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {
jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()
}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config,
batchSystem,
jobStore,
Job._loadRootJob(jobStore),
jobCache=jobCache)
def main():
"""Restarts a toil workflow.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("--version", action='version', version=version)
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
options = parseBasicOptions(parser)
##########################################
#Now run the toil construction/leader
##########################################
setLoggingFromOptions(options)
options.restart = True
with setupToil(options) as (config, batchSystem, jobStore):
# Load the whole jobstore into memory in a batch
logger.info("Downloading entire JobStore")
jobCache = {jobWrapper.jobStoreID: jobWrapper
for jobWrapper in jobStore.jobs()}
logger.info("{} jobs downloaded.".format(len(jobCache)))
jobStore.clean(Job._loadRootJob(jobStore), jobCache=jobCache)
mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore), jobCache=jobCache)
def main():
"""Reports the state of the toil.
"""
##########################################
# Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument(
"jobStore",
type=str,
help=(
"Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."
),
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
help="Print loads of information, particularly all the log files of \
jobs that failed. default=%(default)s",
default=False,
)
parser.add_argument(
"--failIfNotComplete",
dest="failIfNotComplete",
action="store_true",
help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
default=False,
)
parser.add_argument("--version", action="version", version=version)
options = parseBasicOptions(parser)
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
##########################################
# Do some checks.
##########################################
logger.info("Checking if we have files for toil")
assert options.jobStore != None
##########################################
# Survey the status of the job and report.
##########################################
jobStore = loadJobStore(options.jobStore)
try:
rootJob = Job._loadRootJob(jobStore)
except JobException:
print "The root job of the jobStore is not present, the toil workflow has probably completed okay"
sys.exit(0)
toilState = ToilState(jobStore, rootJob)
failedJobs = [
job for job in toilState.updatedJobs | set(toilState.successorCounts.keys()) if job.remainingRetryCount == 0
]
print "There are %i active jobs, %i parent jobs with children, and \
%i totally failed jobs currently in toil workflow: %s" % (
len(toilState.updatedJobs),
len(toilState.successorCounts),
len(failedJobs),
options.jobStore,
)
if options.verbose: # Verbose currently means outputting the files that have failed.
for job in failedJobs:
if job.logJobStoreFileID is not None:
with job.getLogFileHandle(jobStore) as logFileHandle:
logStream(logFileHandle, job.jobStoreID, logger.warn)
else:
print "Log file for job %s is not present" % job.jobStoreID
if len(failedJobs) == 0:
print "There are no failed jobs to report"
if (len(toilState.updatedJobs) + len(toilState.successorCounts)) != 0 and options.failIfNotComplete:
sys.exit(1)
def main():
"""Reports the state of the toil.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser()
parser.add_argument("jobStore", type=str,
help=("Store in which to place job management files \
and the global accessed temporary files"
"(If this is a file path this needs to be globally accessible "
"by all machines running jobs).\n"
"If the store already exists and restart is false an"
" ExistingJobStoreException exception will be thrown."))
parser.add_argument("--verbose", dest="verbose", action="store_true",
help="Print loads of information, particularly all the log files of \
jobs that failed. default=%(default)s",
default=False)
parser.add_argument("--failIfNotComplete", dest="failIfNotComplete", action="store_true",
help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
default=False)
parser.add_argument("--version", action='version', version=version)
options = parseBasicOptions(parser)
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
##########################################
#Do some checks.
##########################################
logger.info("Checking if we have files for toil")
assert options.jobStore != None
##########################################
#Survey the status of the job and report.
##########################################
jobStore = loadJobStore(options.jobStore)
try:
rootJob = Job._loadRootJob(jobStore)
except JobException:
print "The root job of the jobStore is not present, the toil workflow has probably completed okay"
sys.exit(0)
toilState = ToilState(jobStore, rootJob )
# The first element of the toilState.updatedJobs tuple is the jobWrapper we want to inspect
totalJobs = set(toilState.successorCounts.keys()) | \
{jobTuple[0] for jobTuple in toilState.updatedJobs}
failedJobs = [ job for job in totalJobs if job.remainingRetryCount == 0 ]
print "There are %i active jobs, %i parent jobs with children, and \
%i totally failed jobs currently in toil workflow: %s" % \
(len(toilState.updatedJobs), len(toilState.successorCounts),
len(failedJobs), options.jobStore)
if options.verbose: #Verbose currently means outputting the files that have failed.
for job in failedJobs:
if job.logJobStoreFileID is not None:
with job.getLogFileHandle(jobStore) as logFileHandle:
logStream(logFileHandle, job.jobStoreID, logger.warn)
else:
print "Log file for job %s is not present" % job.jobStoreID
if len(failedJobs) == 0:
print "There are no failed jobs to report"
if (len(toilState.updatedJobs) + len(toilState.successorCounts)) != 0 and \
options.failIfNotComplete:
sys.exit(1)
def main():
"""Reports the state of the toil.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser(
"usage: %prog [--jobStore] JOB_TREE_DIR [options]", "%prog 0.1")
parser.add_option(
"--jobStore",
dest="jobStore",
help=
"Job store path. Can also be specified as the single argument to the script.\
default=%default",
default=os.path.abspath("./toil"))
parser.add_option(
"--verbose",
dest="verbose",
action="store_true",
help="Print loads of information, particularly all the log files of \
jobs that failed. default=%default",
default=False)
parser.add_option(
"--failIfNotComplete",
dest="failIfNotComplete",
action="store_true",
help=
"Return exit value of 1 if toil jobs not all completed. default=%default",
default=False)
options, args = parseBasicOptions(parser)
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Do some checks.
##########################################
logger.info("Checking if we have files for toil")
assert options.jobStore != None
##########################################
#Survey the status of the job and report.
##########################################
jobStore = loadJobStore(options.jobStore)
try:
rootJob = Job._loadRootJob(jobStore)
except JobException:
print "The root job of the jobStore is not present, the toil workflow has probably completed okay"
sys.exit(0)
toilState = ToilState(jobStore, rootJob)
failedJobs = [ job for job in toilState.updatedJobs | \
set(toilState.successorCounts.keys()) \
if job.remainingRetryCount == 0 ]
print "There are %i active jobs, %i parent jobs with children, and \
%i totally failed jobs currently in toil workflow: %s" % \
(len(toilState.updatedJobs), len(toilState.successorCounts),
len(failedJobs), options.jobStore)
if options.verbose: #Verbose currently means outputting the files that have failed.
for job in failedJobs:
if job.logJobStoreFileID is not None:
with job.getLogFileHandle(jobStore) as logFileHandle:
logStream(logFileHandle, job.jobStoreID, logger.warn)
else:
print "Log file for job %s is not present" % job.jobStoreID
if len(failedJobs) == 0:
print "There are no failed jobs to report"
if (len(toilState.updatedJobs) + len(toilState.successorCounts)) != 0 and \
options.failIfNotComplete:
sys.exit(1)
def main():
"""Reports the state of the toil.
"""
##########################################
#Construct the arguments.
##########################################
parser = getBasicOptionParser("usage: %prog [--jobStore] JOB_TREE_DIR [options]", "%prog 0.1")
parser.add_option("--jobStore", dest="jobStore",
help="Job store path. Can also be specified as the single argument to the script.\
default=%default", default=os.path.abspath("./toil"))
parser.add_option("--verbose", dest="verbose", action="store_true",
help="Print loads of information, particularly all the log files of \
jobs that failed. default=%default",
default=False)
parser.add_option("--failIfNotComplete", dest="failIfNotComplete", action="store_true",
help="Return exit value of 1 if toil jobs not all completed. default=%default",
default=False)
options, args = parseBasicOptions(parser)
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 #Only toil may be specified as argument
if len(args) == 1: #Allow toil directory as arg
options.jobStore = args[0]
##########################################
#Do some checks.
##########################################
logger.info("Checking if we have files for toil")
assert options.jobStore != None
##########################################
#Survey the status of the job and report.
##########################################
jobStore = loadJobStore(options.jobStore)
try:
rootJob = Job._loadRootJob(jobStore)
except JobException:
print "The root job of the jobStore is not present, the toil workflow has probably completed okay"
sys.exit(0)
toilState = ToilState(jobStore, rootJob )
failedJobs = [ job for job in toilState.updatedJobs | \
set(toilState.successorCounts.keys()) \
if job.remainingRetryCount == 0 ]
print "There are %i active jobs, %i parent jobs with children, and \
%i totally failed jobs currently in toil workflow: %s" % \
(len(toilState.updatedJobs), len(toilState.successorCounts),
len(failedJobs), options.jobStore)
if options.verbose: #Verbose currently means outputting the files that have failed.
for job in failedJobs:
if job.logJobStoreFileID is not None:
with job.getLogFileHandle(jobStore) as logFileHandle:
logStream(logFileHandle, job.jobStoreID, logger.warn)
else:
print "Log file for job %s is not present" % job.jobStoreID
if len(failedJobs) == 0:
print "There are no failed jobs to report"
if (len(toilState.updatedJobs) + len(toilState.successorCounts)) != 0 and \
options.failIfNotComplete:
sys.exit(1)
