def testAtomicityOfNonEmptyDirectoryRenames(self):
for _ in range(100):
parent = self._createTempDir(purpose='parent')
child = os.path.join(parent, 'child')
# Use processes (as opposed to threads) to prevent GIL from ordering things artificially
pool = multiprocessing.Pool(processes=cpu_count())
try:
numTasks = cpu_count() * 10
grandChildIds = pool.map_async(func=partial(
_testAtomicityOfNonEmptyDirectoryRenamesTask, parent,
child),
iterable=list(range(numTasks)))
grandChildIds = grandChildIds.get()
finally:
pool.close()
pool.join()
self.assertEqual(len(grandChildIds), numTasks)
# Assert that we only had one winner
grandChildIds = [n for n in grandChildIds if n is not None]
self.assertEqual(len(grandChildIds), 1)
# Assert that the winner's grandChild wasn't silently overwritten by a looser
expectedGrandChildId = grandChildIds[0]
actualGrandChild = os.path.join(child, 'grandChild')
actualGrandChildId = os.stat(actualGrandChild).st_ino
self.assertEqual(actualGrandChildId, expectedGrandChildId)
def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
# TODO: Only add options for the system the user is specifying?
parser.add_argument("--batchSystem", dest="batchSystem", default=DEFAULT_BATCH_SYSTEM, choices=BATCH_SYSTEMS,
help=f"The type of batch system to run the job(s) with, currently can be one "
f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}")
parser.add_argument("--disableHotDeployment", dest="disableAutoDeployment", action='store_true', default=None,
help="Hot-deployment was renamed to auto-deployment. Option now redirects to "
"--disableAutoDeployment. Left in for backwards compatibility.")
parser.add_argument("--disableAutoDeployment", dest="disableAutoDeployment", action='store_true', default=None,
help="Should auto-deployment of the user script be deactivated? If True, the user "
"script/package should be present at the same location on all workers. Default = False.")
parser.add_argument("--maxLocalJobs", default=cpu_count(),
help=f"For batch systems that support a local queue for housekeeping jobs "
f"(Mesos, GridEngine, htcondor, lsf, slurm, torque). Specifies the maximum "
f"number of these housekeeping jobs to run on the local system. The default "
f"(equal to the number of cores) is a maximum of {cpu_count()} concurrent "
f"local housekeeping jobs.")
parser.add_argument("--manualMemArgs", default=False, action='store_true', dest="manualMemArgs",
help="Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to the qsub "
"call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply alternative arguments. "
"Requires that TOIL_GRIDGENGINE_ARGS be set.")
parser.add_argument("--runCwlInternalJobsOnWorkers", dest="runCwlInternalJobsOnWorkers", action='store_true',
default=None,
help="Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes "
"instead of the primary node. If false (default), then all such jobs are run on "
"the primary node. Setting this to true can speed up the pipeline for very large "
"workflows with many sub-workflows and/or scatters, provided that the worker "
"pool is large enough.")
add_parasol_options(parser)
add_single_machine_options(parser)
add_mesos_options(parser)
add_slurm_options(parser)
add_kubernetes_options(parser)
def _applianceCluster(self, mounts=None, numCores=None):
"""
A context manager for creating and tearing down an appliance cluster.
:param dict|None mounts: Dictionary mapping host paths to container paths. Both the leader
and the worker container will be started with one -v argument per dictionary entry,
as in -v KEY:VALUE.
Beware that if KEY is a path to a directory, its entire content will be deleted
when the cluster is torn down.
:param int numCores: The number of cores to be offered by the Mesos agent process running
in the worker container.
:rtype: (ApplianceTestSupport.Appliance, ApplianceTestSupport.Appliance)
:return: A tuple of the form `(leader, worker)` containing the Appliance instances
representing the respective appliance containers
"""
if numCores is None:
numCores = cpu_count()
# The last container to stop (and the first to start) should clean the mounts.
with self.LeaderThread(self, mounts, cleanMounts=True) as leader:
with self.WorkerThread(self, mounts, numCores) as worker:
yield leader, worker
def setDefaultOptions(config):
"""
Set default options for builtin batch systems. This is required if a Config
object is not constructed from an Options object.
"""
config.batchSystem = "singleMachine"
config.disableAutoDeployment = False
config.environment = {}
config.statePollingWait = None # if not set, will default to seconds in getWaitDuration()
config.maxLocalJobs = cpu_count()
config.manualMemArgs = False
# parasol
config.parasolCommand = 'parasol'
config.parasolMaxBatches = 10000
# single machine
config.scale = 1
config.linkImports = False
config.moveExports = False
# mesos
config.mesosMasterAddress = '%s:5050' % getPublicIP()
# Kubernetes
config.kubernetesHostPath = None
def set_batchsystem_config_defaults(config) -> None:
"""
Set default options for builtin batch systems. This is required if a Config
object is not constructed from an Options object.
"""
config.batchSystem = "single_machine"
config.disableAutoDeployment = False
config.environment = {}
config.statePollingWait = None
config.maxLocalJobs = cpu_count()
config.manualMemArgs = False
config.coalesceStatusCalls = False
# parasol
config.parasolCommand = 'parasol'
config.parasolMaxBatches = 10000
# single machine
config.scale = 1
config.linkImports = False
config.moveExports = False
# mesos
config.mesosMasterAddress = f'{getPublicIP()}:5050'
# SLURM
config.allocate_mem = True
# Kubernetes
config.kubernetesHostPath = None
def testLastProcessStanding(self):
for it in range(10):
log.info('Iteration %d', it)
scope = self._createTempDir()
arena_name = 'thunderdome'
# Use processes (as opposed to threads) to prevent GIL from ordering things artificially
pool = multiprocessing.Pool(processes=cpu_count())
try:
numTasks = 100
results = pool.map_async(func=partial(
_testLastProcessStandingTask, scope, arena_name),
iterable=list(range(numTasks)))
results = results.get()
finally:
pool.close()
pool.join()
self.assertEqual(len(results), numTasks)
for item in results:
# Make sure all workers say they succeeded
self.assertEqual(item, True)
for filename in os.listdir(scope):
assert not filename.startswith(
'precious'), f"File {filename} still exists"
def addOptions(addOptionFn, config):
addOptionFn(
"--batchSystem",
dest="batchSystem",
default=defaultBatchSystem(),
choices=uniqueNames(),
help=
("The type of batch system to run the job(s) with, currently can be one "
"of %s'. default=%s" %
(', '.join(uniqueNames()), defaultBatchSystem())))
addOptionFn(
"--disableHotDeployment",
dest="disableAutoDeployment",
action='store_true',
default=None,
help=
("Hot-deployment was renamed to auto-deployment. Option now redirects to "
"--disableAutoDeployment. Left in for backwards compatibility."))
addOptionFn(
"--disableAutoDeployment",
dest="disableAutoDeployment",
action='store_true',
default=None,
help=
("Should auto-deployment of the user script be deactivated? If True, the user "
"script/package should be present at the same location on all workers. "
"default=false"))
localCores = cpu_count()
addOptionFn("--maxLocalJobs",
default=localCores,
help="For batch systems that support a local queue for "
"housekeeping jobs (Mesos, GridEngine, htcondor, lsf, slurm, "
"torque), the maximum number of these housekeeping jobs to "
"run on the local system. "
"The default (equal to the number of cores) is a maximum of "
"{} concurrent local housekeeping jobs.".format(localCores))
addOptionFn(
"--manualMemArgs",
default=False,
action='store_true',
dest="manualMemArgs",
help=
"Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to "
"the qsub call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply "
"alternative arguments. Requires that TOIL_GRIDGENGINE_ARGS be set.")
addOptionFn(
"--runCwlInternalJobsOnWorkers",
dest="runCwlInternalJobsOnWorkers",
action='store_true',
default=None,
help=
("Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes "
"instead of the primary node. If false (default), then all such jobs are run on "
"the primary node. Setting this to true can speed up the pipeline for very large "
"workflows with many sub-workflows and/or scatters, provided that the worker "
"pool is large enough."))
for o in _options:
o(addOptionFn, config)
class AbstractBatchSystemJobTest(with_metaclass(ABCMeta, ToilTest)):
"""
An abstract base class for batch system tests that use a full Toil workflow rather
than using the batch system directly.
"""
cpuCount = cpu_count()
allocatedCores = sorted({1, 2, cpuCount})
sleepTime = 5
@abstractmethod
def getBatchSystemName(self):
"""
:rtype: (str, AbstractBatchSystem)
"""
raise NotImplementedError
def getOptions(self, tempDir):
"""
Configures options for Toil workflow and makes job store.
:param str tempDir: path to test directory
:return: Toil options object
"""
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "DEBUG"
options.batchSystem = self.batchSystemName
options.workDir = tempDir
options.maxCores = self.cpuCount
return options
def setUp(self):
self.batchSystemName = self.getBatchSystemName()
super(hidden.AbstractBatchSystemJobTest, self).setUp()
def tearDown(self):
super(hidden.AbstractBatchSystemJobTest, self).tearDown()
@slow
def testJobConcurrency(self):
"""
Tests that the batch system is allocating core resources properly for concurrent tasks.
"""
for coresPerJob in self.allocatedCores:
tempDir = self._createTempDir('testFiles')
options = self.getOptions(tempDir)
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
root = Job()
for _ in range(self.cpuCount):
root.addFollowOn(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime,
cores=coresPerJob, memory='1M', disk='1Mi'))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
def set_batchsystem_config_defaults(config) -> None:
"""
Set default and environment-based options for builtin batch systems. This
is required if a Config object is not constructed from an Options object.
"""
# Do the global options across batch systems
config.batchSystem = "single_machine"
config.disableAutoDeployment = False
config.maxLocalJobs = cpu_count()
config.manualMemArgs = False
config.coalesceStatusCalls = False
config.statePollingWait: Optional[Union[
float,
int]] = None # Number of seconds to wait before querying job state
OptionType = TypeVar('OptionType')
def set_option(option_name: str,
parsing_function: Optional[Callable[[Any],
OptionType]] = None,
check_function: Optional[Callable[[OptionType],
None]] = None,
default: Optional[OptionType] = None,
env: Optional[List[str]] = None,
old_names: Optional[List[str]] = None) -> None:
"""
Function to set a batch-system-defined option to its default value, or
one from the environment.
"""
# TODO: deduplicate with Config
option_value = default
if env is not None:
for env_var in env:
# Try all the environment variables
if option_value != default:
break
option_value = os.environ.get(env_var, default)
if option_value is not None or not hasattr(config, option_name):
if parsing_function is not None:
# Parse whatever it is (string, argparse-made list, etc.)
option_value = parsing_function(option_value)
if check_function is not None:
try:
check_function(option_value)
except AssertionError:
raise RuntimeError(
f"The {option_name} option has an invalid value: {option_value}"
)
setattr(config, option_name, option_value)
# Set up defaults from all the batch systems
set_batchsystem_options(None, set_option)
def minigraph_map_all(job, config, gfa_id, fa_id_map, graph_event, keep_gaf):
""" top-level job to run the minigraph mapping in parallel, returns paf """
# hang everything on this job, to self-contain workflow
top_job = Job()
job.addChild(top_job)
mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"),
"cpu",
typeFn=int,
default=1)
mg_cores = min(mg_cores, cpu_count())
# doing the paf conversion is more efficient when done separately for each genome. we can get away
# with doing this if the universal filter (which needs to process everything at once) is disabled
xml_node = findRequiredNode(config.xmlRoot, "graphmap")
paf_per_genome = not getOptionalAttrib(xml_node, "universalMZFilter",
float)
# do the mapping
gaf_id_map = {}
paf_id_map = {}
for event, fa_path_fa_id in fa_id_map.items():
fa_path = fa_path_fa_id[0]
fa_id = fa_path_fa_id[1]
minigraph_map_job = top_job.addChildJobFn(
minigraph_map_one,
config,
event,
fa_path,
fa_id,
gfa_id,
keep_gaf or not paf_per_genome,
paf_per_genome,
# todo: estimate RAM
cores=mg_cores,
disk=5 * (fa_id.size + gfa_id.size))
gaf_id_map[event] = minigraph_map_job.rv(0)
paf_id_map[event] = minigraph_map_job.rv(1)
# convert to paf
if paf_per_genome:
paf_job = top_job.addFollowOnJobFn(merge_pafs, paf_id_map)
else:
paf_job = top_job.addFollowOnJobFn(merge_gafs_into_paf, config,
gaf_id_map)
if not keep_gaf:
gaf_id_map = None
else:
gaf_id_map = paf_job.addFollowOnJobFn(compress_gafs, gaf_id_map).rv()
return paf_job.rv(), gaf_id_map
def main():
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
addCactusWorkflowOptions(parser)
parser.add_argument("seqFile", help = "Seq file")
parser.add_argument("outputHal", type=str, help = "Output HAL file")
#Progressive Cactus Options
parser.add_argument("--configFile", dest="configFile",
help="Specify cactus configuration file",
default=os.path.join(cactusRootPath(), "cactus_progressive_config.xml"))
parser.add_argument("--root", dest="root", help="Name of ancestral node (which"
" must appear in NEWICK tree in <seqfile>) to use as a "
"root for the alignment. Any genomes not below this node "
"in the tree may be used as outgroups but will never appear"
" in the output. If no root is specifed then the root"
" of the tree is used. ", default=None)
parser.add_argument("--latest", dest="latest", action="store_true",
help="Use the latest version of the docker container "
"rather than pulling one matching this version of cactus")
parser.add_argument("--containerImage", dest="containerImage", default=None,
help="Use the the specified pre-built containter image "
"rather than pulling one from quay.io")
parser.add_argument("--binariesMode", choices=["docker", "local", "singularity"],
help="The way to run the Cactus binaries", default=None)
parser.add_argument("--database", choices=["kyoto_tycoon", "redis"],
help="The type of database", default="kyoto_tycoon")
options = parser.parse_args()
setupBinaries(options)
setLoggingFromOptions(options)
enableDumpStack()
# cactus doesn't run with 1 core
if options.batchSystem == 'singleMachine':
if options.maxCores is not None:
if int(options.maxCores) < 2:
raise RuntimeError('Cactus requires --maxCores > 1')
else:
# is there a way to get this out of Toil? That would be more consistent
if cpu_count() < 2:
raise RuntimeError('Only 1 CPU detected. Cactus requires at least 2')
# Mess with some toil options to create useful defaults.
cactus_override_toil_options(options)
start_time = timeit.default_timer()
runCactusProgressive(options)
end_time = timeit.default_timer()
run_time = end_time - start_time
logger.info("Cactus has finished after {} seconds".format(run_time))
def _startMesos(self, numCores=None):
if numCores is None:
numCores = cpu_count()
shutil.rmtree('/tmp/mesos', ignore_errors=True)
self.master = self.MesosMasterThread(numCores)
self.master.start()
self.agent = self.MesosAgentThread(numCores)
self.agent.start()
# Bad Things will happen if the master is not yet ready when Toil tries to use it.
self.wait_for_master()
log.info('Mesos is ready! Running test.')
def __init__(self, fastaID, minLength, dnabrnnOpts):
memory = 4 * 1024 * 1024 * 1024
disk = 2 * (fastaID.size)
# todo: clean up
cores = cpu_count()
RoundedJob.__init__(self,
memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.fastaID = fastaID
self.minLength = minLength
self.dnabrnnOpts = dnabrnnOpts
def __init__(self, blastOptions, seqFileID):
disk = 3 * seqFileID.size
memory = 3 * seqFileID.size
if blastOptions.gpuLastz:
# gpu jobs get the whole node
cores = cpu_count()
else:
cores = None
super(RunSelfBlast, self).__init__(memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.blastOptions = blastOptions
self.seqFileID = seqFileID
def _startParasol(self, numCores=None, memory=None):
if numCores is None:
numCores = cpu_count()
if memory is None:
memory = physicalMemory()
self.numCores = numCores
self.memory = memory
self.leader = self.ParasolLeaderThread()
self.leader.start()
self.worker = self.ParasolWorkerThread()
self.worker.start()
while self.leader.popen is None or self.worker.popen is None:
log.info('Waiting for leader and worker processes')
time.sleep(.1)
def __init__(self, repeatMaskOptions, queryID, targetIDs):
targetsSize = sum(targetID.size for targetID in targetIDs)
memory = 4 * 1024 * 1024 * 1024
disk = 2 * (queryID.size + targetsSize)
if repeatMaskOptions.gpuLastz:
# gpu jobs get the whole node (same hack as used in blast phase)
cores = cpu_count()
else:
cores = None
RoundedJob.__init__(self,
memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.repeatMaskOptions = repeatMaskOptions
self.queryID = queryID
self.targetIDs = targetIDs
def _startMesos(self, numCores=None):
if numCores is None:
numCores = cpu_count()
shutil.rmtree('/tmp/mesos', ignore_errors=True)
self.master = self.MesosMasterThread(numCores)
self.master.start()
self.agent = self.MesosAgentThread(numCores)
self.agent.start()
# Wait for the master to come up.
# Bad Things will happen if the master is not yet ready when Toil tries to use it.
for attempt in retry(predicate=lambda e: True):
with attempt:
log.info('Checking if Mesos is ready...')
with closing(urlopen('http://127.0.0.1:5050/version')) as content:
content.read()
log.info('Mesos is ready! Running test.')
def testNewJobsCanHandleOtherJobDeaths(self):
"""
Create 2 non-local files and then create 2 jobs. The first job registers a deferred job
to delete the second non-local file, deletes the first non-local file and then kills
itself. The second job waits for the first file to be deleted, then sleeps for a few
seconds and then spawns a child. the child of the second does nothing. However starting
it should handle the untimely demise of the first job and run the registered deferred
function that deletes the first file. We assert the absence of the two files at the
end of the run.
"""
# Check to make sure we can run two jobs in parallel
cpus = cpu_count()
assert cpus >= 2, "Not enough CPUs to run two tasks at once"
# There can be no retries
self.options.retryCount = 0
workdir = self._createTempDir(purpose='nonLocalDir')
nonLocalFile1 = os.path.join(workdir, str(uuid4()))
nonLocalFile2 = os.path.join(workdir, str(uuid4()))
open(nonLocalFile1, 'w').close()
open(nonLocalFile2, 'w').close()
assert os.path.exists(nonLocalFile1)
assert os.path.exists(nonLocalFile2)
files = [nonLocalFile1, nonLocalFile2]
root = Job()
# A and B here must run in parallel for this to work
A = Job.wrapJobFn(_testNewJobsCanHandleOtherJobDeaths_A,
files=files,
cores=1)
B = Job.wrapJobFn(_testNewJobsCanHandleOtherJobDeaths_B,
files=files,
cores=1)
C = Job.wrapJobFn(_testNewJobsCanHandleOtherJobDeaths_C,
files=files,
expectedResult=False,
cores=1)
root.addChild(A)
root.addChild(B)
B.addChild(C)
try:
Job.Runner.startToil(root, self.options)
except FailedJobsException as e:
pass
def __init__(self, blastOptions, seqFileID1, seqFileID2):
if hasattr(seqFileID1, "size") and hasattr(seqFileID2, "size"):
disk = 2 * (seqFileID1.size + seqFileID2.size)
memory = 2 * (seqFileID1.size + seqFileID2.size)
else:
disk = None
memory = None
if blastOptions.gpuLastz:
# gpu jobs get the whole node
cores = cpu_count()
else:
cores = None
super(RunBlast, self).__init__(memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.blastOptions = blastOptions
self.seqFileID1 = seqFileID1
self.seqFileID2 = seqFileID2
def __init__(self,
fastaID,
dnabrnnOpts,
cpu,
minLength=None,
mergeLength=None,
action=None):
memory = 4 * 1024 * 1024 * 1024
disk = 2 * (fastaID.size)
cores = min(cpu_count(), cpu)
RoundedJob.__init__(self,
memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.fastaID = fastaID
self.minLength = minLength
self.mergeLength = mergeLength
self.action = action
self.dnabrnnOpts = dnabrnnOpts
def _sendFrameworkMessage(self, driver):
message = None
while True:
# The psutil documentation recommends that we ignore the value returned by the first
# invocation of cpu_percent(). However, we do want to send a sign of life early after
# starting (e.g. to unblock the provisioner waiting for an instance to come up) so
# we call it once and discard the value.
if message is None:
message = Expando(address=self.address)
psutil.cpu_percent()
else:
message.nodeInfo = dict(coresUsed=float(psutil.cpu_percent()) * .01,
memoryUsed=float(psutil.virtual_memory().percent) * .01,
coresTotal=cpu_count(),
memoryTotal=psutil.virtual_memory().total,
workers=len(self.runningTasks))
log.debug("Send framework message: %s", message)
driver.sendFrameworkMessage(encode_data(repr(message).encode('utf-8')))
# Prevent workers launched together from repeatedly hitting the leader at the same time
time.sleep(random.randint(45, 75))
def __init__(self,
fastaID,
dnabrnnOpts,
cpu,
minLength=None,
action=None,
inputBedID=None,
eventName=None):
memory = 4 * 1024 * 1024 * 1024
disk = 2 * (fastaID.size)
cores = min(cpu_count(), cpu)
RoundedJob.__init__(self,
memory=memory,
disk=disk,
cores=cores,
preemptable=True)
self.fastaID = fastaID
self.minLength = minLength
self.action = action
self.dnabrnnOpts = dnabrnnOpts
self.inputBedID = inputBedID #todo: moved to fileMasking --> remove from here
self.eventName = eventName
def testGlobalMutexOrdering(self):
for it in range(10):
log.info('Iteration %d', it)
scope = self._createTempDir()
mutex = 'mutex'
# Use processes (as opposed to threads) to prevent GIL from ordering things artificially
pool = multiprocessing.Pool(processes=cpu_count())
try:
numTasks = 100
results = pool.map_async(func=partial(
_testGlobalMutexOrderingTask, scope, mutex),
iterable=list(range(numTasks)))
results = results.get()
finally:
pool.close()
pool.join()
self.assertEqual(len(results), numTasks)
for item in results:
# Make sure all workers say they succeeded
self.assertEqual(item, True)
def main():
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
addCactusWorkflowOptions(parser)
parser.add_argument("seqFile", help="Seq file")
parser.add_argument(
"cigarsFile",
nargs="+",
help=
"Pairiwse aliginments (from cactus-blast, cactus-refmap or cactus-graphmap)"
)
parser.add_argument("outputHal", type=str, help="Output HAL file")
parser.add_argument(
"--pathOverrides",
nargs="*",
help="paths (multiple allowd) to override from seqFile")
parser.add_argument(
"--pathOverrideNames",
nargs="*",
help="names (must be same number as --paths) of path overrides")
#Progressive Cactus Options
parser.add_argument("--configFile",
dest="configFile",
help="Specify cactus configuration file",
default=os.path.join(cactusRootPath(),
"cactus_progressive_config.xml"))
parser.add_argument(
"--root",
dest="root",
help="Name of ancestral node (which"
" must appear in NEWICK tree in <seqfile>) to use as a "
"root for the alignment. Any genomes not below this node "
"in the tree may be used as outgroups but will never appear"
" in the output. If no root is specifed then the root"
" of the tree is used. ",
default=None,
required=True)
parser.add_argument(
"--latest",
dest="latest",
action="store_true",
help="Use the latest version of the docker container "
"rather than pulling one matching this version of cactus")
parser.add_argument(
"--containerImage",
dest="containerImage",
default=None,
help="Use the the specified pre-built containter image "
"rather than pulling one from quay.io")
parser.add_argument("--binariesMode",
choices=["docker", "local", "singularity"],
help="The way to run the Cactus binaries",
default=None)
parser.add_argument(
"--nonCactusInput",
action="store_true",
help=
"Input lastz cigars do not come from cactus-blast or cactus-refmap: Prepend ids in cigars"
)
parser.add_argument(
"--pangenome",
action="store_true",
help=
"Override some CAF settings whose defaults are not suited to star trees"
)
parser.add_argument(
"--pafInput",
action="store_true",
help="'cigarsFile' arugment is in PAF format, rather than lastz cigars."
)
parser.add_argument("--database",
choices=["kyoto_tycoon", "redis"],
help="The type of database",
default="kyoto_tycoon")
options = parser.parse_args()
setupBinaries(options)
setLoggingFromOptions(options)
enableDumpStack()
if (options.pathOverrides or options.pathOverrideNames):
if not options.pathOverrides or not options.pathOverrideNames or \
len(options.pathOverrideNames) != len(options.pathOverrides):
raise RuntimeError(
'same number of values must be passed to --pathOverrides and --pathOverrideNames'
)
# cactus doesn't run with 1 core
if options.batchSystem == 'singleMachine':
if options.maxCores is not None:
if int(options.maxCores) < 2:
raise RuntimeError('Cactus requires --maxCores > 1')
else:
# is there a way to get this out of Toil? That would be more consistent
if cpu_count() < 2:
raise RuntimeError(
'Only 1 CPU detected. Cactus requires at least 2')
if options.pafInput:
# cactus-graphmap does not do any prepending to simplify interface with minigraph node names
# so it must be done here
options.nonCactusInput = True
options.buildHal = True
options.buildFasta = True
# Mess with some toil options to create useful defaults.
cactus_override_toil_options(options)
start_time = timeit.default_timer()
runCactusAfterBlastOnly(options)
end_time = timeit.default_timer()
run_time = end_time - start_time
logger.info("cactus-align has finished after {} seconds".format(run_time))
def main():
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
addCactusWorkflowOptions(parser)
parser.add_argument("seqFile", help="Seq file")
parser.add_argument(
"cigarsFile",
nargs="*",
help=
"Pairiwse aliginments (from cactus-blast, cactus-refmap or cactus-graphmap)"
)
parser.add_argument("outHal",
type=str,
help="Output HAL file (or directory in --batch mode)")
parser.add_argument(
"--pathOverrides",
nargs="*",
help="paths (multiple allowd) to override from seqFile")
parser.add_argument(
"--pathOverrideNames",
nargs="*",
help="names (must be same number as --paths) of path overrides")
#Pangenome Options
parser.add_argument(
"--pangenome",
action="store_true",
help=
"Activate pangenome mode (suitable for star trees of closely related samples) by overriding several configuration settings."
" The overridden configuration will be saved in <outHal>.pg-conf.xml")
parser.add_argument(
"--pafInput",
action="store_true",
help="'cigarsFile' arugment is in PAF format, rather than lastz cigars."
)
parser.add_argument(
"--usePafSecondaries",
action="store_true",
help=
"use the secondary alignments from the PAF input. They are ignored by default."
)
parser.add_argument("--singleCopySpecies",
type=str,
help="Filter out all self-alignments in given species")
parser.add_argument(
"--barMaskFilter",
type=int,
default=None,
help=
"BAR's POA aligner will ignore softmasked regions greater than this length. (overrides partialOrderAlignmentMaskFilter in config)"
)
parser.add_argument(
"--outVG",
action="store_true",
help="export pangenome graph in VG (.vg) in addition to HAL")
parser.add_argument(
"--outGFA",
action="store_true",
help="export pangenome grpah in GFA (.gfa.gz) in addition to HAL")
parser.add_argument(
"--batch",
action="store_true",
help=
"Launch batch of alignments. Input seqfile is expected to be chromfile as generated by cactus-graphmap-slit"
)
parser.add_argument(
"--stagger",
type=int,
help=
"Stagger alignment jobs in batch mode by this many seconds (to avoid starting all at once)",
default=0)
parser.add_argument(
"--acyclic",
type=str,
help=
"Ensure that given genome is cyclic by deleting all paralogy edges in postprocessing"
)
#Progressive Cactus Options
parser.add_argument("--configFile",
dest="configFile",
help="Specify cactus configuration file",
default=os.path.join(cactusRootPath(),
"cactus_progressive_config.xml"))
parser.add_argument(
"--root",
dest="root",
help="Name of ancestral node (which"
" must appear in NEWICK tree in <seqfile>) to use as a "
"root for the alignment. Any genomes not below this node "
"in the tree may be used as outgroups but will never appear"
" in the output. If no root is specifed then the root"
" of the tree is used. ",
default=None)
parser.add_argument(
"--latest",
dest="latest",
action="store_true",
help="Use the latest version of the docker container "
"rather than pulling one matching this version of cactus")
parser.add_argument(
"--containerImage",
dest="containerImage",
default=None,
help="Use the the specified pre-built containter image "
"rather than pulling one from quay.io")
parser.add_argument("--binariesMode",
choices=["docker", "local", "singularity"],
help="The way to run the Cactus binaries",
default=None)
parser.add_argument(
"--nonCactusInput",
action="store_true",
help=
"Input lastz cigars do not come from cactus-blast or cactus-refmap: Prepend ids in cigars"
)
parser.add_argument("--database",
choices=["kyoto_tycoon", "redis"],
help="The type of database",
default="kyoto_tycoon")
options = parser.parse_args()
setupBinaries(options)
setLoggingFromOptions(options)
enableDumpStack()
if (options.pathOverrides or options.pathOverrideNames):
if not options.pathOverrides or not options.pathOverrideNames or \
len(options.pathOverrideNames) != len(options.pathOverrides):
raise RuntimeError(
'same number of values must be passed to --pathOverrides and --pathOverrideNames'
)
# cactus doesn't run with 1 core
if options.batchSystem == 'singleMachine':
if options.maxCores is not None:
if int(options.maxCores) < 2:
raise RuntimeError('Cactus requires --maxCores > 1')
else:
# is there a way to get this out of Toil? That would be more consistent
if cpu_count() < 2:
raise RuntimeError(
'Only 1 CPU detected. Cactus requires at least 2')
options.buildHal = True
options.buildFasta = True
if options.outHal.startswith('s3://'):
if not has_s3:
raise RuntimeError(
"S3 support requires toil to be installed with [aws]")
# write a little something to the bucket now to catch any glaring problems asap
test_file = os.path.join(getTempDirectory(), 'check')
with open(test_file, 'w') as test_o:
test_o.write("\n")
region = get_aws_region(
options.jobStore) if options.jobStore.startswith('aws:') else None
write_s3(test_file,
options.outHal if options.outHal.endswith('.hal') else
os.path.join(options.outHal, 'test'),
region=region)
options.checkpointInfo = (get_aws_region(options.jobStore),
options.outHal)
else:
options.checkpointInfo = None
if options.batch:
# the output hal is a directory, make sure it's there
if not os.path.isdir(options.outHal):
os.makedirs(options.outHal)
assert len(options.cigarsFile) == 0
else:
assert len(options.cigarsFile) > 0
# Mess with some toil options to create useful defaults.
cactus_override_toil_options(options)
# We set which type of unique ids to expect. Numeric (from cactus-blast) or Eventname (cactus-refmap or cactus-grpahmap)
# This is a bit ugly, since we don't have a good way to differentiate refmap from blast, and use --pangenome as a proxy
# But I don't think there's a real use case yet of making a separate parameter
options.eventNameAsID = os.environ.get('CACTUS_EVENT_NAME_AS_UNIQUE_ID')
if options.eventNameAsID is not None:
options.eventNameAsID = False if not bool(
eventName) or eventName == '0' else True
else:
options.eventNameAsID = options.pangenome or options.pafInput
os.environ['CACTUS_EVENT_NAME_AS_UNIQUE_ID'] = str(
int(options.eventNameAsID))
start_time = timeit.default_timer()
with Toil(options) as toil:
importSingularityImage(options)
if options.restart:
results_dict = toil.restart()
else:
align_jobs = make_batch_align_jobs(options, toil)
results_dict = toil.start(
Job.wrapJobFn(run_batch_align_jobs, align_jobs))
# when using s3 output urls, things get checkpointed as they're made so no reason to export
# todo: make a more unified interface throughout cactus for this
# (see toil-vg's outstore logic which, while not perfect, would be an improvement
if not options.outHal.startswith('s3://'):
if options.batch:
for chrom, results in results_dict.items():
toil.exportFile(
results[0],
makeURL(
os.path.join(options.outHal,
'{}.hal'.format(chrom))))
if options.outVG:
toil.exportFile(
results[1],
makeURL(
os.path.join(options.outHal,
'{}.vg'.format(chrom))))
if options.outGFA:
toil.exportFile(
results[2],
makeURL(
os.path.join(options.outHal,
'{}.gfa.gz'.format(chrom))))
else:
assert len(results_dict) == 1 and None in results_dict
halID, vgID, gfaID = results_dict[None][0], results_dict[None][
1], results_dict[None][2]
# export the hal
toil.exportFile(halID, makeURL(options.outHal))
# export the vg
if options.outVG:
toil.exportFile(
vgID,
makeURL(os.path.splitext(options.outHal)[0] + '.vg'))
if options.outGFA:
toil.exportFile(
gfaID,
makeURL(
os.path.splitext(options.outHal)[0] + '.gfa.gz'))
end_time = timeit.default_timer()
run_time = end_time - start_time
logger.info("cactus-align has finished after {} seconds".format(run_time))
def add_all_batchsystem_options(
parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
# Do the global cross-batch-system arguments
parser.add_argument(
"--batchSystem",
dest="batchSystem",
default=DEFAULT_BATCH_SYSTEM,
choices=BATCH_SYSTEMS,
help=
f"The type of batch system to run the job(s) with, currently can be one "
f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}",
)
parser.add_argument(
"--disableHotDeployment",
dest="disableAutoDeployment",
action="store_true",
default=None,
help=
"Hot-deployment was renamed to auto-deployment. Option now redirects to "
"--disableAutoDeployment. Left in for backwards compatibility.",
)
parser.add_argument(
"--disableAutoDeployment",
dest="disableAutoDeployment",
action="store_true",
default=None,
help=
"Should auto-deployment of the user script be deactivated? If True, the user "
"script/package should be present at the same location on all workers. Default = False.",
)
parser.add_argument(
"--maxLocalJobs",
default=cpu_count(),
help=
f"For batch systems that support a local queue for housekeeping jobs "
f"(Mesos, GridEngine, htcondor, lsf, slurm, torque). Specifies the maximum "
f"number of these housekeeping jobs to run on the local system. The default "
f"(equal to the number of cores) is a maximum of {cpu_count()} concurrent "
f"local housekeeping jobs.",
)
parser.add_argument(
"--manualMemArgs",
default=False,
action="store_true",
dest="manualMemArgs",
help=
"Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to the qsub "
"call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply alternative arguments. "
"Requires that TOIL_GRIDGENGINE_ARGS be set.",
)
parser.add_argument(
"--runCwlInternalJobsOnWorkers",
dest="runCwlInternalJobsOnWorkers",
action="store_true",
default=None,
help=
"Whether to run CWL internal jobs (e.g. CWLScatter) on the worker nodes "
"instead of the primary node. If false (default), then all such jobs are run on "
"the primary node. Setting this to true can speed up the pipeline for very large "
"workflows with many sub-workflows and/or scatters, provided that the worker "
"pool is large enough.",
)
parser.add_argument(
"--coalesceStatusCalls",
dest="coalesceStatusCalls",
action="store_true",
default=None,
help=
("Coalese status calls to prevent the batch system from being overloaded. "
"Currently only supported for LSF. "
"default=false"),
)
parser.add_argument(
"--statePollingWait",
dest="statePollingWait",
type=int,
default=None,
help=
"Time, in seconds, to wait before doing a scheduler query for job state. "
"Return cached results if within the waiting period. Only works for grid "
"engine batch systems such as gridengine, htcondor, torque, slurm, and lsf."
)
for factory in BATCH_SYSTEM_FACTORY_REGISTRY.values():
# All the batch systems are responsible for adding their own options
# with the add_options class method.
try:
batch_system_type = factory()
except ImportError:
# Skip anything we can't import
continue
# Ask the batch system to create its options in the parser
logger.debug('Add options for %s', batch_system_type)
batch_system_type.add_options(parser)
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you
give it as they come in, but in parallel.
Uses a single "daddy" thread to manage a fleet of child processes.
Communication with the daddy thread happens via two queues: one queue of
jobs waiting to be run (the input queue), and one queue of jobs that are
finished/stopped and need to be returned by getUpdatedBatchJob (the output
queue).
When the batch system is shut down, the daddy thread is stopped.
If running in debug-worker mode, jobs are run immediately as they are sent
to the batch system, in the sending thread, and the daddy thread is not
run. But the queues are still used.
"""
@classmethod
def supportsAutoDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
self.config = config
# Limit to the smaller of the user-imposed limit and what we actually
# have on this machine for each resource.
#
# If we don't have up to the limit of the resource (and the resource
# isn't the inlimited sentinel), warn.
if maxCores > self.numCores:
if maxCores != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough cores! User limited to %i but we only have %i.',
maxCores, self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
if maxMemory != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough memory! User limited to %i bytes but we only have %i bytes.',
maxMemory, self.physicalMemory)
maxMemory = self.physicalMemory
workdir = Toil.getLocalWorkflowDir(
config.workflowID, config.workDir
) # config.workDir may be None; this sets a real directory
self.physicalDisk = toil.physicalDisk(workdir)
if maxDisk > self.physicalDisk:
if maxDisk != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough disk space! User limited to %i bytes but we only have %i bytes.',
maxDisk, self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores,
maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
if config.badWorker > 0 and config.debugWorker:
# We can't throw SIGUSR1 at the worker because it is also going to
# be the leader and/or test harness.
raise RuntimeError(
"Cannot use badWorker and debugWorker together; "
"worker would have to kill the leader")
self.debugWorker = config.debugWorker
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs: Dict[str, toil.job.JobDescription] = {}
# A queue of jobs waiting to be executed. Consumed by the daddy thread.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the daddy thread.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs: Dict[str, Info] = {}
# These next two are only used outside debug-worker mode
# A dict mapping PIDs to Popen objects for running jobs.
# Jobs that don't fork are executed one at a time in the main thread.
self.children: Dict[int, subprocess.Popen] = {}
# A dict mapping child PIDs to the Job IDs they are supposed to be running.
self.childToJob: Dict[int, str] = {}
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(int(self.maxCores / self.minCores),
'cores')
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory')
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk')
# If we can't schedule something, we fill this in with a reason why
self.schedulingStatusMessage = None
# We use this event to signal shutdown
self.shuttingDown = Event()
# A thread in charge of managing all our child processes.
# Also takes care of resource accounting.
self.daddyThread = None
# If it breaks it will fill this in
self.daddyException: Optional[Exception] = None
if self.debugWorker:
log.debug('Started batch system %s in worker debug mode.',
id(self))
else:
self.daddyThread = Thread(target=self.daddy, daemon=True)
self.daddyThread.start()
log.debug('Started batch system %s in normal mode.', id(self))
def daddy(self):
"""
Be the "daddy" thread.
Our job is to look at jobs from the input queue.
If a job fits in the available resources, we allocate resources for it
and kick off a child process.
We also check on our children.
When a child finishes, we reap it, release its resources, and put its
information in the output queue.
"""
try:
log.debug('Started daddy thread for batch system %s.', id(self))
while not self.shuttingDown.is_set():
# Main loop
while not self.shuttingDown.is_set():
# Try to start as many jobs as we can try to start
try:
# Grab something from the input queue if available.
args = self.inputQueue.get_nowait()
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
coreFractions = int(jobCores / self.minCores)
# Try to start the child
result = self._startChild(jobCommand, jobID,
coreFractions, jobMemory,
jobDisk, environment)
if result is None:
# We did not get the resources to run this job.
# Requeue last, so we can look at the next job.
# TODO: Have some kind of condition the job can wait on,
# but without threads (queues for jobs needing
# cores/memory/disk individually)?
self.inputQueue.put(args)
break
# Otherwise it's a PID if it succeeded, or False if it couldn't
# start. But we don't care either way here.
except Empty:
# Nothing to run. Stop looking in the queue.
break
# Now check on our children.
for done_pid in self._pollForDoneChildrenIn(self.children):
# A child has actually finished.
# Clean up after it.
self._handleChild(done_pid)
# Then loop again: start and collect more jobs.
# TODO: It would be good to be able to wait on a new job or a finished child, whichever comes first.
# For now we just sleep and loop.
time.sleep(0.01)
# When we get here, we are shutting down.
log.debug(
'Daddy thread cleaning up %d remaining children for batch system %s...',
len(self.children), id(self))
self._stop_and_wait(self.children.values())
log.debug(
'Daddy thread for batch system %s finishing because no children should now exist',
id(self))
# Then exit the thread.
return
except Exception as e:
log.critical(
'Unhandled exception in daddy thread for batch system %s: %s',
id(self), traceback.format_exc())
# Pass the exception back to the main thread so it can stop the next person who calls into us.
self.daddyException = e
raise
def _checkOnDaddy(self):
if self.daddyException is not None:
# The daddy thread broke and we cannot do our job
log.critical(
'Propagating unhandled exception in daddy thread to main thread'
)
exc = self.daddyException
self.daddyException = None
if isinstance(exc, Exception):
raise exc
else:
raise TypeError(
f'Daddy thread failed with non-exception: {exc}')
def _stop_now(self, popens: Sequence[subprocess.Popen]) -> List[int]:
"""
Stop the given child processes and all their children. Does not reap them.
Returns a list of PGIDs killed, where processes may exist that have not
yet received their kill signals.
"""
# We will potentially need to poll these PGIDs to ensure that all
# processes in them are gone.
pgids = []
for popen in popens:
# Kill all the children
if popen.returncode is None:
# Process is not known to be dead. Try and grab its group.
try:
pgid = os.getpgid(popen.pid)
except OSError:
# It just died. Assume the pgid was its PID.
pgid = popen.pid
else:
# It is dead. Try it's PID as a PGID and hope we didn't re-use it.
pgid = popen.pid
if pgid != os.getpgrp():
# The child process really is in its own group, and not ours.
# Kill the group, which hopefully hasn't been reused
log.debug(
'Send shutdown kill to process group %s known to batch system %s',
pgid, id(self))
try:
os.killpg(pgid, signal.SIGKILL)
pgids.append(pgid)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
else:
# Kill the subprocess again through popen in case it somehow
# never managed to make the group.
popen.kill()
return pgids
def _stop_and_wait(self, popens: Sequence[subprocess.Popen]) -> None:
"""
Stop the given child processes and all their children. Blocks until the
processes are gone.
"""
pgids = self._stop_now(popens)
for popen in popens:
# Wait on all the children
popen.wait()
log.debug(
'Process %s known to batch system %s is stopped; it returned %s',
popen.pid, id(self), popen.returncode)
for pgid in pgids:
try:
while True:
# Send a kill to the group again, to see if anything in it
# is still alive. Our first kill might not have been
# delivered yet.
os.killpg(pgid, signal.SIGKILL)
# If that worked it is still alive, so wait for the kernel
# to stop fooling around and kill it.
log.warning(
'Sent redundant shutdown kill to surviving process group %s known to batch system %s',
pgid, id(self))
time.sleep(0.1)
except ProcessLookupError:
# The group is actually gone now.
pass
except PermissionError:
# The group is not only gone but reused
pass
def _pollForDoneChildrenIn(self, pid_to_popen):
"""
See if any children represented in the given dict from PID to Popen
object have finished.
Return a collection of their PIDs.
Guarantees that each child's exit code will be gettable via wait() on
the child's Popen object (i.e. does not reap the child, unless via
Popen).
"""
# We keep our found PIDs in a set so we can work around waitid showing
# us the same one repeatedly.
ready = set()
# Find the waitid function
waitid = getattr(os, 'waitid', None)
if callable(waitid):
# waitid exists (not Mac)
while True:
# Poll for any child to have exit, but don't reap it. Leave reaping
# to the Popen.
# TODO: What if someone else in Toil wants to do this syscall?
# TODO: Is this one-notification-per-done-child with WNOHANG? Or
# can we miss some? Or do we see the same one repeatedly until it
# is reaped?
try:
siginfo = waitid(os.P_ALL, -1,
os.WEXITED | os.WNOWAIT | os.WNOHANG)
except ChildProcessError:
# This happens when there is nothing to wait on right now,
# instead of the weird C behavior of overwriting a field in
# a pointed-to struct.
siginfo = None
if siginfo is not None and siginfo.si_pid in pid_to_popen and siginfo.si_pid not in ready:
# Something new finished
ready.add(siginfo.si_pid)
else:
# Nothing we own that we haven't seen before has finished.
return ready
else:
# On Mac there's no waitid and no way to wait and not reap.
# Fall back on polling all the Popen objects.
# To make this vaguely efficient we have to return done children in
# batches.
for pid, popen in pid_to_popen.items():
if popen.poll() is not None:
# Process is done
ready.add(pid)
log.debug('Child %d has stopped', pid)
# Return all the done processes we found
return ready
def _runDebugJob(self, jobCommand, jobID, environment):
"""
Run the jobCommand right now, in the current thread.
May only be called in debug-worker mode.
Assumes resources are available.
"""
assert self.debugWorker
# TODO: It is not possible to kill running jobs in forkless mode,
# because they are run immediately in the main thread.
info = Info(time.time(), None, None, killIntended=False)
self.runningJobs[jobID] = info
if jobCommand.startswith("_toil_worker "):
# We can actually run in this thread
jobName, jobStoreLocator, jobStoreID = jobCommand.split()[
1:4] # Parse command
jobStore = Toil.resumeJobStore(jobStoreLocator)
toil_worker.workerScript(
jobStore,
jobStore.config,
jobName,
jobStoreID,
redirectOutputToLogFile=not self.debugWorker
) # Call the worker
else:
# Run synchronously. If starting or running the command fails, let the exception stop us.
subprocess.check_call(jobCommand,
shell=True,
env=dict(os.environ, **environment))
self.runningJobs.pop(jobID)
if not info.killIntended:
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=0,
wallTime=time.time() - info.time,
exitReason=None))
def getSchedulingStatusMessage(self):
# Implement the abstractBatchSystem's scheduling status message API
return self.schedulingStatusMessage
def _setSchedulingStatusMessage(self, message):
"""
If we can't run a job, we record a short message about why not. If the
leader wants to know what is up with us (for example, to diagnose a
deadlock), it can ask us for the message.
"""
self.schedulingStatusMessage = message
def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk,
environment):
"""
Start a child process for the given job.
Allocate its required resources and save it and save it in our bookkeeping structures.
If the job is started, returns its PID.
If the job fails to start, reports it as failed and returns False.
If the job cannot get the resources it needs to start, returns None.
"""
# We fill this in if we manage to actually start the child.
popen = None
# This is when we started working on the job.
startTime = time.time()
# See if we can fit the job in our resource pools right now.
if self.coreFractions.acquireNow(coreFractions):
# We got some cores
if self.memory.acquireNow(jobMemory):
# We got some memory
if self.disk.acquireNow(jobDisk):
# We got the final resource, disk.
# Actually run the job.
# When it finishes we will release what it was using.
# So it is important to not lose track of the child process.
try:
# Launch the job.
# Make sure it is in its own session (and thus its own
# process group) so that, if the user signals the
# workflow, Toil will be responsible for killing the
# job. This also makes sure that we can signal the job
# and all its children together. We assume that the
# process group ID will equal the PID of the process we
# are starting.
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(
os.environ,
**environment),
start_new_session=True)
except Exception:
# If the job can't start, make sure we release resources now
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.error('Could not start job %s: %s', jobID,
traceback.format_exc())
# Report as failed.
self.outputQueue.put(
UpdatedBatchJobInfo(
jobID=jobID,
exitStatus=EXIT_STATUS_UNAVAILABLE_VALUE,
wallTime=0,
exitReason=None))
# Free resources
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
# Complain it broke.
return False
else:
# If the job did start, record it
self.children[popen.pid] = popen
# Make sure we can look it up by PID later
self.childToJob[popen.pid] = jobID
# Record that the job is running, and the resources it is using
info = Info(startTime,
popen, (coreFractions, jobMemory, jobDisk),
killIntended=False)
self.runningJobs[jobID] = info
log.debug('Launched job %s as child %d', jobID,
popen.pid)
# Report success starting the job
# Note that if a PID were somehow 0 it would look like False
assert popen.pid != 0
return popen.pid
else:
# We can't get disk, so free cores and memory
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self._setSchedulingStatusMessage(
'Not enough disk to run job %s' % jobID)
else:
# Free cores, since we can't get memory
self.coreFractions.release(coreFractions)
self._setSchedulingStatusMessage(
'Not enough memory to run job %s' % jobID)
else:
self._setSchedulingStatusMessage('Not enough cores to run job %s' %
jobID)
# If we get here, we didn't succeed or fail starting the job.
# We didn't manage to get the resources.
# Report that.
return None
def _handleChild(self, pid: int) -> None:
"""
Handle a child process PID that has finished.
The PID must be for a child job we started.
Not thread safe to run at the same time as we are making more children.
Remove the child from our bookkeeping structures and free its resources.
"""
# Look up the child
popen = self.children[pid]
jobID = self.childToJob[pid]
info = self.runningJobs[jobID]
# Unpack the job resources
(coreFractions, jobMemory, jobDisk) = info.resources
# Clean up our records of the job.
self.runningJobs.pop(jobID)
self.childToJob.pop(pid)
self.children.pop(pid)
if popen.returncode is None or not callable(getattr(
os, 'waitid', None)):
# It isn't reaped yet, or we have to reap all children to see if thay're done.
# Before we reap it (if possible), kill its PID as a PGID to make sure
# it isn't leaving children behind.
# TODO: This is a PGID re-use risk on Mac because the process is
# reaped already and the PGID may have been reused.
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
# See how the child did, and reap it.
statusCode = popen.wait()
if statusCode != 0 and not info.killIntended:
log.error("Got exit code %i (indicating failure) "
"from job %s.", statusCode, self.jobs[jobID])
if not info.killIntended:
# Report if the job failed and we didn't kill it.
# If we killed it then it shouldn't show up in the queue.
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=statusCode,
wallTime=time.time() - info.time,
exitReason=None))
# Make absolutely sure all processes in the group have received their
# kill signals and been cleaned up.
# TODO: this opens a PGID reuse risk; we reaped the process and its
# PGID may have been re-used. But it probably hasn't been and we
# definitely want to make sure all its children died before saying the
# job is done. Some might not be dead yet if we don't do this.
# TODO: can we safely do this before reaping? Or would we sit forever
# signaling a dead but unreaped process?
try:
while True:
# Send a kill to the group again, to see if anything in it
# is still alive. Our first kill might not have been
# delivered yet.
os.killpg(pid, signal.SIGKILL)
# If that worked it is still alive, so wait for the kernel
# to stop fooling around and kill it.
log.warning(
'Sent redundant job completion kill to surviving process group %s known to batch system %s',
pid, id(self))
time.sleep(0.1)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
# Free up the job's resources.
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.debug('Child %d for job %s succeeded', pid, jobID)
def issueBatchJob(self, jobDesc):
"""Adds the command and resources to a queue to be run."""
self._checkOnDaddy()
# Round cores to minCores and apply scale.
# Make sure to give minCores even if asked for 0 cores, or negative or something.
cores = max(
math.ceil(jobDesc.cores * self.scale / self.minCores) *
self.minCores, self.minCores)
# Don't do our own assertions about job size vs. our configured size.
# The abstract batch system can handle it.
self.checkResourceRequest(jobDesc.memory,
cores,
jobDesc.disk,
job_name=jobDesc.jobName,
detail=f'Scale is set to {self.scale}.')
log.debug(
f"Issuing the command: {jobDesc.command} with "
f"memory: {jobDesc.memory}, cores: {cores}, disk: {jobDesc.disk}")
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobDesc.command
if self.debugWorker:
# Run immediately, blocking for return.
# Ignore resource requirements; we run one job at a time
self._runDebugJob(jobDesc.command, jobID, self.environment.copy())
else:
# Queue the job for later
self.inputQueue.put(
(jobDesc.command, jobID, cores, jobDesc.memory, jobDesc.disk,
self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs: Sequence[str]) -> None:
"""Kills jobs by ID."""
self._checkOnDaddy()
log.debug('Killing jobs: {}'.format(jobIDs))
# Collect the popen handles for the jobs we have to stop
popens: List[subprocess.Popen] = []
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
if info.popen is not None:
popens.append(info.popen)
else:
# No popen if running in forkless mode currently
assert self.debugWorker
log.critical("Can't kill job: %s in debug mode" % jobID)
# Stop them all in a batch. Don't reap, because we need the daddy
# thread to reap them to mark the jobs as not running anymore.
self._stop_now(popens)
for jobID in jobIDs:
while jobID in self.runningJobs:
# Wait for the daddy thread to collect them.
time.sleep(0.01)
def getIssuedBatchJobIDs(self):
"""Just returns all the jobs that have been run, but not yet returned as updated."""
self._checkOnDaddy()
return list(self.jobs.keys())
def getRunningBatchJobIDs(self):
self._checkOnDaddy()
now = time.time()
return {
jobID: now - info.time
for jobID, info in list(self.runningJobs.items())
}
def shutdown(self):
"""
Cleanly terminate and join daddy thread.
"""
if self.daddyThread is not None:
# Tell the daddy thread to stop.
self.shuttingDown.set()
# Wait for it to stop.
self.daddyThread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""Returns a tuple of a no-longer-running job, the return value of its process, and its runtime, or None."""
self._checkOnDaddy()
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
self.jobs.pop(item.jobID)
log.debug("Ran jobID: %s with exit value: %i", item.jobID,
item.exitStatus)
return item
@classmethod
def setOptions(cls, setOption):
setOption("scale", default=1)
def testAvailableCores(self):
self.assertTrue(cpu_count() >= numCores)
def main():
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
addCactusWorkflowOptions(parser)
parser.add_argument("seqFile", help = "Seq file")
parser.add_argument("blastOutput", nargs="+", help = "Blast output (from cactus-blast)")
parser.add_argument("outputHal", type=str, help = "Output HAL file")
parser.add_argument("--pathOverrides", nargs="*", help="paths (multiple allowd) to override from seqFile")
parser.add_argument("--pathOverrideNames", nargs="*", help="names (must be same number as --paths) of path overrides")
#Progressive Cactus Options
parser.add_argument("--configFile", dest="configFile",
help="Specify cactus configuration file",
default=os.path.join(cactusRootPath(), "cactus_progressive_config.xml"))
parser.add_argument("--root", dest="root", help="Name of ancestral node (which"
" must appear in NEWICK tree in <seqfile>) to use as a "
"root for the alignment. Any genomes not below this node "
"in the tree may be used as outgroups but will never appear"
" in the output. If no root is specifed then the root"
" of the tree is used. ", default=None, required=True)
parser.add_argument("--latest", dest="latest", action="store_true",
help="Use the latest version of the docker container "
"rather than pulling one matching this version of cactus")
parser.add_argument("--containerImage", dest="containerImage", default=None,
help="Use the the specified pre-built containter image "
"rather than pulling one from quay.io")
parser.add_argument("--binariesMode", choices=["docker", "local", "singularity"],
help="The way to run the Cactus binaries", default=None)
parser.add_argument("--nonBlastInput", action="store_true",
help="Input does not come from cactus-blast: Do not append ids to fasta names")
parser.add_argument("--nonBlastMegablockFilter", action="store_true",
help="By default, the megablock filter is off for --nonBlastInput, as it does not play"
"nicely with reference-based alignments. This flag will turn it back on")
parser.add_argument("--pafInput", action="store_true",
help="'blastOutput' input is in paf format, rather than lastz cigars.")
options = parser.parse_args()
setupBinaries(options)
setLoggingFromOptions(options)
enableDumpStack()
if (options.pathOverrides or options.pathOverrideNames):
if not options.pathOverrides or not options.pathOverrideNames or \
len(options.pathOverrideNames) != len(options.pathOverrides):
raise RuntimeError('same number of values must be passed to --pathOverrides and --pathOverrideNames')
# cactus doesn't run with 1 core
if options.batchSystem == 'singleMachine':
if options.maxCores is not None:
if int(options.maxCores) < 2:
raise RuntimeError('Cactus requires --maxCores > 1')
else:
# is there a way to get this out of Toil? That would be more consistent
if cpu_count() < 2:
raise RuntimeError('Only 1 CPU detected. Cactus requires at least 2')
# tokyo_cabinet is no longer supported
options.database = "kyoto_tycoon"
options.database = 'kyoto_tycoon'
options.buildHal = True
options.buildFasta = True
# Mess with some toil options to create useful defaults.
cactus_override_toil_options(options)
start_time = timeit.default_timer()
runCactusAfterBlastOnly(options)
end_time = timeit.default_timer()
run_time = end_time - start_time
logger.info("cactus-align has finished after {} seconds".format(run_time))
def main():
parser = ArgumentParser()
Job.Runner.addToilOptions(parser)
addCactusWorkflowOptions(parser)
parser.add_argument("seqFile", help="Seq file")
parser.add_argument("outputHal", type=str, help="Output HAL file")
#Progressive Cactus Options
parser.add_argument("--configFile",
dest="configFile",
help="Specify cactus configuration file",
default=os.path.join(cactusRootPath(),
"cactus_progressive_config.xml"))
parser.add_argument(
"--root",
dest="root",
help="Name of ancestral node (which"
" must appear in NEWICK tree in <seqfile>) to use as a "
"root for the alignment. Any genomes not below this node "
"in the tree may be used as outgroups but will never appear"
" in the output. If no root is specifed then the root"
" of the tree is used. ",
default=None)
parser.add_argument(
"--latest",
dest="latest",
action="store_true",
help="Use the latest version of the docker container "
"rather than pulling one matching this version of cactus")
parser.add_argument(
"--containerImage",
dest="containerImage",
default=None,
help="Use the the specified pre-built containter image "
"rather than pulling one from quay.io")
parser.add_argument("--binariesMode",
choices=["docker", "local", "singularity"],
help="The way to run the Cactus binaries",
default=None)
options = parser.parse_args()
setupBinaries(options)
setLoggingFromOptions(options)
enableDumpStack()
# cactus doesn't run with 1 core
if options.batchSystem == 'singleMachine':
if options.maxCores is not None:
if int(options.maxCores) < 2:
raise RuntimeError('Cactus requires --maxCores > 1')
else:
# is there a way to get this out of Toil? That would be more consistent
if cpu_count() < 2:
raise RuntimeError(
'Only 1 CPU detected. Cactus requires at least 2')
# tokyo_cabinet is no longer supported
options.database = "kyoto_tycoon"
# Mess with some toil options to create useful defaults.
# Caching generally slows down the cactus workflow, plus some
# methods like readGlobalFileStream don't support forced
# reads directly from the job store rather than from cache.
options.disableCaching = True
# Job chaining breaks service termination timing, causing unused
# databases to accumulate and waste memory for no reason.
options.disableChaining = True
# The default deadlockWait is currently 60 seconds. This can cause
# issues if the database processes take a while to actually begin
# after they're issued. Change it to at least an hour so that we
# don't preemptively declare a deadlock.
if options.deadlockWait is None or options.deadlockWait < 3600:
options.deadlockWait = 3600
if options.retryCount is None and options.batchSystem != 'singleMachine':
# If the user didn't specify a retryCount value, make it 5
# instead of Toil's default (1).
options.retryCount = 5
start_time = timeit.default_timer()
runCactusProgressive(options)
end_time = timeit.default_timer()
run_time = end_time - start_time
logger.info("Cactus has finished after {} seconds".format(run_time))
