def _startParasol(self, numCores=None, memory=None):
if numCores is None:
numCores = cpu_count()
if memory is None:
memory = physicalMemory()
self.numCores = numCores
self.memory = memory
self.leader = self.ParasolLeaderThread()
self.leader.start()
self.worker = self.ParasolWorkerThread()
self.worker.start()
while self.leader.popen is None or self.worker.popen is None:
log.info('Waiting for leader and worker processes')
time.sleep(.1)
def _startParasol(self, numCores=None, memory=None):
if numCores is None:
numCores = multiprocessing.cpu_count()
if memory is None:
memory = physicalMemory()
self.numCores = numCores
self.memory = memory
self.leader = self.ParasolLeaderThread()
self.leader.start()
self.worker = self.ParasolWorkerThread()
self.worker.start()
while self.leader.popen is None or self.worker.popen is None:
log.info('Waiting for leader and worker processes')
time.sleep(.1)
def prepare_input(job, sample, config, enqueue_consolidation=True):
# job prep
config = argparse.Namespace(**vars(config))
uuid, url, contig_name, reference_url, params_url = sample
config.uuid = uuid
config.contig_name = contig_name
config.reference_url = reference_url
config.params_url = params_url
if config.intermediate_file_location is not None:
config.intermediate_file_location = os.path.join(
config.intermediate_file_location, uuid)
mkdir_p(config.intermediate_file_location)
work_dir = job.fileStore.getLocalTempDir()
start = time.time()
log(job, "{}".format(datetime.datetime.now()), config.uuid, 'START')
log(
job,
"Preparing input with URL:{}, contig:{}, reference_url:{}, params_url:{}"
.format(url, contig_name, reference_url,
params_url), uuid, 'prepare_input')
# todo global resource estimation
config.maxCores = min(config.maxCores, multiprocessing.cpu_count())
config.defaultCores = min(MP_CPU, config.maxCores)
config.maxMemory = min(config.maxMemory, int(physicalMemory() * .95))
#config.disk
# download references - TOIL_JOBSTORE_PROTOCOL queries are so this function can be imported
#ref fasta
if reference_url.startswith(TOIL_JOBSTORE_PROTOCOL):
ref_genome_fileid = reference_url.replace(TOIL_JOBSTORE_PROTOCOL, '',
1)
ref_genome_filename = "{}.reference.{}.fa".format(uuid, contig_name)
job.fileStore.readGlobalFile(
ref_genome_fileid, os.path.join(work_dir, ref_genome_filename))
else:
download_url(reference_url, work_dir=work_dir)
ref_genome_filename = os.path.basename(reference_url)
ref_genome_fileid = job.fileStore.writeGlobalFile(
os.path.join(work_dir, ref_genome_filename))
ref_genome_size = os.stat(os.path.join(work_dir,
ref_genome_filename)).st_size
config.reference_genome_fileid = ref_genome_fileid
#params
if params_url.startswith(TOIL_JOBSTORE_PROTOCOL):
params_fileid = params_url.replace(TOIL_JOBSTORE_PROTOCOL, '', 1)
else:
download_url(params_url, work_dir=work_dir)
params_filename = os.path.basename(params_url)
params_fileid = job.fileStore.writeGlobalFile(
os.path.join(work_dir, params_filename))
config.params_fileid = params_fileid
# download bam
if url.startswith(TOIL_JOBSTORE_PROTOCOL):
bam_filename = "{}.input.{}.bam".format(uuid, contig_name)
job.fileStore.readGlobalFile(
url.replace(TOIL_JOBSTORE_PROTOCOL, '', 1),
os.path.join(work_dir, bam_filename))
else:
download_url(url, work_dir=work_dir)
bam_filename = os.path.basename(url)
data_bam_location = os.path.join("/data", bam_filename)
workdir_bam_location = os.path.join(work_dir, bam_filename)
# index the bam
_index_bam(job, config, work_dir, bam_filename)
# sanity check
workdir_bai_location = os.path.join(work_dir, bam_filename + ".bai")
if not os.path.isfile(workdir_bai_location):
raise UserError("BAM index file not created for {}: {}".format(
bam_filename, workdir_bai_location))
# get start and end location
start_idx = sys.maxint
end_idx = 0
with closing(
pysam.AlignmentFile(
workdir_bam_location,
'rb' if bam_filename.endswith("bam") else 'r')) as aln:
for read in aln.fetch():
align_start = read.reference_start
align_end = read.reference_end
start_idx = min([start_idx, align_start])
end_idx = max([end_idx, align_end])
log(job, "start_pos:{}, end_pos:{}".format(config.uuid, start_idx,
end_idx), uuid, 'prepare_input')
# get reads from positions
chunk_infos = list()
idx = start_idx
while idx < end_idx:
ci = {CI_UUID: uuid}
ci[CI_CHUNK_BOUNDARY_START] = idx
chunk_start = idx - config.partition_margin
ci[CI_CHUNK_START] = chunk_start
idx += config.partition_size
ci[CI_CHUNK_BOUNDARY_END] = idx
chunk_end = idx + config.partition_margin
ci[CI_CHUNK_END] = chunk_end
chunk_infos.append(ci)
# enqueue jobs
log(job, "Enqueueing {} jobs".format(len(chunk_infos)), uuid,
'prepare_input')
idx = 0
enqueued_jobs = 0
returned_tarballs = list()
for ci in chunk_infos:
#prep
ci[CI_CHUNK_INDEX] = idx
chunk_start = ci[CI_CHUNK_START]
chunk_end = ci[CI_CHUNK_END]
chunk_position_description = "{}:{}-{}".format(config.contig_name,
chunk_start, chunk_end)
bam_split_command = [
"view", "-b", data_bam_location, chunk_position_description
]
chunk_name = "{}.{}.bam".format(config.uuid, idx)
#write chunk
chunk_location = os.path.join(work_dir, chunk_name)
with open(chunk_location, 'w') as out:
docker_call(job,
config,
work_dir,
bam_split_command,
DOCKER_SAMTOOLS_IMG,
DOCKER_SAMTOOLS_TAG,
outfile=out)
#document read count
chunk_size = os.stat(chunk_location).st_size
ci[CI_CHUNK_SIZE] = chunk_size
ci[CI_REF_FA_SIZE] = ref_genome_size
read_count = prepare_input__get_bam_read_count(job, work_dir,
chunk_name)
ci[CI_READ_COUNT] = read_count
log(
job,
"chunk from {} for idx {} is {}b ({}mb) and has {} reads".format(
chunk_position_description, idx, chunk_size,
int(chunk_size / 1024 / 1024),
read_count), uuid, 'prepare_input')
if config.intermediate_file_location is not None:
copy_files(file_paths=[chunk_location],
output_dir=config.intermediate_file_location)
# enqueue marginPhase job
if read_count > 0:
chunk_fileid = job.fileStore.writeGlobalFile(chunk_location)
mp_cores = config.defaultCores
mp_mem = int(
min(
int(chunk_size * MP_MEM_BAM_FACTOR +
ref_genome_size * MP_MEM_REF_FACTOR),
config.maxMemory))
mp_disk = int(
min(
int(chunk_size * MP_DSK_BAM_FACTOR +
ref_genome_size * MP_DSK_REF_FACTOR +
(0 if config.cpecan_probabilities else
MP_DSK_CPECAN_FACTOR) * chunk_size), config.maxDisk))
log(
job,
"requesting {} cores, {}b ({}mb) disk, {}b ({}gb) mem".format(
mp_cores, mp_disk, int(mp_disk / 1024 / 1024), mp_mem,
int(mp_mem / 1024 / 1024 / 1024)),
"{}.{}".format(uuid, idx), 'prepare_input')
mp_mem = str(int(mp_mem / 1024)) + "K"
mp_disk = str(int(mp_disk) / 1024) + "K"
margin_phase_job = job.addChildJobFn(run_margin_phase,
config,
chunk_fileid,
ci,
memory=mp_mem,
cores=mp_cores,
disk=mp_disk)
returned_tarballs.append(margin_phase_job.rv())
enqueued_jobs += 1
idx += 1
log(job, "Enqueued {} jobs".format(enqueued_jobs), uuid, 'prepare_input')
# enqueue merging and consolidation job
merge_job = job.addFollowOnJobFn(merge_chunks, config, returned_tarballs)
final_return_value = merge_job.rv()
if enqueue_consolidation:
consolidation_job = merge_job.addFollowOnJobFn(consolidate_output,
config, merge_job.rv())
final_return_value = consolidation_job.rv()
# log
log_generic_job_debug(job, config.uuid, 'prepare_input', work_dir=work_dir)
log_time(job, "prepare_input", start, config.uuid)
# return appropriate output
return final_return_value
def testNestedResourcesDoNotBlock(self):
"""
Resources are requested in the order Memory > Cpu > Disk.
Test that inavailability of cpus for one job that is scheduled does not block another job
that can run.
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
options.maxCores = 4
from toil import physicalMemory
availableMemory = physicalMemory()
options.batchSystem = self.batchSystemName
outFile = os.path.join(tempDir, 'counter')
open(outFile, 'w').close()
root = Job()
blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
cores=2, memory='1M', disk='1M')
firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
cores=1, memory='1M', disk='1M')
secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
writeVal='sJ', cores=1, memory='1M', disk='1M')
# Should block off 50% of memory while waiting for it's 3 cores
firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
writeVal='fJC', cores=3, memory=int(old_div(availableMemory,2)), disk='1M')
# These two shouldn't be able to run before B because there should be only
# (50% of memory - 1M) available (firstJobChild should be blocking 50%)
secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJC', cores=2, memory=int(old_div(availableMemory,1.5)),
disk='1M')
secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJGC', cores=2, memory=int(old_div(availableMemory,1.5)),
disk='1M')
root.addChild(blocker)
root.addChild(firstJob)
root.addChild(secondJob)
firstJob.addChild(firstJobChild)
secondJob.addChild(secondJobChild)
secondJobChild.addChild(secondJobGrandChild)
"""
The tree is:
root
/ | \
b fJ sJ
| |
fJC sJC
|
sJGC
But the order of execution should be
root > b , fJ, sJ > sJC > sJGC > fJC
since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the
resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC
should not block them, and should only run after they finish.
"""
Job.Runner.startToil(root, options)
with open(outFile) as oFH:
outString = oFH.read()
# The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
# time. We look for all possible permutations.
possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])])
assert outString.startswith(possibleStarts)
assert outString.endswith('sJCsJGCfJC')
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you
give it as they come in, but in parallel.
Uses a single "daddy" thread to manage a fleet of child processes.
Communication with the daddy thread happens via two queues: one queue of
jobs waiting to be run (the input queue), and one queue of jobs that are
finished/stopped and need to be returned by getUpdatedBatchJob (the output
queue).
When the batch system is shut down, the daddy thread is stopped.
If running in debug-worker mode, jobs are run immediately as they are sent
to the batch system, in the sending thread, and the daddy thread is not
run. But the queues are still used.
"""
@classmethod
def supportsAutoDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
self.config = config
# Limit to the smaller of the user-imposed limit and what we actually
# have on this machine for each resource.
#
# If we don't have up to the limit of the resource (and the resource
# isn't the inlimited sentinel), warn.
if maxCores > self.numCores:
if maxCores != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough cores! User limited to %i but we only have %i.',
maxCores, self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
if maxMemory != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough memory! User limited to %i bytes but we only have %i bytes.',
maxMemory, self.physicalMemory)
maxMemory = self.physicalMemory
workdir = Toil.getLocalWorkflowDir(
config.workflowID, config.workDir
) # config.workDir may be None; this sets a real directory
self.physicalDisk = toil.physicalDisk(workdir)
if maxDisk > self.physicalDisk:
if maxDisk != sys.maxsize:
# We have an actually specified limit and not the default
log.warning(
'Not enough disk space! User limited to %i bytes but we only have %i bytes.',
maxDisk, self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores,
maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
if config.badWorker > 0 and config.debugWorker:
# We can't throw SIGUSR1 at the worker because it is also going to
# be the leader and/or test harness.
raise RuntimeError(
"Cannot use badWorker and debugWorker together; "
"worker would have to kill the leader")
self.debugWorker = config.debugWorker
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs: Dict[str, toil.job.JobDescription] = {}
# A queue of jobs waiting to be executed. Consumed by the daddy thread.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the daddy thread.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs: Dict[str, Info] = {}
# These next two are only used outside debug-worker mode
# A dict mapping PIDs to Popen objects for running jobs.
# Jobs that don't fork are executed one at a time in the main thread.
self.children: Dict[int, subprocess.Popen] = {}
# A dict mapping child PIDs to the Job IDs they are supposed to be running.
self.childToJob: Dict[int, str] = {}
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(int(self.maxCores / self.minCores),
'cores')
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory')
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk')
# If we can't schedule something, we fill this in with a reason why
self.schedulingStatusMessage = None
# We use this event to signal shutdown
self.shuttingDown = Event()
# A thread in charge of managing all our child processes.
# Also takes care of resource accounting.
self.daddyThread = None
# If it breaks it will fill this in
self.daddyException: Optional[Exception] = None
if self.debugWorker:
log.debug('Started batch system %s in worker debug mode.',
id(self))
else:
self.daddyThread = Thread(target=self.daddy, daemon=True)
self.daddyThread.start()
log.debug('Started batch system %s in normal mode.', id(self))
def daddy(self):
"""
Be the "daddy" thread.
Our job is to look at jobs from the input queue.
If a job fits in the available resources, we allocate resources for it
and kick off a child process.
We also check on our children.
When a child finishes, we reap it, release its resources, and put its
information in the output queue.
"""
try:
log.debug('Started daddy thread for batch system %s.', id(self))
while not self.shuttingDown.is_set():
# Main loop
while not self.shuttingDown.is_set():
# Try to start as many jobs as we can try to start
try:
# Grab something from the input queue if available.
args = self.inputQueue.get_nowait()
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
coreFractions = int(jobCores / self.minCores)
# Try to start the child
result = self._startChild(jobCommand, jobID,
coreFractions, jobMemory,
jobDisk, environment)
if result is None:
# We did not get the resources to run this job.
# Requeue last, so we can look at the next job.
# TODO: Have some kind of condition the job can wait on,
# but without threads (queues for jobs needing
# cores/memory/disk individually)?
self.inputQueue.put(args)
break
# Otherwise it's a PID if it succeeded, or False if it couldn't
# start. But we don't care either way here.
except Empty:
# Nothing to run. Stop looking in the queue.
break
# Now check on our children.
for done_pid in self._pollForDoneChildrenIn(self.children):
# A child has actually finished.
# Clean up after it.
self._handleChild(done_pid)
# Then loop again: start and collect more jobs.
# TODO: It would be good to be able to wait on a new job or a finished child, whichever comes first.
# For now we just sleep and loop.
time.sleep(0.01)
# When we get here, we are shutting down.
log.debug(
'Daddy thread cleaning up %d remaining children for batch system %s...',
len(self.children), id(self))
self._stop_and_wait(self.children.values())
log.debug(
'Daddy thread for batch system %s finishing because no children should now exist',
id(self))
# Then exit the thread.
return
except Exception as e:
log.critical(
'Unhandled exception in daddy thread for batch system %s: %s',
id(self), traceback.format_exc())
# Pass the exception back to the main thread so it can stop the next person who calls into us.
self.daddyException = e
raise
def _checkOnDaddy(self):
if self.daddyException is not None:
# The daddy thread broke and we cannot do our job
log.critical(
'Propagating unhandled exception in daddy thread to main thread'
)
exc = self.daddyException
self.daddyException = None
if isinstance(exc, Exception):
raise exc
else:
raise TypeError(
f'Daddy thread failed with non-exception: {exc}')
def _stop_now(self, popens: Sequence[subprocess.Popen]) -> List[int]:
"""
Stop the given child processes and all their children. Does not reap them.
Returns a list of PGIDs killed, where processes may exist that have not
yet received their kill signals.
"""
# We will potentially need to poll these PGIDs to ensure that all
# processes in them are gone.
pgids = []
for popen in popens:
# Kill all the children
if popen.returncode is None:
# Process is not known to be dead. Try and grab its group.
try:
pgid = os.getpgid(popen.pid)
except OSError:
# It just died. Assume the pgid was its PID.
pgid = popen.pid
else:
# It is dead. Try it's PID as a PGID and hope we didn't re-use it.
pgid = popen.pid
if pgid != os.getpgrp():
# The child process really is in its own group, and not ours.
# Kill the group, which hopefully hasn't been reused
log.debug(
'Send shutdown kill to process group %s known to batch system %s',
pgid, id(self))
try:
os.killpg(pgid, signal.SIGKILL)
pgids.append(pgid)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
else:
# Kill the subprocess again through popen in case it somehow
# never managed to make the group.
popen.kill()
return pgids
def _stop_and_wait(self, popens: Sequence[subprocess.Popen]) -> None:
"""
Stop the given child processes and all their children. Blocks until the
processes are gone.
"""
pgids = self._stop_now(popens)
for popen in popens:
# Wait on all the children
popen.wait()
log.debug(
'Process %s known to batch system %s is stopped; it returned %s',
popen.pid, id(self), popen.returncode)
for pgid in pgids:
try:
while True:
# Send a kill to the group again, to see if anything in it
# is still alive. Our first kill might not have been
# delivered yet.
os.killpg(pgid, signal.SIGKILL)
# If that worked it is still alive, so wait for the kernel
# to stop fooling around and kill it.
log.warning(
'Sent redundant shutdown kill to surviving process group %s known to batch system %s',
pgid, id(self))
time.sleep(0.1)
except ProcessLookupError:
# The group is actually gone now.
pass
except PermissionError:
# The group is not only gone but reused
pass
def _pollForDoneChildrenIn(self, pid_to_popen):
"""
See if any children represented in the given dict from PID to Popen
object have finished.
Return a collection of their PIDs.
Guarantees that each child's exit code will be gettable via wait() on
the child's Popen object (i.e. does not reap the child, unless via
Popen).
"""
# We keep our found PIDs in a set so we can work around waitid showing
# us the same one repeatedly.
ready = set()
# Find the waitid function
waitid = getattr(os, 'waitid', None)
if callable(waitid):
# waitid exists (not Mac)
while True:
# Poll for any child to have exit, but don't reap it. Leave reaping
# to the Popen.
# TODO: What if someone else in Toil wants to do this syscall?
# TODO: Is this one-notification-per-done-child with WNOHANG? Or
# can we miss some? Or do we see the same one repeatedly until it
# is reaped?
try:
siginfo = waitid(os.P_ALL, -1,
os.WEXITED | os.WNOWAIT | os.WNOHANG)
except ChildProcessError:
# This happens when there is nothing to wait on right now,
# instead of the weird C behavior of overwriting a field in
# a pointed-to struct.
siginfo = None
if siginfo is not None and siginfo.si_pid in pid_to_popen and siginfo.si_pid not in ready:
# Something new finished
ready.add(siginfo.si_pid)
else:
# Nothing we own that we haven't seen before has finished.
return ready
else:
# On Mac there's no waitid and no way to wait and not reap.
# Fall back on polling all the Popen objects.
# To make this vaguely efficient we have to return done children in
# batches.
for pid, popen in pid_to_popen.items():
if popen.poll() is not None:
# Process is done
ready.add(pid)
log.debug('Child %d has stopped', pid)
# Return all the done processes we found
return ready
def _runDebugJob(self, jobCommand, jobID, environment):
"""
Run the jobCommand right now, in the current thread.
May only be called in debug-worker mode.
Assumes resources are available.
"""
assert self.debugWorker
# TODO: It is not possible to kill running jobs in forkless mode,
# because they are run immediately in the main thread.
info = Info(time.time(), None, None, killIntended=False)
self.runningJobs[jobID] = info
if jobCommand.startswith("_toil_worker "):
# We can actually run in this thread
jobName, jobStoreLocator, jobStoreID = jobCommand.split()[
1:4] # Parse command
jobStore = Toil.resumeJobStore(jobStoreLocator)
toil_worker.workerScript(
jobStore,
jobStore.config,
jobName,
jobStoreID,
redirectOutputToLogFile=not self.debugWorker
) # Call the worker
else:
# Run synchronously. If starting or running the command fails, let the exception stop us.
subprocess.check_call(jobCommand,
shell=True,
env=dict(os.environ, **environment))
self.runningJobs.pop(jobID)
if not info.killIntended:
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=0,
wallTime=time.time() - info.time,
exitReason=None))
def getSchedulingStatusMessage(self):
# Implement the abstractBatchSystem's scheduling status message API
return self.schedulingStatusMessage
def _setSchedulingStatusMessage(self, message):
"""
If we can't run a job, we record a short message about why not. If the
leader wants to know what is up with us (for example, to diagnose a
deadlock), it can ask us for the message.
"""
self.schedulingStatusMessage = message
def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk,
environment):
"""
Start a child process for the given job.
Allocate its required resources and save it and save it in our bookkeeping structures.
If the job is started, returns its PID.
If the job fails to start, reports it as failed and returns False.
If the job cannot get the resources it needs to start, returns None.
"""
# We fill this in if we manage to actually start the child.
popen = None
# This is when we started working on the job.
startTime = time.time()
# See if we can fit the job in our resource pools right now.
if self.coreFractions.acquireNow(coreFractions):
# We got some cores
if self.memory.acquireNow(jobMemory):
# We got some memory
if self.disk.acquireNow(jobDisk):
# We got the final resource, disk.
# Actually run the job.
# When it finishes we will release what it was using.
# So it is important to not lose track of the child process.
try:
# Launch the job.
# Make sure it is in its own session (and thus its own
# process group) so that, if the user signals the
# workflow, Toil will be responsible for killing the
# job. This also makes sure that we can signal the job
# and all its children together. We assume that the
# process group ID will equal the PID of the process we
# are starting.
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(
os.environ,
**environment),
start_new_session=True)
except Exception:
# If the job can't start, make sure we release resources now
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.error('Could not start job %s: %s', jobID,
traceback.format_exc())
# Report as failed.
self.outputQueue.put(
UpdatedBatchJobInfo(
jobID=jobID,
exitStatus=EXIT_STATUS_UNAVAILABLE_VALUE,
wallTime=0,
exitReason=None))
# Free resources
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
# Complain it broke.
return False
else:
# If the job did start, record it
self.children[popen.pid] = popen
# Make sure we can look it up by PID later
self.childToJob[popen.pid] = jobID
# Record that the job is running, and the resources it is using
info = Info(startTime,
popen, (coreFractions, jobMemory, jobDisk),
killIntended=False)
self.runningJobs[jobID] = info
log.debug('Launched job %s as child %d', jobID,
popen.pid)
# Report success starting the job
# Note that if a PID were somehow 0 it would look like False
assert popen.pid != 0
return popen.pid
else:
# We can't get disk, so free cores and memory
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self._setSchedulingStatusMessage(
'Not enough disk to run job %s' % jobID)
else:
# Free cores, since we can't get memory
self.coreFractions.release(coreFractions)
self._setSchedulingStatusMessage(
'Not enough memory to run job %s' % jobID)
else:
self._setSchedulingStatusMessage('Not enough cores to run job %s' %
jobID)
# If we get here, we didn't succeed or fail starting the job.
# We didn't manage to get the resources.
# Report that.
return None
def _handleChild(self, pid: int) -> None:
"""
Handle a child process PID that has finished.
The PID must be for a child job we started.
Not thread safe to run at the same time as we are making more children.
Remove the child from our bookkeeping structures and free its resources.
"""
# Look up the child
popen = self.children[pid]
jobID = self.childToJob[pid]
info = self.runningJobs[jobID]
# Unpack the job resources
(coreFractions, jobMemory, jobDisk) = info.resources
# Clean up our records of the job.
self.runningJobs.pop(jobID)
self.childToJob.pop(pid)
self.children.pop(pid)
if popen.returncode is None or not callable(getattr(
os, 'waitid', None)):
# It isn't reaped yet, or we have to reap all children to see if thay're done.
# Before we reap it (if possible), kill its PID as a PGID to make sure
# it isn't leaving children behind.
# TODO: This is a PGID re-use risk on Mac because the process is
# reaped already and the PGID may have been reused.
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
# See how the child did, and reap it.
statusCode = popen.wait()
if statusCode != 0 and not info.killIntended:
log.error("Got exit code %i (indicating failure) "
"from job %s.", statusCode, self.jobs[jobID])
if not info.killIntended:
# Report if the job failed and we didn't kill it.
# If we killed it then it shouldn't show up in the queue.
self.outputQueue.put(
UpdatedBatchJobInfo(jobID=jobID,
exitStatus=statusCode,
wallTime=time.time() - info.time,
exitReason=None))
# Make absolutely sure all processes in the group have received their
# kill signals and been cleaned up.
# TODO: this opens a PGID reuse risk; we reaped the process and its
# PGID may have been re-used. But it probably hasn't been and we
# definitely want to make sure all its children died before saying the
# job is done. Some might not be dead yet if we don't do this.
# TODO: can we safely do this before reaping? Or would we sit forever
# signaling a dead but unreaped process?
try:
while True:
# Send a kill to the group again, to see if anything in it
# is still alive. Our first kill might not have been
# delivered yet.
os.killpg(pid, signal.SIGKILL)
# If that worked it is still alive, so wait for the kernel
# to stop fooling around and kill it.
log.warning(
'Sent redundant job completion kill to surviving process group %s known to batch system %s',
pid, id(self))
time.sleep(0.1)
except ProcessLookupError:
# It is dead already
pass
except PermissionError:
# It isn't ours actually. Ours is dead.
pass
# Free up the job's resources.
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.debug('Child %d for job %s succeeded', pid, jobID)
def issueBatchJob(self, jobDesc):
"""Adds the command and resources to a queue to be run."""
self._checkOnDaddy()
# Round cores to minCores and apply scale.
# Make sure to give minCores even if asked for 0 cores, or negative or something.
cores = max(
math.ceil(jobDesc.cores * self.scale / self.minCores) *
self.minCores, self.minCores)
# Don't do our own assertions about job size vs. our configured size.
# The abstract batch system can handle it.
self.checkResourceRequest(jobDesc.memory,
cores,
jobDesc.disk,
job_name=jobDesc.jobName,
detail=f'Scale is set to {self.scale}.')
log.debug(
f"Issuing the command: {jobDesc.command} with "
f"memory: {jobDesc.memory}, cores: {cores}, disk: {jobDesc.disk}")
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobDesc.command
if self.debugWorker:
# Run immediately, blocking for return.
# Ignore resource requirements; we run one job at a time
self._runDebugJob(jobDesc.command, jobID, self.environment.copy())
else:
# Queue the job for later
self.inputQueue.put(
(jobDesc.command, jobID, cores, jobDesc.memory, jobDesc.disk,
self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs: Sequence[str]) -> None:
"""Kills jobs by ID."""
self._checkOnDaddy()
log.debug('Killing jobs: {}'.format(jobIDs))
# Collect the popen handles for the jobs we have to stop
popens: List[subprocess.Popen] = []
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
if info.popen is not None:
popens.append(info.popen)
else:
# No popen if running in forkless mode currently
assert self.debugWorker
log.critical("Can't kill job: %s in debug mode" % jobID)
# Stop them all in a batch. Don't reap, because we need the daddy
# thread to reap them to mark the jobs as not running anymore.
self._stop_now(popens)
for jobID in jobIDs:
while jobID in self.runningJobs:
# Wait for the daddy thread to collect them.
time.sleep(0.01)
def getIssuedBatchJobIDs(self):
"""Just returns all the jobs that have been run, but not yet returned as updated."""
self._checkOnDaddy()
return list(self.jobs.keys())
def getRunningBatchJobIDs(self):
self._checkOnDaddy()
now = time.time()
return {
jobID: now - info.time
for jobID, info in list(self.runningJobs.items())
}
def shutdown(self):
"""
Cleanly terminate and join daddy thread.
"""
if self.daddyThread is not None:
# Tell the daddy thread to stop.
self.shuttingDown.set()
# Wait for it to stop.
self.daddyThread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""Returns a tuple of a no-longer-running job, the return value of its process, and its runtime, or None."""
self._checkOnDaddy()
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
self.jobs.pop(item.jobID)
log.debug("Ran jobID: %s with exit value: %i", item.jobID,
item.exitStatus)
return item
@classmethod
def setOptions(cls, setOption):
setOption("scale", default=1)
def testNestedResourcesDoNotBlock(self):
"""
Resources are requested in the order Memory > Cpu > Disk.
Test that inavailability of cpus for one job that is scheduled does not block another job
that can run.
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
options.maxCores = 4
from toil import physicalMemory
availableMemory = physicalMemory()
options.batchSystem = self.batchSystemName
outFile = os.path.join(tempDir, 'counter')
open(outFile, 'w').close()
root = Job()
blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
cores=2, memory='1M', disk='1M')
firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
cores=1, memory='1M', disk='1M')
secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
writeVal='sJ', cores=1, memory='1M', disk='1M')
# Should block off 50% of memory while waiting for it's 3 cores
firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
writeVal='fJC', cores=3, memory=int(availableMemory/2), disk='1M')
# These two shouldn't be able to run before B because there should be only
# (50% of memory - 1M) available (firstJobChild should be blocking 50%)
secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJC', cores=2, memory=int(availableMemory/1.5),
disk='1M')
secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJGC', cores=2, memory=int(availableMemory/1.5),
disk='1M')
root.addChild(blocker)
root.addChild(firstJob)
root.addChild(secondJob)
firstJob.addChild(firstJobChild)
secondJob.addChild(secondJobChild)
secondJobChild.addChild(secondJobGrandChild)
"""
The tree is:
root
/ | \
b fJ sJ
| |
fJC sJC
|
sJGC
But the order of execution should be
root > b , fJ, sJ > sJC > sJGC > fJC
since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the
resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC
should not block them, and should only run after they finish.
"""
Job.Runner.startToil(root, options)
with open(outFile) as oFH:
outString = oFH.read()
# The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
# time. We look for all possible permutations.
possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])])
assert outString.startswith(possibleStarts)
assert outString.endswith('sJCsJGCfJC')
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you give it as they
come in, but in parallel.
"""
@classmethod
def supportsAutoDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = multiprocessing.cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value. One important invariant of this class is that each worker thread represents a CPU
requirement of minCores, meaning that we can never run more than numCores / minCores jobs
concurrently.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
if maxCores > self.numCores:
log.warn('Limiting maxCores to CPU count of system (%i).', self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
log.warn('Limiting maxMemory to physically available memory (%i).', self.physicalMemory)
maxMemory = self.physicalMemory
self.physicalDisk = toil.physicalDisk(config)
if maxDisk > self.physicalDisk:
log.warn('Limiting maxDisk to physically available disk (%i).', self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
# Number of worker threads that will be started
self.debugWorker = config.debugWorker
self.numWorkers = int(old_div(self.maxCores, self.minCores))
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs = {}
"""
:type: dict[str,toil.job.JobNode]
"""
# A queue of jobs waiting to be executed. Consumed by the workers.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the workers.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs = {}
"""
:type: dict[str,Info]
"""
# The list of worker threads
self.workerThreads = []
"""
:type list[Thread]
"""
# Variables involved with non-blocking resource acquisition
self.acquisitionTimeout = 5
self.acquisitionRetryDelay = 10
self.aquisitionCondition = Condition()
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(self.numWorkers, 'cores', self.acquisitionTimeout)
# A lock to work around the lack of thread-safety in Python's subprocess module
self.popenLock = Lock()
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory', self.acquisitionTimeout)
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)
if not self.debugWorker:
log.debug('Setting up the thread pool with %i workers, '
'given a minimum CPU fraction of %f '
'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
for i in range(self.numWorkers):
worker = Thread(target=self.worker, args=(self.inputQueue,))
self.workerThreads.append(worker)
worker.start()
else:
log.debug('Started in worker debug mode.')
def _runWorker(self, jobCommand, jobID, environment):
"""
Run the jobCommand using the worker and wait for it to finish.
The worker is forked unless it is a '_toil_worker' job and
debugWorker is True.
"""
startTime = time.time() # Time job is started
if self.debugWorker and "_toil_worker" in jobCommand:
# Run the worker without forking
jobName, jobStoreLocator, jobStoreID = jobCommand.split()[1:] # Parse command
jobStore = Toil.resumeJobStore(jobStoreLocator)
# TODO: The following does not yet properly populate self.runningJobs so it is not possible to kill
# running jobs in forkless mode - see the "None" value in place of popen
info = Info(time.time(), None, killIntended=False)
try:
self.runningJobs[jobID] = info
try:
toil_worker.workerScript(jobStore, jobStore.config, jobName, jobStoreID,
redirectOutputToLogFile=not self.debugWorker) # Call the worker
finally:
self.runningJobs.pop(jobID)
finally:
if not info.killIntended:
self.outputQueue.put((jobID, 0, time.time() - startTime))
else:
with self.popenLock:
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(os.environ, **environment))
info = Info(time.time(), popen, killIntended=False)
try:
self.runningJobs[jobID] = info
try:
statusCode = popen.wait()
if statusCode != 0 and not info.killIntended:
log.error("Got exit code %i (indicating failure) "
"from job %s.", statusCode, self.jobs[jobID])
finally:
self.runningJobs.pop(jobID)
finally:
if not info.killIntended:
self.outputQueue.put((jobID, statusCode, time.time() - startTime))
# Note: The input queue is passed as an argument because the corresponding attribute is reset
# to None in shutdown()
def worker(self, inputQueue):
while True:
if self.debugWorker and inputQueue.empty():
return
args = inputQueue.get()
if args is None:
break
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
while True:
try:
coreFractions = int(old_div(jobCores, self.minCores))
log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory,
self.memory)
with self.memory.acquisitionOf(jobMemory):
log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
'request of %f cores', coreFractions, self.coreFractions,
jobCores)
with self.coreFractions.acquisitionOf(coreFractions):
with self.disk.acquisitionOf(jobDisk):
self._runWorker(jobCommand, jobID, environment)
except ResourcePool.AcquisitionTimeoutException as e:
log.debug('Could not acquire enough (%s) to run job (%s). Requested: (%s), '
'Avaliable: %s. Sleeping for 10s.', e.resource, jobID, e.requested,
e.available)
with self.aquisitionCondition:
# Make threads sleep for the given delay, or until another job finishes.
# Whichever is sooner.
self.aquisitionCondition.wait(timeout=self.acquisitionRetryDelay)
continue
else:
log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
self.coreFractions.value, self.memory.value)
with self.aquisitionCondition:
# Wake up sleeping threads
self.aquisitionCondition.notifyAll()
break
def issueBatchJob(self, jobNode):
"""
Adds the command and resources to a queue to be run.
"""
# Round cores to minCores and apply scale
cores = math.ceil(jobNode.cores * self.scale / self.minCores) * self.minCores
assert cores <= self.maxCores, ('The job {} is requesting {} cores, more than the maximum of '
'{} cores this batch system was configured with. Scale is '
'set to {}.'.format(jobNode.jobName, cores, self.maxCores, self.scale))
assert cores >= self.minCores
assert jobNode.memory <= self.maxMemory, ('The job {} is requesting {} bytes of memory, more than '
'the maximum of {} this batch system was configured '
'with.'.format(jobNode.jobName, jobNode.memory, self.maxMemory))
self.checkResourceRequest(jobNode.memory, cores, jobNode.disk)
log.debug("Issuing the command: %s with memory: %i, cores: %i, disk: %i" % (
jobNode.command, jobNode.memory, cores, jobNode.disk))
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobNode.command
self.inputQueue.put((jobNode.command, jobID, cores, jobNode.memory,
jobNode.disk, self.environment.copy()))
if self.debugWorker: # then run immediately, blocking for return
self.worker(self.inputQueue)
return jobID
def killBatchJobs(self, jobIDs):
"""
Kills jobs by ID
"""
log.debug('Killing jobs: {}'.format(jobIDs))
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
if info.popen != None:
os.kill(info.popen.pid, 9)
else:
# No popen if running in forkless mode currently
assert self.debugWorker
log.critical("Can't kill job: %s in debug mode" % jobID)
while jobID in self.runningJobs:
pass
def getIssuedBatchJobIDs(self):
"""
Just returns all the jobs that have been run, but not yet returned as updated.
"""
return list(self.jobs.keys())
def getRunningBatchJobIDs(self):
now = time.time()
return {jobID: now - info.time for jobID, info in list(self.runningJobs.items())}
def shutdown(self):
"""
Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join
all worker threads.
"""
# Remove reference to inputQueue (raises exception if inputQueue is used after method call)
inputQueue = self.inputQueue
self.inputQueue = None
for i in range(self.numWorkers):
inputQueue.put(None)
for thread in self.workerThreads:
thread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""
Returns a map of the run jobs and the return value of their processes.
"""
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
jobID, exitValue, wallTime = item
jobCommand = self.jobs.pop(jobID)
log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
return jobID, exitValue, wallTime
@classmethod
def setOptions(cls, setOption):
setOption("scale", default=1)
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you give it as they
come in, but in parallel.
"""
@classmethod
def supportsHotDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = multiprocessing.cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value. One important invariant of this class is that each worker thread represents a CPU
requirement of minCores, meaning that we can never run more than numCores / minCores jobs
concurrently.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
if maxCores > self.numCores:
log.warn('Limiting maxCores to CPU count of system (%i).',
self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
log.warn('Limiting maxMemory to physically available memory (%i).',
self.physicalMemory)
maxMemory = self.physicalMemory
self.physicalDisk = toil.physicalDisk(config)
if maxDisk > self.physicalDisk:
log.warn('Limiting maxDisk to physically available disk (%i).',
self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores,
maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
# Number of worker threads that will be started
self.numWorkers = int(self.maxCores / self.minCores)
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs = {}
"""
:type: dict[str,toil.job.JobNode]
"""
# A queue of jobs waiting to be executed. Consumed by the workers.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the workers.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs = {}
"""
:type: dict[str,Info]
"""
# The list of worker threads
self.workerThreads = []
"""
:type list[Thread]
"""
# Variables involved with non-blocking resource acquisition
self.acquisitionTimeout = 5
self.acquisitionRetryDelay = 10
self.aquisitionCondition = Condition()
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(self.numWorkers, 'cores',
self.acquisitionTimeout)
# A lock to work around the lack of thread-safety in Python's subprocess module
self.popenLock = Lock()
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory',
self.acquisitionTimeout)
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)
log.debug(
'Setting up the thread pool with %i workers, '
'given a minimum CPU fraction of %f '
'and a maximum CPU value of %i.', self.numWorkers, self.minCores,
maxCores)
for i in xrange(self.numWorkers):
worker = Thread(target=self.worker, args=(self.inputQueue, ))
self.workerThreads.append(worker)
worker.start()
def _getDebugCmd(self, jobCommand):
"""Calculate useful debugging command line, handling CWL runs.
Tries to print out underlying CWL base command being run if possible,
otherwise defaulting to the toil jobCommand.
"""
debug_cmd = jobCommand
cmd_args = jobCommand.split()
if len(cmd_args) == 3 and cmd_args[0] == "_toil_worker":
_, job_store_locator, job_store_id = cmd_args
if job_store_locator.startswith("file:") or os.path.exists(
job_store_locator):
import cPickle
import marshal as pickler
job_store_locator = job_store_locator.replace("file:", "")
job_cmd_file = os.path.join(job_store_locator, "tmp",
job_store_id, "job")
with open(job_cmd_file) as in_handle:
job = pickler.load(in_handle)
if job.get("command"):
command_parts = job["command"].split()
input_pickle_file = os.path.join(job_store_locator, "tmp",
command_parts[1])
with open(input_pickle_file) as in_handle:
input = cPickle.load(in_handle)
if (hasattr(input, "cwltool")
and hasattr(input.cwltool, "tool")
and "baseCommand" in input.cwltool.tool):
debug_cmd = " ".join(input.cwltool.tool["baseCommand"])
return debug_cmd
# Note: The input queue is passed as an argument because the corresponding attribute is reset
# to None in shutdown()
def worker(self, inputQueue):
while True:
args = inputQueue.get()
if args is None:
log.debug('Received queue sentinel.')
break
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
while True:
try:
coreFractions = int(jobCores / self.minCores)
log.debug(
'Acquiring %i bytes of memory from a pool of %s.',
jobMemory, self.memory)
with self.memory.acquisitionOf(jobMemory):
log.debug(
'Acquiring %i fractional cores from a pool of %s to satisfy a '
'request of %f cores', coreFractions,
self.coreFractions, jobCores)
with self.coreFractions.acquisitionOf(coreFractions):
with self.disk.acquisitionOf(jobDisk):
log.debug("Executing command: '%s'.",
self._getDebugCmd(jobCommand))
startTime = time.time() #Time job is started
with self.popenLock:
popen = subprocess.Popen(
jobCommand,
shell=True,
env=dict(os.environ, **environment))
statusCode = None
info = Info(time.time(),
popen,
killIntended=False)
try:
self.runningJobs[jobID] = info
try:
statusCode = popen.wait()
if 0 != statusCode:
if statusCode != -9 or not info.killIntended:
log.error(
"Got exit code %i (indicating failure) "
"from job %s.", statusCode,
self.jobs[jobID])
finally:
self.runningJobs.pop(jobID)
finally:
if statusCode is not None and not info.killIntended:
self.outputQueue.put(
(jobID, statusCode,
time.time() - startTime))
except ResourcePool.AcquisitionTimeoutException as e:
log.debug(
'Could not acquire enough (%s) to run job. Requested: (%s), '
'Avaliable: %s. Sleeping for 10s.', e.resource,
e.requested, e.available)
with self.aquisitionCondition:
# Make threads sleep for the given delay, or until another job finishes.
# Whichever is sooner.
self.aquisitionCondition.wait(
timeout=self.acquisitionRetryDelay)
continue
else:
log.debug(
'Finished job. self.coreFractions ~ %s and self.memory ~ %s',
self.coreFractions.value, self.memory.value)
with self.aquisitionCondition:
# Wake up sleeping threads
self.aquisitionCondition.notifyAll()
break
log.debug('Exiting worker thread normally.')
def issueBatchJob(self, jobNode):
"""
Adds the command and resources to a queue to be run.
"""
# Round cores to minCores and apply scale
cores = math.ceil(
jobNode.cores * self.scale / self.minCores) * self.minCores
assert cores <= self.maxCores, (
'The job is requesting {} cores, more than the maximum of '
'{} cores this batch system was configured with. Scale is '
'set to {}.'.format(cores, self.maxCores, self.scale))
assert cores >= self.minCores
assert jobNode.memory <= self.maxMemory, (
'The job is requesting {} bytes of memory, more than '
'the maximum of {} this batch system was configured '
'with.'.format(jobNode.memory, self.maxMemory))
self.checkResourceRequest(jobNode.memory, cores, jobNode.disk)
log.debug(
"Issuing the command: %s with memory: %i, cores: %i, disk: %i" %
(jobNode.command, jobNode.memory, cores, jobNode.disk))
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobNode.command
self.inputQueue.put(
(jobNode.command, jobID, cores, jobNode.memory, jobNode.disk,
self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs):
"""
Kills jobs by ID
"""
log.debug('Killing jobs: {}'.format(jobIDs))
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
os.kill(info.popen.pid, 9)
while jobID in self.runningJobs:
pass
def getIssuedBatchJobIDs(self):
"""
Just returns all the jobs that have been run, but not yet returned as updated.
"""
return self.jobs.keys()
def getRunningBatchJobIDs(self):
now = time.time()
return {
jobID: now - info.time
for jobID, info in self.runningJobs.iteritems()
}
def shutdown(self):
"""
Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join
all worker threads.
"""
# Remove reference to inputQueue (raises exception if inputQueue is used after method call)
inputQueue = self.inputQueue
self.inputQueue = None
for i in xrange(self.numWorkers):
inputQueue.put(None)
for thread in self.workerThreads:
thread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""
Returns a map of the run jobs and the return value of their processes.
"""
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
jobID, exitValue, wallTime = item
jobCommand = self.jobs.pop(jobID)
log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
return jobID, exitValue, wallTime
@classmethod
def getRescueBatchJobFrequency(cls):
"""
This should not really occur, wihtout an error. To exercise the system we allow it every 90 minutes.
"""
return 5400
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you
give it as they come in, but in parallel.
Uses a single "daddy" thread to manage a fleet of child processes.
Communication with the daddy thread happens via two queues: one queue of
jobs waiting to be run (the input queue), and one queue of jobs that are
finished/stopped and need to be returned by getUpdatedBatchJob (the output
queue).
When the batch system is shut down, the daddy thread is stopped.
If running in debug-worker mode, jobs are run immediately as they are sent
to the batch system, in the sending thread, and the daddy thread is not
run. But the queues are still used.
"""
@classmethod
def supportsAutoDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
if maxCores > self.numCores:
log.warning('Limiting maxCores to CPU count of system (%i).',
self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
log.warning(
'Limiting maxMemory to physically available memory (%i).',
self.physicalMemory)
maxMemory = self.physicalMemory
self.physicalDisk = toil.physicalDisk(config)
if maxDisk > self.physicalDisk:
log.warning('Limiting maxDisk to physically available disk (%i).',
self.physicalDisk)
maxDisk = self.physicalDisk
super(SingleMachineBatchSystem, self).__init__(config, maxCores,
maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
if config.badWorker > 0 and config.debugWorker:
# We can't throw SIGUSR1 at the worker because it is also going to
# be the leader and/or test harness.
raise RuntimeError(
"Cannot use badWorker and debugWorker together; "
"worker would have to kill the leader")
self.debugWorker = config.debugWorker
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs = {}
"""
:type: dict[str,toil.job.JobNode]
"""
# A queue of jobs waiting to be executed. Consumed by the daddy thread.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the daddy thread.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs = {}
"""
:type: dict[str,Info]
"""
# These next two are only used outside debug-worker mode
# A dict mapping PIDs to Popen objects for running jobs.
# Jobs that don't fork are executed one at a time in the main thread.
self.children = {}
"""
:type: dict[int,subprocess.Popen]
"""
# A dict mapping child PIDs to the Job IDs they are supposed to be running.
self.childToJob = {}
"""
:type: dict[int,str]
"""
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(
int(old_div(self.maxCores, self.minCores)), 'cores')
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory, 'memory')
# A pool representing the available space in bytes
self.disk = ResourcePool(self.maxDisk, 'disk')
# We use this event to signal shutdown
self.shuttingDown = Event()
# A thread in charge of managing all our child processes.
# Also takes care of resource accounting.
self.daddyThread = None
# If it breaks it will fill this in
self.daddyException = None
if self.debugWorker:
log.debug('Started in worker debug mode.')
else:
self.daddyThread = Thread(target=self.daddy, daemon=True)
self.daddyThread.start()
log.debug('Started in normal mode.')
def daddy(self):
"""
Be the "daddy" thread.
Our job is to look at jobs from the input queue.
If a job fits in the available resources, we allocate resources for it
and kick off a child process.
We also check on our children.
When a child finishes, we reap it, release its resources, and put its
information in the output queue.
"""
try:
log.debug('Started daddy thread.')
while not self.shuttingDown.is_set():
# Main loop
while not self.shuttingDown.is_set():
# Try to start as many jobs as we can try to start
try:
# Grab something from the input queue if available.
args = self.inputQueue.get_nowait()
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
coreFractions = int(old_div(jobCores, self.minCores))
# Try to start the child
result = self._startChild(jobCommand, jobID,
coreFractions, jobMemory,
jobDisk, environment)
log.debug('Tried to start job %s and got: %s', jobID,
str(result))
if result is None:
# We did not get the resources to run this job.
# Requeue last, so we can look at the next job.
# TODO: Have some kind of condition the job can wait on,
# but without threads (queues for jobs needing
# cores/memory/disk individually)?
self.inputQueue.put(args)
break
# Otherwise it's a PID if it succeeded, or False if it couldn't
# start. But we don't care either way here.
except Empty:
# Nothing to run. Stop looking in the queue.
break
# Now check on our children.
for done_pid in self._pollForDoneChildrenIn(self.children):
# A child has actually finished.
# Clean up after it.
self._handleChild(done_pid)
# Then loop again: start and collect more jobs.
# TODO: It would be good to be able to wait on a new job or a finished child, whichever comes first.
# For now we just sleep and loop.
time.sleep(0.01)
# When we get here, we are shutting down.
for popen in self.children.values():
# Kill all the children, going through popen to avoid signaling re-used PIDs.
popen.kill()
for popen in self.children.values():
# Reap all the children
popen.wait()
# Then exit the thread.
return
except Exception as e:
log.critical('Unhandled exception in daddy thread: %s',
traceback.format_exc())
# Pass the exception back to the main thread so it can stop the next person who calls into us.
self.daddyException = e
raise
def _checkOnDaddy(self):
if self.daddyException is not None:
# The daddy thread broke and we cannot do our job
log.critical(
'Propagating unhandled exception in daddy thread to main thread'
)
exc = self.daddyException
self.daddyException = None
raise exc
def _pollForDoneChildrenIn(self, pid_to_popen):
"""
See if any children represented in the given dict from PID to Popen
object have finished.
Return a collection of their PIDs.
Guarantees that each child's exit code will be gettable via wait() on
the child's Popen object (i.e. does not reap the child, unless via
Popen).
"""
# We keep our found PIDs in a set so we can work around waitid showing
# us the same one repeatedly.
ready = set()
# Find the waitid function
waitid = getattr(os, 'waitid', None)
if callable(waitid):
# waitid exists (not Mac)
while True:
# Poll for any child to have exit, but don't reap it. Leave reaping
# to the Popen.
# TODO: What if someone else in Toil wants to do this syscall?
# TODO: Is this one-notification-per-done-child with WNOHANG? Or
# can we miss some? Or do we see the same one repeatedly until it
# is reaped?
try:
siginfo = waitid(os.P_ALL, -1,
os.WEXITED | os.WNOWAIT | os.WNOHANG)
except ChildProcessError:
# This happens when there is nothing to wait on right now,
# instead of the weird C behavior of overwriting a field in
# a pointed-to struct.
siginfo = None
if siginfo is not None and siginfo.si_pid in pid_to_popen and siginfo.si_pid not in ready:
# Something new finished
ready.add(siginfo.si_pid)
else:
# Nothing we own that we haven't seen before has finished.
return ready
else:
# On Mac there's no waitid and no way to wait and not reap.
# Fall back on polling all the Popen objects.
# To make this vaguely efficient we have to return done children in
# batches.
for pid, popen in pid_to_popen.items():
if popen.poll() is not None:
# Process is done
ready.add(pid)
log.debug('Child %d has stopped', pid)
# Return all the done processes we found
return ready
def _runDebugJob(self, jobCommand, jobID, environment):
"""
Run the jobCommand right now, in the current thread.
May only be called in debug-worker mode.
Assumes resources are available.
"""
assert self.debugWorker
# TODO: It is not possible to kill running jobs in forkless mode,
# because they are run immediately in the main thread.
info = Info(time.time(), None, None, killIntended=False)
self.runningJobs[jobID] = info
if jobCommand.startswith("_toil_worker "):
# We can actually run in this thread
jobName, jobStoreLocator, jobStoreID = jobCommand.split()[
1:] # Parse command
jobStore = Toil.resumeJobStore(jobStoreLocator)
toil_worker.workerScript(
jobStore,
jobStore.config,
jobName,
jobStoreID,
redirectOutputToLogFile=not self.debugWorker
) # Call the worker
else:
# Run synchronously. If starting or running the command fails, let the exception stop us.
subprocess.check_call(jobCommand,
shell=True,
env=dict(os.environ, **environment))
self.runningJobs.pop(jobID)
if not info.killIntended:
self.outputQueue.put((jobID, 0, time.time() - info.time))
def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk,
environment):
"""
Start a child process for the given job.
Allocate its required resources and save it and save it in our bookkeeping structures.
If the job is started, returns its PID.
If the job fails to start, reports it as failed and returns False.
If the job cannot get the resources it needs to start, returns None.
"""
# We fill this in if we manage to actually start the child.
popen = None
# This is when we started working on the job.
startTime = time.time()
# See if we can fit the job in our resource pools right now.
if self.coreFractions.acquireNow(coreFractions):
# We got some cores
if self.memory.acquireNow(jobMemory):
# We got some memory
if self.disk.acquireNow(jobDisk):
# We got the final resource, disk.
# Actually run the job.
# When it finishes we will release what it was using.
# So it is important to not lose track of the child process.
try:
# Launch the job
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(
os.environ,
**environment))
except Exception:
# If the job can't start, make sure we release resources now
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.error('Could not start job %s: %s', jobID,
traceback.format_exc())
# Report as failed.
# TODO: what should the exit code be?
self.outputQueue.put((jobID, -1, 0))
# Free resources
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
# Complain it broke.
return False
else:
# If the job did start, record it
self.children[popen.pid] = popen
# Make sure we can look it up by PID later
self.childToJob[popen.pid] = jobID
# Record that the job is running, and the resources it is using
info = Info(startTime,
popen, (coreFractions, jobMemory, jobDisk),
killIntended=False)
self.runningJobs[jobID] = info
log.debug('Launched job %s as child %d', jobID,
popen.pid)
# Report success starting the job
# Note that if a PID were somehow 0 it would look like False
assert popen.pid != 0
return popen.pid
else:
# We can't get disk, so free cores and memory
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
log.debug('Not enough disk to run job %s', jobID)
else:
# Free cores, since we can't get memory
self.coreFractions.release(coreFractions)
log.debug('Not enough memory to run job %s', jobID)
else:
log.debug('Not enough cores to run job %s', jobID)
# If we get here, we didn't succeed or fail starting the job.
# We didn't manage to get the resources.
# Report that.
return None
def _handleChild(self, pid):
"""
Handle a child process PID that has finished.
The PID must be for a child job we started.
Not thread safe to run at the same time as we are making more children.
Remove the child from our bookkeeping structures and free its resources.
"""
# Look up the child
popen = self.children[pid]
jobID = self.childToJob[pid]
info = self.runningJobs[jobID]
# Unpack the job resources
(coreFractions, jobMemory, jobDisk) = info.resources
# Clean up our records of the job.
self.runningJobs.pop(jobID)
self.childToJob.pop(pid)
self.children.pop(pid)
# See how the child did, and reap it.
statusCode = popen.wait()
if statusCode != 0 and not info.killIntended:
log.error("Got exit code %i (indicating failure) "
"from job %s.", statusCode, self.jobs[jobID])
if not info.killIntended:
# Report if the job failed and we didn't kill it.
# If we killed it then it shouldn't show up in the queue.
self.outputQueue.put((jobID, statusCode, time.time() - info.time))
# Free up the job's resources.
self.coreFractions.release(coreFractions)
self.memory.release(jobMemory)
self.disk.release(jobDisk)
log.debug('Child %d for job %s succeeded', pid, jobID)
def issueBatchJob(self, jobNode):
"""Adds the command and resources to a queue to be run."""
self._checkOnDaddy()
# Round cores to minCores and apply scale
cores = math.ceil(
jobNode.cores * self.scale / self.minCores) * self.minCores
assert cores <= self.maxCores, (
'The job {} is requesting {} cores, more than the maximum of '
'{} cores this batch system was configured with. Scale is '
'set to {}.'.format(jobNode.jobName, cores, self.maxCores,
self.scale))
assert cores >= self.minCores
assert jobNode.memory <= self.maxMemory, (
'The job {} is requesting {} bytes of memory, more than '
'the maximum of {} this batch system was configured '
'with.'.format(jobNode.jobName, jobNode.memory, self.maxMemory))
self.checkResourceRequest(jobNode.memory, cores, jobNode.disk)
log.debug(
"Issuing the command: %s with memory: %i, cores: %i, disk: %i" %
(jobNode.command, jobNode.memory, cores, jobNode.disk))
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = jobNode.command
if self.debugWorker:
# Run immediately, blocking for return.
# Ignore resource requirements; we run one job at a time
self._runDebugJob(jobNode.command, jobID, self.environment.copy())
else:
# Queue the job for later
self.inputQueue.put(
(jobNode.command, jobID, cores, jobNode.memory, jobNode.disk,
self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs):
"""Kills jobs by ID."""
self._checkOnDaddy()
log.debug('Killing jobs: {}'.format(jobIDs))
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
if info.popen != None:
log.debug('Send kill to PID %s', info.popen.pid)
info.popen.kill()
log.debug('Sent kill to PID %s', info.popen.pid)
else:
# No popen if running in forkless mode currently
assert self.debugWorker
log.critical("Can't kill job: %s in debug mode" % jobID)
while jobID in self.runningJobs:
pass
def getIssuedBatchJobIDs(self):
"""Just returns all the jobs that have been run, but not yet returned as updated."""
self._checkOnDaddy()
return list(self.jobs.keys())
def getRunningBatchJobIDs(self):
self._checkOnDaddy()
now = time.time()
return {
jobID: now - info.time
for jobID, info in list(self.runningJobs.items())
}
def shutdown(self):
"""
Cleanly terminate and join daddy thread.
"""
if self.daddyThread is not None:
# Tell the daddy thread to stop.
self.shuttingDown.set()
# Wait for it to stop.
self.daddyThread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""Returns a tuple of a no-longer-running job, the return value of its process, and its runtime, or None."""
self._checkOnDaddy()
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
jobID, exitValue, wallTime = item
jobCommand = self.jobs.pop(jobID)
log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
return jobID, exitValue, wallTime
@classmethod
def setOptions(cls, setOption):
setOption("scale", default=1)
class SingleMachineBatchSystem(BatchSystemSupport):
"""
The interface for running jobs on a single machine, runs all the jobs you give it as they
come in, but in parallel.
"""
@classmethod
def supportsHotDeployment(cls):
return False
@classmethod
def supportsWorkerCleanup(cls):
return True
numCores = multiprocessing.cpu_count()
minCores = 0.1
"""
The minimal fractional CPU. Tasks with a smaller core requirement will be rounded up to this
value. One important invariant of this class is that each worker thread represents a CPU
requirement of minCores, meaning that we can never run more than numCores / minCores jobs
concurrently.
"""
physicalMemory = toil.physicalMemory()
def __init__(self, config, maxCores, maxMemory, maxDisk):
if maxCores > self.numCores:
log.warn('Limiting maxCores to CPU count of system (%i).', self.numCores)
maxCores = self.numCores
if maxMemory > self.physicalMemory:
log.warn('Limiting maxMemory to physically available memory (%i).', self.physicalMemory)
maxMemory = self.physicalMemory
super(SingleMachineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
assert self.maxCores >= self.minCores
assert self.maxMemory >= 1
# The scale allows the user to apply a factor to each task's cores requirement, thereby
# squeezing more tasks onto each core (scale < 1) or stretching tasks over more cores
# (scale > 1).
self.scale = config.scale
# Number of worker threads that will be started
self.numWorkers = int(self.maxCores / self.minCores)
# A counter to generate job IDs and a lock to guard it
self.jobIndex = 0
self.jobIndexLock = Lock()
# A dictionary mapping IDs of submitted jobs to the command line
self.jobs = {}
"""
:type: dict[str,str]
"""
# A queue of jobs waiting to be executed. Consumed by the workers.
self.inputQueue = Queue()
# A queue of finished jobs. Produced by the workers.
self.outputQueue = Queue()
# A dictionary mapping IDs of currently running jobs to their Info objects
self.runningJobs = {}
"""
:type: dict[str,Info]
"""
# The list of worker threads
self.workerThreads = []
"""
:type list[Thread]
"""
# A pool representing available CPU in units of minCores
self.coreFractions = ResourcePool(self.numWorkers)
# A lock to work around the lack of thread-safety in Python's subprocess module
self.popenLock = Lock()
# A pool representing available memory in bytes
self.memory = ResourcePool(self.maxMemory)
log.info('Setting up the thread pool with %i workers, '
'given a minimum CPU fraction of %f '
'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
for i in xrange(self.numWorkers):
worker = Thread(target=self.worker, args=(self.inputQueue,))
self.workerThreads.append(worker)
worker.start()
# Note: The input queue is passed as an argument because the corresponding attribute is reset
# to None in shutdown()
def worker(self, inputQueue):
while True:
args = inputQueue.get()
if args is None:
log.debug('Received queue sentinel.')
break
jobCommand, jobID, jobCores, jobMemory, jobDisk, environment = args
try:
coreFractions = int(jobCores / self.minCores)
log.debug('Acquiring %i bytes of memory from a pool of %s.', jobMemory, self.memory)
with self.memory.acquisitionOf(jobMemory):
log.debug('Acquiring %i fractional cores from a pool of %s to satisfy a '
'request of %f cores', coreFractions, self.coreFractions, jobCores)
with self.coreFractions.acquisitionOf(coreFractions):
log.info("Executing command: '%s'.", jobCommand)
startTime = time.time() #Time job is started
with self.popenLock:
popen = subprocess.Popen(jobCommand,
shell=True,
env=dict(os.environ, **environment))
statusCode = None
info = Info(time.time(), popen, killIntended=False)
try:
self.runningJobs[jobID] = info
try:
statusCode = popen.wait()
if 0 != statusCode:
if statusCode != -9 or not info.killIntended:
log.error("Got exit code %i (indicating failure) from "
"command '%s'.", statusCode, jobCommand)
finally:
self.runningJobs.pop(jobID)
finally:
if statusCode is not None and not info.killIntended:
self.outputQueue.put((jobID, statusCode, time.time() - startTime))
finally:
log.debug('Finished job. self.coreFractions ~ %s and self.memory ~ %s',
self.coreFractions.value, self.memory.value)
log.debug('Exiting worker thread normally.')
def issueBatchJob(self, command, memory, cores, disk, preemptable):
"""
Adds the command and resources to a queue to be run.
"""
# Round cores to minCores and apply scale
cores = math.ceil(cores * self.scale / self.minCores) * self.minCores
assert cores <= self.maxCores, ('The job is requesting {} cores, more than the maximum of '
'{} cores this batch system was configured with. Scale is '
'set to {}.'.format(cores, self.maxCores, self.scale))
assert cores >= self.minCores
assert memory <= self.maxMemory, ('The job is requesting {} bytes of memory, more than '
'the maximum of {} this batch system was configured '
'with.'.format(memory, self.maxMemory))
self.checkResourceRequest(memory, cores, disk)
log.debug("Issuing the command: %s with memory: %i, cores: %i, disk: %i" % (
command, memory, cores, disk))
with self.jobIndexLock:
jobID = self.jobIndex
self.jobIndex += 1
self.jobs[jobID] = command
self.inputQueue.put((command, jobID, cores, memory, disk, self.environment.copy()))
return jobID
def killBatchJobs(self, jobIDs):
"""
Kills jobs by ID
"""
log.debug('Killing jobs: {}'.format(jobIDs))
for jobID in jobIDs:
if jobID in self.runningJobs:
info = self.runningJobs[jobID]
info.killIntended = True
os.kill(info.popen.pid, 9)
while jobID in self.runningJobs:
pass
def getIssuedBatchJobIDs(self):
"""
Just returns all the jobs that have been run, but not yet returned as updated.
"""
return self.jobs.keys()
def getRunningBatchJobIDs(self):
now = time.time()
return {jobID: now - info.time for jobID, info in self.runningJobs.iteritems()}
def shutdown(self):
"""
Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join
all worker threads.
"""
# Remove reference to inputQueue (raises exception if inputQueue is used after method call)
inputQueue = self.inputQueue
self.inputQueue = None
for i in xrange(self.numWorkers):
inputQueue.put(None)
for thread in self.workerThreads:
thread.join()
BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
def getUpdatedBatchJob(self, maxWait):
"""
Returns a map of the run jobs and the return value of their processes.
"""
try:
item = self.outputQueue.get(timeout=maxWait)
except Empty:
return None
jobID, exitValue, wallTime = item
self.jobs.pop(jobID)
log.debug("Ran jobID: %s with exit value: %i", jobID, exitValue)
return jobID, exitValue, wallTime
@classmethod
def getRescueBatchJobFrequency(cls):
"""
This should not really occur, wihtout an error. To exercise the system we allow it every 90 minutes.
"""
return 5400