def issueBatchJob(self, jobNode):
"""
Issues the following command returning a unique jobID. Command is the string to run, memory
is an int giving the number of bytes the job needs to run in and cores is the number of cpus
needed for the job and error-file is the path of the file to place any std-err/std-out in.
"""
localID = self.handleLocalJob(jobNode)
if localID:
return localID
self.checkResourceRequest(jobNode.memory, jobNode.cores, jobNode.disk)
jobID = self.getNextJobID()
job = ToilJob(jobID=jobID,
name=str(jobNode),
resources=MesosShape(wallTime=0, **jobNode._requirements),
command=jobNode.command,
userScript=self.userScript,
environment=self.environment.copy(),
workerCleanupInfo=self.workerCleanupInfo)
jobType = job.resources
log.debug("Queueing the job command: %s with job id: %s ...", jobNode.command, str(jobID))
# TODO: round all elements of resources
self.taskResources[jobID] = job.resources
self.jobQueues.insertJob(job, jobType)
log.debug("... queued")
return jobID
def issueBatchJob(self, command, memory, cpu, disk):
"""
Issues the following command returning a unique jobID. Command is the string to run, memory is an int giving
the number of bytes the batchjob needs to run in and cpu is the number of cpus needed for the batchjob and error-file
is the path of the file to place any std-err/std-out in.
"""
# puts batchjob into job_type_queue to be run by Mesos, AND puts jobID in current_job[]
self.checkResourceRequest(memory, cpu, disk)
jobID = self.nextJobID
self.nextJobID += 1
batchjob = ToilJob(jobID=jobID,
resources=ResourceRequirement(memory=memory,
cpu=cpu,
disk=disk),
command=command,
userScript=self.userScript,
toilDistribution=self.toilDistribution)
job_type = batchjob.resources
log.debug(
"Queueing the batchjob command: %s with batchjob id: %s ..." %
(command, str(jobID)))
self.jobQueueList[job_type].append(batchjob)
log.debug("... queued")
return jobID
def _getJob(self, cores=1, memory=1000, disk=5000, preemptable=True):
from toil.batchSystems.mesos import MesosShape
from toil.batchSystems.mesos import ToilJob
resources = MesosShape(wallTime=0, cores=cores, memory=memory, disk=disk, preemptable=preemptable)
job = ToilJob(jobID=str(uuid.uuid4()),
name=str(uuid.uuid4()),
resources=resources,
command="do nothing",
userScript=None,
environment=None,
workerCleanupInfo=None)
return job
def issueBatchJob(self,
jobNode: JobDescription,
job_environment: Optional[Dict[str, str]] = None):
"""
Issues the following command returning a unique jobID. Command is the string to run, memory
is an int giving the number of bytes the job needs to run in and cores is the number of cpus
needed for the job and error-file is the path of the file to place any std-err/std-out in.
"""
localID = self.handleLocalJob(jobNode)
if localID:
return localID
mesos_resources = {
"memory": jobNode.memory,
"cores": jobNode.cores,
"disk": jobNode.disk,
"preemptable": jobNode.preemptable
}
self.checkResourceRequest(memory=mesos_resources["memory"],
cores=mesos_resources["cores"],
disk=mesos_resources["disk"])
jobID = self.getNextJobID()
environment = self.environment.copy()
if job_environment:
environment.update(job_environment)
job = ToilJob(jobID=jobID,
name=str(jobNode),
resources=MesosShape(wallTime=0, **mesos_resources),
command=jobNode.command,
userScript=self.userScript,
environment=environment,
workerCleanupInfo=self.workerCleanupInfo)
jobType = job.resources
log.debug("Queueing the job command: %s with job id: %s ...",
jobNode.command, str(jobID))
# TODO: round all elements of resources
self.taskResources[jobID] = job.resources
self.jobQueues.insertJob(job, jobType)
log.debug("... queued")
return jobID
def issueBatchJob(self, jobNode):
"""
Issues the following command returning a unique jobID. Command is the string to run, memory
is an int giving the number of bytes the job needs to run in and cores is the number of cpus
needed for the job and error-file is the path of the file to place any std-err/std-out in.
"""
self.checkResourceRequest(jobNode.memory, jobNode.cores, jobNode.disk)
jobID = next(self.unusedJobID)
job = ToilJob(jobID=jobID,
resources=ResourceRequirement(**jobNode._requirements),
command=jobNode.command,
userScript=self.userScript,
environment=self.environment.copy(),
workerCleanupInfo=self.workerCleanupInfo)
jobType = job.resources
log.debug("Queueing the job command: %s with job id: %s ...",
jobNode.command, str(jobID))
self.jobQueues[jobType].append(job)
log.debug("... queued")
return jobID