def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
"""
Test the ClusterScaler class with different patterns of job creation. Tests ascertain that
autoscaling occurs and that all the jobs are run.
"""
# First do simple test of creating 100 preemptable and non-premptable jobs and check the
# jobs are completed okay, then print the amount of worker time expended and the total
# number of worker nodes used.
mock = MockBatchSystemAndProvisioner(config, secondsPerJob=2.0)
mock.start()
clusterScaler = ScalerThread(mock, mock, config)
clusterScaler.start()
try:
# Add 100 jobs to complete
list(map(lambda x: mock.addJob(jobShape=jobShape),
list(range(numJobs))))
list(map(lambda x: mock.addJob(jobShape=jobShape, preemptable=True),
list(range(numPreemptableJobs))))
# Add some completed jobs
for preemptable in (True, False):
if preemptable and numPreemptableJobs > 0 or not preemptable and numJobs > 0:
# Add 1000 random jobs
for _ in range(1000):
x = mock.getNodeShape(nodeType=jobShape)
iJ = JobNode(jobStoreID=1,
requirements=dict(
memory=random.choice(list(range(1, x.memory))),
cores=random.choice(list(range(1, x.cores))),
disk=random.choice(list(range(1, x.disk))),
preemptable=preemptable),
command=None,
jobName='testClusterScaling', unitName='')
clusterScaler.addCompletedJob(iJ, random.choice(list(range(1, x.wallTime))))
startTime = time.time()
# Wait while the cluster processes the jobs
while (mock.getNumberOfJobsIssued(preemptable=False) > 0
or mock.getNumberOfJobsIssued(preemptable=True) > 0
or mock.getNumberOfNodes() > 0 or mock.getNumberOfNodes(preemptable=True) > 0):
logger.debug("Running, non-preemptable queue size: %s, non-preemptable workers: %s, "
"preemptable queue size: %s, preemptable workers: %s" %
(mock.getNumberOfJobsIssued(preemptable=False),
mock.getNumberOfNodes(preemptable=False),
mock.getNumberOfJobsIssued(preemptable=True),
mock.getNumberOfNodes(preemptable=True)))
clusterScaler.check()
time.sleep(0.5)
logger.debug("We waited %s for cluster to finish" % (time.time() - startTime))
finally:
clusterScaler.shutdown()
mock.shutDown()
# Print some info about the autoscaling
logger.debug("Total-jobs: %s: Max-workers: %s, "
"Total-worker-time: %s, Worker-time-per-job: %s" %
(mock.totalJobs, sum(mock.maxWorkers.values()),
mock.totalWorkerTime,
old_div(mock.totalWorkerTime, mock.totalJobs) if mock.totalJobs > 0 else 0.0))
def __init__(self, config, batchSystem, provisioner, jobStore, rootJob, jobCache=None):
"""
:param toil.common.Config config:
:param toil.batchSystems.abstractBatchSystem.AbstractBatchSystem batchSystem:
:param toil.provisioners.abstractProvisioner.AbstractProvisioner provisioner
:param toil.jobStores.abstractJobStore.AbstractJobStore jobStore:
:param toil.jobGraph.JobGraph rootJob
If jobCache is passed, it must be a dict from job ID to pre-existing
JobGraph objects. Jobs will be loaded from the cache (which can be
downloaded from the jobStore in a batch) during the construction of the ToilState object.
"""
# Object containing parameters for the run
self.config = config
# The job store
self.jobStore = jobStore
self.jobStoreLocator = config.jobStore
# Get a snap shot of the current state of the jobs in the jobStore
self.toilState = ToilState(jobStore, rootJob, jobCache=jobCache)
logger.debug("Found %s jobs to start and %i jobs with successors to run",
len(self.toilState.updatedJobs), len(self.toilState.successorCounts))
# Batch system
self.batchSystem = batchSystem
assert len(self.batchSystem.getIssuedBatchJobIDs()) == 0 #Batch system must start with no active jobs!
logger.debug("Checked batch system has no running jobs and no updated jobs")
# Map of batch system IDs to IssuedJob tuples
self.jobBatchSystemIDToIssuedJob = {}
# Number of preempetable jobs currently being run by batch system
self.preemptableJobsIssued = 0
# Tracking the number service jobs issued,
# this is used limit the number of services issued to the batch system
self.serviceJobsIssued = 0
self.serviceJobsToBeIssued = [] # A queue of service jobs that await scheduling
#Equivalents for service jobs to be run on preemptable nodes
self.preemptableServiceJobsIssued = 0
self.preemptableServiceJobsToBeIssued = []
# Timing of the jobGraph rescuing method
self.timeSinceJobsLastRescued = None
# Hash to store number of times a job is lost by the batch system,
# used to decide if to reissue an apparently missing job
self.reissueMissingJobs_missingHash = {}
# Class used to create/destroy nodes in the cluster, may be None if
# using a statically defined cluster
self.provisioner = provisioner
# Create cluster scaling thread if the provisioner is not None
self.clusterScaler = None
if self.provisioner is not None and len(self.provisioner.nodeTypes) > 0:
self.clusterScaler = ScalerThread(self.provisioner, self, self.config)
# A service manager thread to start and terminate services
self.serviceManager = ServiceManager(jobStore, self.toilState)
# A thread to manage the aggregation of statistics and logging from the run
self.statsAndLogging = StatsAndLogging(self.jobStore, self.config)
# Set used to monitor deadlocked jobs
self.potentialDeadlockedJobs = set()
self.potentialDeadlockTime = 0
# A dashboard that runs on the leader node in AWS clusters to track the state
# of the cluster
self.toilMetrics = None
# internal jobs we should not expose at top level debugging
self.debugJobNames = ("CWLJob", "CWLWorkflow", "CWLScatter", "CWLGather",
"ResolveIndirect")
def testClusterScalingMultipleNodeTypes(self):
smallNode = Shape(20, 5, 10, 10, False)
mediumNode = Shape(20, 10, 10, 10, False)
largeNode = Shape(20, 20, 10, 10, False)
numJobs = 100
config = Config()
# Make defaults dummy values
config.defaultMemory = 1
config.defaultCores = 1
config.defaultDisk = 1
# No preemptable nodes/jobs
config.preemptableNodeTypes = []
config.minPreemptableNodes = []
config.maxPreemptableNodes = [] # No preemptable nodes
# Make sure the node types don't have to be ordered
config.nodeTypes = [largeNode, smallNode, mediumNode]
config.minNodes = [0, 0, 0]
config.maxNodes = [10, 10] # test expansion of this list
# Algorithm parameters
config.targetTime = defaultTargetTime
config.betaInertia = 0.1
config.scaleInterval = 3
mock = MockBatchSystemAndProvisioner(config, secondsPerJob=2.0)
clusterScaler = ScalerThread(mock, mock, config)
clusterScaler.start()
mock.start()
try:
# Add small jobs
list(
map(lambda x: mock.addJob(jobShape=smallNode),
list(range(numJobs))))
list(
map(lambda x: mock.addJob(jobShape=mediumNode),
list(range(numJobs))))
# Add medium completed jobs
for i in range(1000):
iJ = JobNode(jobStoreID=1,
requirements=dict(memory=random.choice(
range(smallNode.memory, mediumNode.memory)),
cores=mediumNode.cores,
disk=largeNode.cores,
preemptable=False),
command=None,
jobName='testClusterScaling',
unitName='')
clusterScaler.addCompletedJob(iJ, random.choice(range(1, 10)))
while mock.getNumberOfJobsIssued() > 0 or mock.getNumberOfNodes(
) > 0:
logger.debug("%i nodes currently provisioned" %
mock.getNumberOfNodes())
# Make sure there are no large nodes
self.assertEqual(mock.getNumberOfNodes(nodeType=largeNode), 0)
clusterScaler.check()
time.sleep(0.5)
finally:
clusterScaler.shutdown()
mock.shutDown()
# Make sure jobs ran on both the small and medium node types
self.assertTrue(mock.totalJobs > 0)
self.assertTrue(mock.maxWorkers[smallNode] > 0)
self.assertTrue(mock.maxWorkers[mediumNode] > 0)
self.assertEqual(mock.maxWorkers[largeNode], 0)