class TestSchedulerIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
# planner = None
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def testSchdulerCheckIngestReady(self):
"""
Check the return status of check_ingest_capacity is correct
"""
pipelines = self.telescope.pipelines
observation = self.telescope.observations[0]
max_ingest = self.telescope.max_ingest
# There should be capacity
self.assertEqual(0.0, self.env.now)
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertTrue(ret)
# Let's remove capacity to check it returns false
tmp = self.cluster._resources['available']
self.cluster._resources['available'] = self.cluster._resources[
'available'][:3]
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertFalse(ret)
self.cluster._resources['available'] = tmp
self.assertEqual(10, len(self.cluster._resources['available']))
def testSchedulerProvisionsIngest(self):
"""
Ensure that the scheduler correcly coordinates ingest onto the Cluster
and into the Buffer
Returns
-------
"""
pipelines = self.telescope.pipelines
max_ingest = self.telescope.max_ingest
observation = self.telescope.observations[0]
ready_status = self.scheduler.check_ingest_capacity(
observation, pipelines, max_ingest)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
observation.status = RunStatus.WAITING
status = self.env.process(
self.scheduler.allocate_ingest(observation, pipelines,
self.planner))
self.env.run(until=1)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=30)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
class TestSchedulerIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
self.cluster = Cluster(self.env, CLUSTER_CONFIG)
self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
FifoAlgorithm)
def testSchdulerCheckIngestReady(self):
"""
Check the return status of check_ingest_capacity is correct
"""
pipelines = {"continuum": {"demand": 5}}
observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type="continuum",
data_rate=2)
# There should be capacity
self.assertEqual(0.0, self.env.now)
ret = self.scheduler.check_ingest_capacity(observation, pipelines)
self.assertTrue(ret)
# Let's remove capacity to check it returns false
tmp = self.cluster.available_resources
self.cluster.available_resources = self.cluster.available_resources[:3]
ret = self.scheduler.check_ingest_capacity(observation, pipelines)
self.assertFalse(ret)
self.cluster.available_resources = tmp
self.assertEqual(10, len(self.cluster.available_resources))
def testSchedulerProvisionsIngest(self):
"""
Ensure that the scheduler correcly coordinates ingest onto the Cluster
and into the Buffer
Returns
-------
"""
pipelines = {"continuum": {"demand": 5}}
observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type="continuum",
data_rate=2)
ready_status = self.scheduler.check_ingest_capacity(
observation, pipelines)
observation.status = RunStatus.WAITING
status = self.env.process(
self.scheduler.allocate_ingest(observation, pipelines))
self.env.run(until=1)
self.assertEqual(5, len(self.cluster.available_resources))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(498, self.buffer.hot.current_capacity)
self.env.run(until=11)
self.assertEqual(10, len(self.cluster.available_resources))
self.assertEqual(5, len(self.cluster.finished_tasks))
self.assertEqual(480, self.buffer.hot.current_capacity)