class TestSchedulerIngest(unittest.TestCase): def setUp(self) -> None: self.env = simpy.Environment() config = Config(CONFIG) self.cluster = Cluster(self.env, config) self.buffer = Buffer(self.env, self.cluster, config) self.scheduler = Scheduler(self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan) self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster, SHADOWPlanning('heft')) # planner = None self.telescope = Telescope(self.env, config, self.planner, self.scheduler) def testSchdulerCheckIngestReady(self): """ Check the return status of check_ingest_capacity is correct """ pipelines = self.telescope.pipelines observation = self.telescope.observations[0] max_ingest = self.telescope.max_ingest # There should be capacity self.assertEqual(0.0, self.env.now) ret = self.scheduler.check_ingest_capacity(observation, pipelines, max_ingest) self.assertTrue(ret) # Let's remove capacity to check it returns false tmp = self.cluster._resources['available'] self.cluster._resources['available'] = self.cluster._resources[ 'available'][:3] ret = self.scheduler.check_ingest_capacity(observation, pipelines, max_ingest) self.assertFalse(ret) self.cluster._resources['available'] = tmp self.assertEqual(10, len(self.cluster._resources['available'])) def testSchedulerProvisionsIngest(self): """ Ensure that the scheduler correcly coordinates ingest onto the Cluster and into the Buffer Returns ------- """ pipelines = self.telescope.pipelines max_ingest = self.telescope.max_ingest observation = self.telescope.observations[0] ready_status = self.scheduler.check_ingest_capacity( observation, pipelines, max_ingest) self.env.process(self.cluster.run()) self.env.process(self.buffer.run()) observation.status = RunStatus.WAITING status = self.env.process( self.scheduler.allocate_ingest(observation, pipelines, self.planner)) self.env.run(until=1) self.assertEqual(5, len(self.cluster._resources['available'])) # After 1 timestep, data in the HotBuffer should be 2 self.assertEqual(496, self.buffer.hot[0].current_capacity) self.env.run(until=30) self.assertEqual(10, len(self.cluster._resources['available'])) self.assertEqual(5, len(self.cluster._tasks['finished'])) self.assertEqual(500, self.buffer.hot[0].current_capacity) self.assertEqual(210, self.buffer.cold[0].current_capacity)
class TestSchedulerIngest(unittest.TestCase): def setUp(self) -> None: self.env = simpy.Environment() self.cluster = Cluster(self.env, CLUSTER_CONFIG) self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG) self.scheduler = Scheduler(self.env, self.buffer, self.cluster, FifoAlgorithm) def testSchdulerCheckIngestReady(self): """ Check the return status of check_ingest_capacity is correct """ pipelines = {"continuum": {"demand": 5}} observation = Observation('planner_observation', OBS_START_TME, OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW, type="continuum", data_rate=2) # There should be capacity self.assertEqual(0.0, self.env.now) ret = self.scheduler.check_ingest_capacity(observation, pipelines) self.assertTrue(ret) # Let's remove capacity to check it returns false tmp = self.cluster.available_resources self.cluster.available_resources = self.cluster.available_resources[:3] ret = self.scheduler.check_ingest_capacity(observation, pipelines) self.assertFalse(ret) self.cluster.available_resources = tmp self.assertEqual(10, len(self.cluster.available_resources)) def testSchedulerProvisionsIngest(self): """ Ensure that the scheduler correcly coordinates ingest onto the Cluster and into the Buffer Returns ------- """ pipelines = {"continuum": {"demand": 5}} observation = Observation('planner_observation', OBS_START_TME, OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW, type="continuum", data_rate=2) ready_status = self.scheduler.check_ingest_capacity( observation, pipelines) observation.status = RunStatus.WAITING status = self.env.process( self.scheduler.allocate_ingest(observation, pipelines)) self.env.run(until=1) self.assertEqual(5, len(self.cluster.available_resources)) # After 1 timestep, data in the HotBuffer should be 2 self.assertEqual(498, self.buffer.hot.current_capacity) self.env.run(until=11) self.assertEqual(10, len(self.cluster.available_resources)) self.assertEqual(5, len(self.cluster.finished_tasks)) self.assertEqual(480, self.buffer.hot.current_capacity)