Exemplo n.º 1
0
class TestSchedulerIngest(unittest.TestCase):
    def setUp(self) -> None:
        self.env = simpy.Environment()
        config = Config(CONFIG)
        self.cluster = Cluster(self.env, config)
        self.buffer = Buffer(self.env, self.cluster, config)
        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   DynamicAlgorithmFromPlan)
        self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
                               SHADOWPlanning('heft'))
        # planner = None
        self.telescope = Telescope(self.env, config, self.planner,
                                   self.scheduler)

    def testSchdulerCheckIngestReady(self):
        """
        Check the return status of check_ingest_capacity is correct
        """
        pipelines = self.telescope.pipelines
        observation = self.telescope.observations[0]

        max_ingest = self.telescope.max_ingest
        # There should be capacity
        self.assertEqual(0.0, self.env.now)
        ret = self.scheduler.check_ingest_capacity(observation, pipelines,
                                                   max_ingest)
        self.assertTrue(ret)

        # Let's remove capacity to check it returns false
        tmp = self.cluster._resources['available']
        self.cluster._resources['available'] = self.cluster._resources[
            'available'][:3]
        ret = self.scheduler.check_ingest_capacity(observation, pipelines,
                                                   max_ingest)
        self.assertFalse(ret)
        self.cluster._resources['available'] = tmp
        self.assertEqual(10, len(self.cluster._resources['available']))

    def testSchedulerProvisionsIngest(self):
        """
        Ensure that the scheduler correcly coordinates ingest onto the Cluster
        and into the Buffer

        Returns
        -------
        """
        pipelines = self.telescope.pipelines
        max_ingest = self.telescope.max_ingest
        observation = self.telescope.observations[0]

        ready_status = self.scheduler.check_ingest_capacity(
            observation, pipelines, max_ingest)
        self.env.process(self.cluster.run())
        self.env.process(self.buffer.run())
        observation.status = RunStatus.WAITING
        status = self.env.process(
            self.scheduler.allocate_ingest(observation, pipelines,
                                           self.planner))

        self.env.run(until=1)
        self.assertEqual(5, len(self.cluster._resources['available']))
        # After 1 timestep, data in the HotBuffer should be 2
        self.assertEqual(496, self.buffer.hot[0].current_capacity)
        self.env.run(until=30)
        self.assertEqual(10, len(self.cluster._resources['available']))
        self.assertEqual(5, len(self.cluster._tasks['finished']))
        self.assertEqual(500, self.buffer.hot[0].current_capacity)
        self.assertEqual(210, self.buffer.cold[0].current_capacity)
Exemplo n.º 2
0
class TestSchedulerIngest(unittest.TestCase):
    def setUp(self) -> None:
        self.env = simpy.Environment()
        self.cluster = Cluster(self.env, CLUSTER_CONFIG)
        self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG)
        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   FifoAlgorithm)

    def testSchdulerCheckIngestReady(self):
        """
		Check the return status of check_ingest_capacity is correct
		"""
        pipelines = {"continuum": {"demand": 5}}
        observation = Observation('planner_observation',
                                  OBS_START_TME,
                                  OBS_DURATION,
                                  OBS_DEMAND,
                                  OBS_WORKFLOW,
                                  type="continuum",
                                  data_rate=2)
        # There should be capacity
        self.assertEqual(0.0, self.env.now)
        ret = self.scheduler.check_ingest_capacity(observation, pipelines)
        self.assertTrue(ret)

        # Let's remove capacity to check it returns false
        tmp = self.cluster.available_resources
        self.cluster.available_resources = self.cluster.available_resources[:3]
        ret = self.scheduler.check_ingest_capacity(observation, pipelines)
        self.assertFalse(ret)
        self.cluster.available_resources = tmp
        self.assertEqual(10, len(self.cluster.available_resources))

    def testSchedulerProvisionsIngest(self):
        """
		Ensure that the scheduler correcly coordinates ingest onto the Cluster
		and into the Buffer

		Returns
		-------
		"""
        pipelines = {"continuum": {"demand": 5}}

        observation = Observation('planner_observation',
                                  OBS_START_TME,
                                  OBS_DURATION,
                                  OBS_DEMAND,
                                  OBS_WORKFLOW,
                                  type="continuum",
                                  data_rate=2)
        ready_status = self.scheduler.check_ingest_capacity(
            observation, pipelines)
        observation.status = RunStatus.WAITING
        status = self.env.process(
            self.scheduler.allocate_ingest(observation, pipelines))
        self.env.run(until=1)
        self.assertEqual(5, len(self.cluster.available_resources))
        # After 1 timestep, data in the HotBuffer should be 2
        self.assertEqual(498, self.buffer.hot.current_capacity)
        self.env.run(until=11)
        self.assertEqual(10, len(self.cluster.available_resources))
        self.assertEqual(5, len(self.cluster.finished_tasks))
        self.assertEqual(480, self.buffer.hot.current_capacity)