Exemple #1
0
class TestClusterTaskAllocation(unittest.TestCase):
    def setUp(self) -> None:
        """
        TODO
        Set up tasks and cluster to attempt allocation of tasks to resources
        """
        self.env = simpy.Environment()
        config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=config)
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
        self.observation = self.telescope.observations[0]
        self.machine = self.cluster.machines[0]
        self.task = Task('test_0', 0, 2, self.machine, [])

    def test_allocate_task_to_cluster(self):
        self.env.process(
            self.cluster.allocate_task_to_cluster(self.task, self.machine))
        self.env.run(until=1)
        self.assertEqual(self.task, self.cluster._tasks['running'][0])

    def test_duplication_allocation(self):
        ret = self.env.process(
            self.cluster.allocate_task_to_cluster(self.task, self.machine))
        self.env.run(until=1)
        newtask = Task('test_2', 8, 12, self.machine, [])
        new_ret = self.env.process(
            self.cluster.allocate_task_to_cluster(newtask, self.machine))
        self.assertRaises(RuntimeError, self.env.run, until=2)
Exemple #2
0
    def setUp(self):
        """
        Repeating above test cases but with delays to determine that delay
        flags reach us.
        Returns
        -------

        """

        self.env = simpy.Environment()
        config = Config(INTEGRATION)
        self.cluster = Cluster(self.env, config)
        self.buffer = Buffer(self.env, self.cluster, config)
        dm = DelayModel(0.9, "normal",
                   DelayModel.DelayDegree.HIGH)
        self.planner = Planner(
            self.env, PLANNING_ALGORITHM,
            self.cluster, SHADOWPlanning('heft',delay_model=dm), delay_model=dm
        )

        self.scheduler = Scheduler(
            self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan()
        )
        self.telescope = Telescope(
            self.env, config, self.planner, self.scheduler
        )
        self.env.process(self.cluster.run())
        self.env.process(self.buffer.run())
        self.scheduler.start()
        self.env.process(self.scheduler.run())
        self.env.process(self.telescope.run())
Exemple #3
0
    def __init__(self, machine_configs, task_configs, algorithm, event_file):
        self.env = simpy.Environment()
        cluster = Cluster()
        cluster.add_machines(machine_configs)

        task_broker = Broker(self.env, task_configs)

        scheduler = Scheduler(self.env, algorithm)

        self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
                                     event_file)
Exemple #4
0
 def setUp(self) -> None:
     self.env = simpy.Environment()
     config = Config(CONFIG)
     self.cluster = Cluster(self.env, config)
     self.buffer = Buffer(self.env, self.cluster, config)
     self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                DynamicAlgorithmFromPlan)
     self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
                            SHADOWPlanning('heft'))
     # planner = None
     self.telescope = Telescope(self.env, config, self.planner,
                                self.scheduler)
Exemple #5
0
	def setUp(self) -> None:
		self.env = simpy.Environment()
		self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
		self.observation = Observation(
			'planner_observation',
			OBS_START_TME,
			OBS_DURATION,
			OBS_DEMAND,
			OBS_WORKFLOW,
			type=None,
			data_rate=None
		)
Exemple #6
0
    def setUp(self) -> None:
        """
        Setup a cluster and a simulation environment to test ingest pipelines
        """
        self.env = simpy.Environment()
        config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=config)
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
        self.observation = self.telescope.observations[0]
Exemple #7
0
    def setUp(self) -> None:
        """
        Batch scheduling setup
        """
        self.env = simpy.Environment()
        config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=config)
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
        self.observation = self.telescope.observations[0]
Exemple #8
0
    def setUp(self) -> None:
        self.env = simpy.Environment()
        self.config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=self.config)
        self.buffer = Buffer(env=self.env,
                             cluster=self.cluster,
                             config=self.config)
        self.scheduler = Scheduler(env=self.env,
                                   buffer=self.buffer,
                                   cluster=self.cluster,
                                   algorithm=None)
        self.planner = Planner(self.env, 'heft', self.cluster,
                               SHADOWPlanning('heft'))
Exemple #9
0
class TestIngest(unittest.TestCase):

	def setUp(self) -> None:
		self.env = simpy.Environment()
		self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
		self.observation = Observation(
			'planner_observation',
			OBS_START_TME,
			OBS_DURATION,
			OBS_DEMAND,
			OBS_WORKFLOW,
			type=None,
			data_rate=None
		)

	def testClusterCheckIngest(self):
		retval = self.cluster.check_ingest_capacity(pipeline_demand=5)
		self.assertTrue(retval)

	def testClusterProvisionIngest(self):
		duration = self.observation.duration
		pipeline_demand = 5
		self.env.process(self.cluster.run())
		peek = self.env.peek()
		self.env.process(self.cluster.provision_ingest_resources(
			pipeline_demand,
			duration)
		)
		self.env.run(until=1)
		self.assertEqual(1, self.env.now)
		# self.process(self.run_ingest(duration,pipeline_demand))
		# for task in self.cluster.running_tasks:
		#  	self.assertEqual(TaskStatus.RUNNING, task.task_status)
		self.assertEqual(5, len(self.cluster.available_resources))
		self.assertEqual(5, len(self.cluster.running_tasks))
		self.env.run(until=10)
		self.assertEqual(5, len(self.cluster.available_resources))
		self.env.run(until=20)
		self.assertEqual(10,len(self.cluster.available_resources))
		self.assertEqual(20,self.env.now)

	def run_ingest(self, duration,demand):
		retval = self.cluster.provision_ingest_resources(
			demand,
			duration
		)
		for task in self.cluster.running_tasks:
			self.assertEqual(TaskStatus.SCHEDULED, task.task_status)
Exemple #10
0
    def setUp(self) -> None:
        """
        TODO
        Set up tasks and cluster to attempt allocation of tasks to resources
        """
        self.env = simpy.Environment()
        config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=config)
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
        self.observation = self.telescope.observations[0]
        self.machine = self.cluster.machines[0]
        self.task = Task('test_0', 0, 2, self.machine, [])
Exemple #11
0
    def setUp(self) -> None:
        """
        Create a planner and a `simpy` environment in which to run dummy
        simulations for the purpose of ensuring the planner works nicely
        when selecting 'batch' as a static scheduling method.
        Returns
        -------

        """
        self.env = simpy.Environment()
        config = Config(CONFIG)
        self.model = BatchPlanning('batch')

        self.cluster = Cluster(self.env, config=config)
        self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
        self.planner = Planner(
            self.env,
            PLAN_ALGORITHM,
            self.cluster,
            self.model,
        )
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
Exemple #12
0
    def setUp(self) -> None:
        """
        This test scenario is going to test the edge cases like
        double-allocation that may happen.

        For example - the use of curr_allocs should result in the
        _process_current_schedule not double_allocation, so we will test this.
        Returns
        -------

        """
        self.env = simpy.Environment()
        config = Config(CONFIG)
        sched_algorithm = DynamicAlgorithmFromPlan()

        self.cluster = Cluster(env=self.env, config=config)
        self.telescope = Telescope(self.env,
                                   config,
                                   planner=None,
                                   scheduler=None)
        self.buffer = Buffer(self.env, self.cluster, config)

        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   sched_algorithm)

        self.observation = self.telescope.observations[0]
        self.machine = self.cluster.machines[0]
Exemple #13
0
    def setUp(self):
        self.env = simpy.Environment()
        config = Config(INTEGRATION)
        self.cluster = Cluster(self.env, config)
        self.buffer = Buffer(self.env, self.cluster, config)
        self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
                               SHADOWPlanning('heft'))

        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   DynamicAlgorithmFromPlan())
        self.telescope = Telescope(self.env, config, self.planner,
                                   self.scheduler)
        self.env.process(self.cluster.run())
        self.env.process(self.buffer.run())
        self.scheduler.start()
        self.env.process(self.scheduler.run())
        self.env.process(self.telescope.run())
Exemple #14
0
 def setUp(self):
     self.env = simpy.Environment()
     cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
     buffer = Buffer(env=self.env, cluster=cluster, config=BUFFER_CONFIG)
     self.scheduler = Scheduler(env=self.env,
                                buffer=buffer,
                                cluster=cluster,
                                algorithm=None)
     planner = Planner(self.env, 'heft', cluster)
Exemple #15
0
    def setUp(self):
        self.algorithm = BatchProcessing
        self.env = simpy.Environment()
        config = Config(CONFIG)
        self.cluster = Cluster(self.env, config=config)
        self.buffer = Buffer(self.env, self.cluster, config)
        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   DynamicAlgorithmFromPlan())
        self.algorithm = BatchProcessing()
        self.model = BatchPlanning('batch')
        self.planner = Planner(
            self.env,
            'heft',
            self.cluster,
            self.model,
        )

        self.telescope = Telescope(self.env, config, self.planner,
                                   self.scheduler)
Exemple #16
0
	def testClusterConfigFileExists(self):
		"""
		Initialise a Cluster object with the machine config file
		"""
		cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
		# This is a homogeneous file, so each flops value should be 95
		for machine in cluster.machines:
			# machine is an object instance of Machine
			self.assertEqual(84, machine.cpu)
			self.assertEqual(10, machine.bandwidth)
Exemple #17
0
 def setUp(self):
     self.env = simpy.Environment()
     self.cluster = Cluster(self.env, CLUSTER_CONFIG)
     self.planner = Planner(self.env, PLAN_ALGORITHM, self.cluster)
     self.observation = Observation('planner_observation',
                                    OBS_START_TME,
                                    OBS_DURATION,
                                    OBS_DEMAND,
                                    OBS_WORKFLOW,
                                    type=None,
                                    data_rate=None)
Exemple #18
0
 def setUp(self):
     self.env = simpy.Environment()
     sched_algorithm = FifoAlgorithm()
     self.planner = Planner(self.env, PLAN_ALGORITHM, MACHINE_CONFIG)
     self.cluster = Cluster(self.env, CLUSTER_CONFIG)
     # self.buffer = Buffer(self.env, self.cluster)
     # self.algorithms = Scheduler(self.env,
     # sched_algorithm, self.buffer, self.cluster)
     self.observation = Observation('planner_observation', OBS_START_TME,
                                    OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW)
     pass
Exemple #19
0
class TestSchedulerIntegration(unittest.TestCase):
    def setUp(self):
        self.env = simpy.Environment()
        config = Config(INTEGRATION)
        self.cluster = Cluster(self.env, config)
        self.buffer = Buffer(self.env, self.cluster, config)
        self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
                               SHADOWPlanning('heft'))

        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   DynamicAlgorithmFromPlan())
        self.telescope = Telescope(self.env, config, self.planner,
                                   self.scheduler)
        self.env.process(self.cluster.run())
        self.env.process(self.buffer.run())
        self.scheduler.start()
        self.env.process(self.scheduler.run())
        self.env.process(self.telescope.run())

    def test_FIFO_with_buffer(self):
        """
        Demonstrate that the scheduler accurately schedules when we have
        other Actors working in tandem.

        Expectations:
            - After 1 timestep in the simualtion, we have 5 resources
            available of the 10 that we start with.
            -
        Returns
        -------

        """
        self.env.run(until=1)

        self.assertEqual(10, len(self.cluster._resources['available']))
        # This takes timestep, data in the HotBuffer should be 4
        self.env.run(until=2)
        self.assertEqual(5, len(self.cluster._resources['available']))
        self.assertEqual(496, self.buffer.hot[0].current_capacity)
        self.env.run(until=31)
        self.assertEqual(5, len(self.cluster._tasks['finished']))
        # self.assertEqual(500, self.buffer.hot[0].current_capacity)
        self.assertEqual(210, self.buffer.cold[0].current_capacity)
        self.env.run(until=32)
        # Ensure the time
        self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
        # 30 timesteps until we finish everything + 81 timesteps to complete
        # workflow plan.
        self.env.run(until=124)
        # As we have been processing the current observation, we are also
        # ingestting the next one.
        self.assertEqual(250, self.buffer.cold[0].current_capacity)
Exemple #20
0
 def setUp(self):
     self.env = simpy.Environment()
     sched_algorithm = DynamicAlgorithmFromPlan()
     config = Config(LONG_CONFIG)
     self.cluster = Cluster(self.env, config)
     planning_model = SHADOWPlanning(algorithm=PLANNING_ALGORITHM)
     self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
                            planning_model)
     self.buffer = Buffer(self.env, self.cluster, config)
     self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                sched_algorithm)
     self.telescope = Telescope(self.env, config, self.planner,
                                self.scheduler)
Exemple #21
0
	def __init__(
			self,
			env,
			telescope_config,
			cluster_config,
			buffer_config,
			planning_algorithm,
			scheduling_algorithm,
			event_file,
			visualisation=False
	):

		self.env = env
		# Event file setup
		self.event_file = event_file
		self.visualisation = visualisation
		if event_file is not None:
			self.monitor = Monitor(self)
		if visualisation:
			self.visualiser = Visualiser(self)
		# Process necessary config files

		# Initiaise Actor and Resource objects

		self.cluster = Cluster(env, cluster_config)
		self.buffer = Buffer(env, self.cluster, config=buffer_config)
		self.planner = Planner(env, planning_algorithm, cluster_config)
		self.scheduler = Scheduler(
			env, self.buffer, self.cluster, scheduling_algorithm
		)

		self.telescope = Telescope(
			env=self.env,
			config=telescope_config,
			planner=self.planner,
			scheduler=self.scheduler
		)
Exemple #22
0
    def setUp(self):
        self.env = simpy.Environment()
        self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)

        self.buffer = Buffer(env=self.env,
                             cluster=self.cluster,
                             config=BUFFER_CONFIG)
        self.planner = Planner(self.env, PLAN_ALGORITHM, self.cluster)
        self.observation = Observation('scheduler_observation',
                                       OBS_START_TME,
                                       OBS_DURATION,
                                       OBS_DEMAND,
                                       OBS_WORKFLOW,
                                       type='continuum',
                                       data_rate=2)
Exemple #23
0
 def setUp(self):
     self.env = simpy.Environment()
     config = Config(CONFIG)
     self.model = SHADOWPlanning('heft')
     self.cluster = Cluster(self.env, config=config)
     self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
     self.planner = Planner(
         self.env,
         PLAN_ALGORITHM,
         self.cluster,
         self.model,
     )
     self.observation = Observation('planner_observation',
                                    OBS_START_TME,
                                    OBS_DURATION,
                                    OBS_DEMAND,
                                    OBS_WORKFLOW,
                                    data_rate=OBS_DATA_RATE)
Exemple #24
0
 def setUp(self):
     self.env = simpy.Environment()
     sched_algorithm = DynamicAlgorithmFromPlan()
     config = Config(HEFT_CONFIG)
     dm = DelayModel(0.1, "normal")
     self.model = SHADOWPlanning('heft', dm)
     self.cluster = Cluster(self.env, config=config)
     self.buffer = Buffer(self.env, self.cluster, config)
     self.planner = Planner(self.env,
                            PLAN_ALGORITHM,
                            self.cluster,
                            self.model,
                            delay_model=dm)
     self.observation = Observation('planner_observation',
                                    OBS_START_TME,
                                    OBS_DURATION,
                                    OBS_DEMAND,
                                    OBS_WORKFLOW,
                                    data_rate=OBS_DATA_RATE)
Exemple #25
0
class TestTelescopeIngest(unittest.TestCase):
    def setUp(self) -> None:
        self.env = simpy.Environment()
        self.config = Config(CONFIG)

        self.cluster = Cluster(env=self.env, config=self.config)
        self.buffer = Buffer(env=self.env,
                             cluster=self.cluster,
                             config=self.config)
        self.scheduler = Scheduler(env=self.env,
                                   buffer=self.buffer,
                                   cluster=self.cluster,
                                   algorithm=None)
        self.planner = Planner(self.env, 'heft', self.cluster,
                               SHADOWPlanning('heft'))

    def testIngest(self):
        telescope = Telescope(env=self.env,
                              config=self.config,
                              planner=self.planner,
                              scheduler=self.scheduler)
        self.assertEqual(0, telescope.telescope_use)
        self.env.process(telescope.run())
        self.env.process(self.cluster.run())
        self.scheduler.start()
        self.env.process(self.scheduler.run())
        self.env.process(self.buffer.run())
        self.env.run(until=2)
        self.assertEqual(36, telescope.telescope_use)
        self.assertEqual(5, len(self.cluster._resources['available']))
        # After 1 timestep, data in the HotBuffer should be 2
        self.assertEqual(496, self.buffer.hot[0].current_capacity)
        self.env.run(until=11)
        self.assertEqual(len([self.buffer.hot[0].observations["transfer"]]), 1)
        self.assertEqual(462, self.buffer.hot[0].current_capacity)
        self.assertEqual(248, self.buffer.cold[0].current_capacity)
        self.env.run(until=12)
        self.assertEqual(0, telescope.telescope_use)
        self.assertEqual(10, len(self.cluster._resources['available']))
        self.assertEqual(5, len(self.cluster._tasks['finished']))
Exemple #26
0
    def setUp(self):
        self.env = simpy.Environment()
        sched_algorithm = FifoAlgorithm()
        self.planner = Planner(self.env, test_data.planning_algorithm,
                               test_data.machine_config)
        self.cluster = Cluster(self.env, CLUSTER_CONFIG)
        self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG)
        self.observations = [
            Observation('scheduler_observation',
                        OBS_START_TME,
                        OBS_DURATION,
                        OBS_DEMAND,
                        OBS_WORKFLOW,
                        type='continuum',
                        data_rate=5)
        ]
        telescopemax = 36  # maximum number of antennas

        self.telescope = Telescope(self.env, OBSERVATION_CONFIG,
                                   self.scheduler, self.planner)
        self.scheduler = Scheduler(self.env, sched_algorithm, self.buffer,
                                   self.cluster)
Exemple #27
0
    def setUp(self):
        """

        Returns
        -------

        """
        """
        setup the buffer and do config stuff
        :return: Nothing
        """
        self.env = simpy.Environment()
        self.config = Config(CONFIG)
        self.cluster = Cluster(env=self.env, config=self.config)
        self.buffer = Buffer(self.env, self.cluster, self.config)
        self.observation = Observation(
            name='test_observation',
            start=OBS_START_TME,
            duration=OBS_DURATION,
            demand=OBS_DEMAND,
            workflow=OBS_WORKFLOW,
            data_rate=5,
        )
Exemple #28
0
class TestSchedulerDelayHelpers(unittest.TestCase):
    def setUp(self):
        self.env = simpy.Environment()
        config = Config(INTEGRATION)
        self.cluster = Cluster(self.env, config)
        self.buffer = Buffer(self.env, self.cluster, config)
        self.planner = Planner(self.env,
                               PLANNING_ALGORITHM,
                               self.cluster,
                               SHADOWPlanning('heft'),
                               delay_model=DelayModel(0.3, "normal"))

        self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
                                   DynamicAlgorithmFromPlan())
        self.telescope = Telescope(self.env, config, self.planner,
                                   self.scheduler)
        self.env.process(self.cluster.run())
        self.env.process(self.buffer.run())
        self.scheduler.start()
        self.env.process(self.scheduler.run())
        self.env.process(self.telescope.run())

    def test_propogate_delay_returns_updated_workflow(self):
        """
Exemple #29
0
 def setUp(self):
     self.env = simpy.Environment()
     self.config = Config(CONFIG)
     self.cluster = Cluster(env=self.env, config=self.config)
Exemple #30
0
 def setUp(self):
     self.env = simpy.Environment()
     self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)