class TestClusterTaskAllocation(unittest.TestCase):
def setUp(self) -> None:
"""
TODO
Set up tasks and cluster to attempt allocation of tasks to resources
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
self.task = Task('test_0', 0, 2, self.machine, [])
def test_allocate_task_to_cluster(self):
self.env.process(
self.cluster.allocate_task_to_cluster(self.task, self.machine))
self.env.run(until=1)
self.assertEqual(self.task, self.cluster._tasks['running'][0])
def test_duplication_allocation(self):
ret = self.env.process(
self.cluster.allocate_task_to_cluster(self.task, self.machine))
self.env.run(until=1)
newtask = Task('test_2', 8, 12, self.machine, [])
new_ret = self.env.process(
self.cluster.allocate_task_to_cluster(newtask, self.machine))
self.assertRaises(RuntimeError, self.env.run, until=2)
def setUp(self):
"""
Repeating above test cases but with delays to determine that delay
flags reach us.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
dm = DelayModel(0.9, "normal",
DelayModel.DelayDegree.HIGH)
self.planner = Planner(
self.env, PLANNING_ALGORITHM,
self.cluster, SHADOWPlanning('heft',delay_model=dm), delay_model=dm
)
self.scheduler = Scheduler(
self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan()
)
self.telescope = Telescope(
self.env, config, self.planner, self.scheduler
)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def __init__(self, machine_configs, task_configs, algorithm, event_file):
self.env = simpy.Environment()
cluster = Cluster()
cluster.add_machines(machine_configs)
task_broker = Broker(self.env, task_configs)
scheduler = Scheduler(self.env, algorithm)
self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
event_file)
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
# planner = None
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def setUp(self) -> None: self.env = simpy.Environment() self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG) self.observation = Observation( 'planner_observation', OBS_START_TME, OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW, type=None, data_rate=None )
def setUp(self) -> None:
"""
Setup a cluster and a simulation environment to test ingest pipelines
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
def setUp(self) -> None:
"""
Batch scheduling setup
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
def setUp(self) -> None:
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=self.config)
self.scheduler = Scheduler(env=self.env,
buffer=self.buffer,
cluster=self.cluster,
algorithm=None)
self.planner = Planner(self.env, 'heft', self.cluster,
SHADOWPlanning('heft'))
class TestIngest(unittest.TestCase): def setUp(self) -> None: self.env = simpy.Environment() self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG) self.observation = Observation( 'planner_observation', OBS_START_TME, OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW, type=None, data_rate=None ) def testClusterCheckIngest(self): retval = self.cluster.check_ingest_capacity(pipeline_demand=5) self.assertTrue(retval) def testClusterProvisionIngest(self): duration = self.observation.duration pipeline_demand = 5 self.env.process(self.cluster.run()) peek = self.env.peek() self.env.process(self.cluster.provision_ingest_resources( pipeline_demand, duration) ) self.env.run(until=1) self.assertEqual(1, self.env.now) # self.process(self.run_ingest(duration,pipeline_demand)) # for task in self.cluster.running_tasks: # self.assertEqual(TaskStatus.RUNNING, task.task_status) self.assertEqual(5, len(self.cluster.available_resources)) self.assertEqual(5, len(self.cluster.running_tasks)) self.env.run(until=10) self.assertEqual(5, len(self.cluster.available_resources)) self.env.run(until=20) self.assertEqual(10,len(self.cluster.available_resources)) self.assertEqual(20,self.env.now) def run_ingest(self, duration,demand): retval = self.cluster.provision_ingest_resources( demand, duration ) for task in self.cluster.running_tasks: self.assertEqual(TaskStatus.SCHEDULED, task.task_status)
def setUp(self) -> None:
"""
TODO
Set up tasks and cluster to attempt allocation of tasks to resources
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
self.task = Task('test_0', 0, 2, self.machine, [])
def setUp(self) -> None:
"""
Create a planner and a `simpy` environment in which to run dummy
simulations for the purpose of ensuring the planner works nicely
when selecting 'batch' as a static scheduling method.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.model = BatchPlanning('batch')
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
self.planner = Planner(
self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
def setUp(self) -> None:
"""
This test scenario is going to test the edge cases like
double-allocation that may happen.
For example - the use of curr_allocs should result in the
_process_current_schedule not double_allocation, so we will test this.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(CONFIG)
sched_algorithm = DynamicAlgorithmFromPlan()
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
sched_algorithm)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def setUp(self):
self.env = simpy.Environment()
cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
buffer = Buffer(env=self.env, cluster=cluster, config=BUFFER_CONFIG)
self.scheduler = Scheduler(env=self.env,
buffer=buffer,
cluster=cluster,
algorithm=None)
planner = Planner(self.env, 'heft', cluster)
def setUp(self):
self.algorithm = BatchProcessing
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.algorithm = BatchProcessing()
self.model = BatchPlanning('batch')
self.planner = Planner(
self.env,
'heft',
self.cluster,
self.model,
)
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def testClusterConfigFileExists(self): """ Initialise a Cluster object with the machine config file """ cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG) # This is a homogeneous file, so each flops value should be 95 for machine in cluster.machines: # machine is an object instance of Machine self.assertEqual(84, machine.cpu) self.assertEqual(10, machine.bandwidth)
def setUp(self):
self.env = simpy.Environment()
self.cluster = Cluster(self.env, CLUSTER_CONFIG)
self.planner = Planner(self.env, PLAN_ALGORITHM, self.cluster)
self.observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type=None,
data_rate=None)
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = FifoAlgorithm()
self.planner = Planner(self.env, PLAN_ALGORITHM, MACHINE_CONFIG)
self.cluster = Cluster(self.env, CLUSTER_CONFIG)
# self.buffer = Buffer(self.env, self.cluster)
# self.algorithms = Scheduler(self.env,
# sched_algorithm, self.buffer, self.cluster)
self.observation = Observation('planner_observation', OBS_START_TME,
OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW)
pass
class TestSchedulerIntegration(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_FIFO_with_buffer(self):
"""
Demonstrate that the scheduler accurately schedules when we have
other Actors working in tandem.
Expectations:
- After 1 timestep in the simualtion, we have 5 resources
available of the 10 that we start with.
-
Returns
-------
"""
self.env.run(until=1)
self.assertEqual(10, len(self.cluster._resources['available']))
# This takes timestep, data in the HotBuffer should be 4
self.env.run(until=2)
self.assertEqual(5, len(self.cluster._resources['available']))
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
# self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
self.env.run(until=32)
# Ensure the time
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
# 30 timesteps until we finish everything + 81 timesteps to complete
# workflow plan.
self.env.run(until=124)
# As we have been processing the current observation, we are also
# ingestting the next one.
self.assertEqual(250, self.buffer.cold[0].current_capacity)
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = DynamicAlgorithmFromPlan()
config = Config(LONG_CONFIG)
self.cluster = Cluster(self.env, config)
planning_model = SHADOWPlanning(algorithm=PLANNING_ALGORITHM)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
planning_model)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
sched_algorithm)
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def __init__( self, env, telescope_config, cluster_config, buffer_config, planning_algorithm, scheduling_algorithm, event_file, visualisation=False ): self.env = env # Event file setup self.event_file = event_file self.visualisation = visualisation if event_file is not None: self.monitor = Monitor(self) if visualisation: self.visualiser = Visualiser(self) # Process necessary config files # Initiaise Actor and Resource objects self.cluster = Cluster(env, cluster_config) self.buffer = Buffer(env, self.cluster, config=buffer_config) self.planner = Planner(env, planning_algorithm, cluster_config) self.scheduler = Scheduler( env, self.buffer, self.cluster, scheduling_algorithm ) self.telescope = Telescope( env=self.env, config=telescope_config, planner=self.planner, scheduler=self.scheduler )
def setUp(self):
self.env = simpy.Environment()
self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=BUFFER_CONFIG)
self.planner = Planner(self.env, PLAN_ALGORITHM, self.cluster)
self.observation = Observation('scheduler_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type='continuum',
data_rate=2)
def setUp(self):
self.env = simpy.Environment()
config = Config(CONFIG)
self.model = SHADOWPlanning('heft')
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
self.planner = Planner(
self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
)
self.observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
data_rate=OBS_DATA_RATE)
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = DynamicAlgorithmFromPlan()
config = Config(HEFT_CONFIG)
dm = DelayModel(0.1, "normal")
self.model = SHADOWPlanning('heft', dm)
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
delay_model=dm)
self.observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
data_rate=OBS_DATA_RATE)
class TestTelescopeIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=self.config)
self.scheduler = Scheduler(env=self.env,
buffer=self.buffer,
cluster=self.cluster,
algorithm=None)
self.planner = Planner(self.env, 'heft', self.cluster,
SHADOWPlanning('heft'))
def testIngest(self):
telescope = Telescope(env=self.env,
config=self.config,
planner=self.planner,
scheduler=self.scheduler)
self.assertEqual(0, telescope.telescope_use)
self.env.process(telescope.run())
self.env.process(self.cluster.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.buffer.run())
self.env.run(until=2)
self.assertEqual(36, telescope.telescope_use)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=11)
self.assertEqual(len([self.buffer.hot[0].observations["transfer"]]), 1)
self.assertEqual(462, self.buffer.hot[0].current_capacity)
self.assertEqual(248, self.buffer.cold[0].current_capacity)
self.env.run(until=12)
self.assertEqual(0, telescope.telescope_use)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = FifoAlgorithm()
self.planner = Planner(self.env, test_data.planning_algorithm,
test_data.machine_config)
self.cluster = Cluster(self.env, CLUSTER_CONFIG)
self.buffer = Buffer(self.env, self.cluster, BUFFER_CONFIG)
self.observations = [
Observation('scheduler_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
type='continuum',
data_rate=5)
]
telescopemax = 36 # maximum number of antennas
self.telescope = Telescope(self.env, OBSERVATION_CONFIG,
self.scheduler, self.planner)
self.scheduler = Scheduler(self.env, sched_algorithm, self.buffer,
self.cluster)
def setUp(self):
"""
Returns
-------
"""
"""
setup the buffer and do config stuff
:return: Nothing
"""
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(self.env, self.cluster, self.config)
self.observation = Observation(
name='test_observation',
start=OBS_START_TME,
duration=OBS_DURATION,
demand=OBS_DEMAND,
workflow=OBS_WORKFLOW,
data_rate=5,
)
class TestSchedulerDelayHelpers(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env,
PLANNING_ALGORITHM,
self.cluster,
SHADOWPlanning('heft'),
delay_model=DelayModel(0.3, "normal"))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_propogate_delay_returns_updated_workflow(self):
"""
def setUp(self):
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
def setUp(self):
self.env = simpy.Environment()
self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG)
