def setUp(self):
"""
Repeating above test cases but with delays to determine that delay
flags reach us.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
dm = DelayModel(0.9, "normal",
DelayModel.DelayDegree.HIGH)
self.planner = Planner(
self.env, PLANNING_ALGORITHM,
self.cluster, SHADOWPlanning('heft',delay_model=dm), delay_model=dm
)
self.scheduler = Scheduler(
self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan()
)
self.telescope = Telescope(
self.env, config, self.planner, self.scheduler
)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def setUp(self) -> None:
"""
This test scenario is going to test the edge cases like
double-allocation that may happen.
For example - the use of curr_allocs should result in the
_process_current_schedule not double_allocation, so we will test this.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(CONFIG)
sched_algorithm = DynamicAlgorithmFromPlan()
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
sched_algorithm)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
def setUp(self) -> None:
"""
Create a planner and a `simpy` environment in which to run dummy
simulations for the purpose of ensuring the planner works nicely
when selecting 'batch' as a static scheduling method.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.model = BatchPlanning('batch')
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
self.planner = Planner(
self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
def setUp(self):
self.env = simpy.Environment()
self.config = Config(CONFIG)
cluster = Cluster(env=self.env, config=self.config)
buffer = Buffer(env=self.env, cluster=cluster, config=self.config)
self.scheduler = Scheduler(env=self.env,
buffer=buffer,
cluster=cluster,
algorithm=None)
planner = Planner(self.env, 'heft', cluster, SHADOWPlanning)
def testClusterConfigFileExists(self):
"""
Initialise a Cluster object with the machine config file
"""
config = Config(CONFIG)
cluster = Cluster(env=self.env, config=config)
# This is a homogeneous file, so each flops value should be 95
for machine in cluster.machines:
# machine is an object instance of Machine
self.assertEqual(5040, machine.cpu)
self.assertEqual(600, machine.bandwidth)
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
# planner = None
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def setUp(self) -> None:
"""
Setup a cluster and a simulation environment to test ingest pipelines
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
def setUp(self) -> None:
"""
Batch scheduling setup
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = DynamicAlgorithmFromPlan()
config = Config(LONG_CONFIG)
self.cluster = Cluster(self.env, config)
planning_model = SHADOWPlanning(algorithm=PLANNING_ALGORITHM)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
planning_model)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
sched_algorithm)
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def setUp(self) -> None:
"""
TODO
Set up tasks and cluster to attempt allocation of tasks to resources
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
self.task = Task('test_0', 0, 2, self.machine, [])
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def setUp(self):
self.env = simpy.Environment()
config = Config(CONFIG)
self.model = SHADOWPlanning('heft')
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
self.planner = Planner(
self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
)
self.observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
data_rate=OBS_DATA_RATE)
def setUp(self):
self.env = simpy.Environment()
sched_algorithm = DynamicAlgorithmFromPlan()
config = Config(HEFT_CONFIG)
dm = DelayModel(0.1, "normal")
self.model = SHADOWPlanning('heft', dm)
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env,
PLAN_ALGORITHM,
self.cluster,
self.model,
delay_model=dm)
self.observation = Observation('planner_observation',
OBS_START_TME,
OBS_DURATION,
OBS_DEMAND,
OBS_WORKFLOW,
data_rate=OBS_DATA_RATE)
def setUp(self):
self.algorithm = BatchProcessing
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config=config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.algorithm = BatchProcessing()
self.model = BatchPlanning('batch')
self.planner = Planner(
self.env,
'heft',
self.cluster,
self.model,
)
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def setUp(self):
"""
Returns
-------
"""
"""
setup the buffer and do config stuff
:return: Nothing
"""
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(self.env, self.cluster, self.config)
self.observation = Observation(
name='test_observation',
start=OBS_START_TME,
duration=OBS_DURATION,
demand=OBS_DEMAND,
workflow=OBS_WORKFLOW,
data_rate=5,
)
def setUp(self):
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
def setUp(self):
self.env = simpy.Environment()
config = Config(CONFIG)
self.model = SHADOWPlanning('heft')
self.cluster = Cluster(env=self.env, config=config)
self.buffer = Buffer(env=self.env, cluster=self.cluster, config=config)
def __init__(self,
env,
config,
instrument,
planning_model,
planning_algorithm,
scheduling,
delay=None,
timestamp=None,
to_file=False,
hdf5_path=None,
**kwargs):
#: :py:obj:`simpy.Environment` object
self.env = env
if timestamp:
#: :py:obj:`~topsim.core.monitor.Monitor` instance
self.monitor = Monitor(self, timestamp)
self._timestamp = timestamp
else:
sim_start_time = f'{time.time()}'.split('.')[0]
self._timestamp = sim_start_time
self.monitor = Monitor(self, sim_start_time)
# Process necessary config files
self._cfg_path = config #: Configuration path
# Initiaise Actor and Resource objects
cfg = Config(config)
#: :py:obj:`~topsim.core.cluster.Cluster` instance
self.cluster = Cluster(env, cfg)
#: :py:obj:`~topsim.core.buffer.Buffer` instance
self.buffer = Buffer(env, self.cluster, cfg)
planning_algorithm = planning_algorithm
planning_model = planning_model
if not delay:
# TODO Have this approach replicated so we don't specify the
# model outside the simulation.
delay = DelayModel(0.0, "normal", DelayModel.DelayDegree.NONE)
self.planner = Planner(env, planning_algorithm, self.cluster,
planning_model, delay)
scheduling_algorithm = scheduling()
#: :py:obj:`~topsim.core.scheduler.Scheduler` instance
self.scheduler = Scheduler(env, self.buffer, self.cluster,
scheduling_algorithm)
#: User-defined :py:obj:`~topsim.core.instrument.Instrument` instance
self.instrument = instrument(env=self.env,
config=cfg,
planner=self.planner,
scheduler=self.scheduler)
#: :py:obj:`bool` Flag for producing simulation output in a `.pkl`
# file.
self.to_file = to_file
if self.to_file and hdf5_path:
try:
if os.path.exists(hdf5_path):
LOGGER.warning('Output HDF5 path already exists, '
'simulation appended to existing file')
self._hdf5_store = pd.HDFStore(hdf5_path)
self._hdf5_store.close()
except ValueError(
'Check pandas.HDFStore documentation for valid file path'):
raise
elif self.to_file and not hdf5_path:
raise ValueError(
'Attempted to initialise Simulation object that outputs'
'to file without providing file path')
else:
LOGGER.info(
'Simulation output will not be stored directly to file')
if 'delimiters' in kwargs:
#: Used to separate different simulations in HDF5 output
self._delimiters = kwargs['delimiters']
else:
self._delimiters = ''
self.running = False