class TestSchedulerIntegration(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_FIFO_with_buffer(self):
"""
Demonstrate that the scheduler accurately schedules when we have
other Actors working in tandem.
Expectations:
- After 1 timestep in the simualtion, we have 5 resources
available of the 10 that we start with.
-
Returns
-------
"""
self.env.run(until=1)
self.assertEqual(10, len(self.cluster._resources['available']))
# This takes timestep, data in the HotBuffer should be 4
self.env.run(until=2)
self.assertEqual(5, len(self.cluster._resources['available']))
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
# self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
self.env.run(until=32)
# Ensure the time
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
# 30 timesteps until we finish everything + 81 timesteps to complete
# workflow plan.
self.env.run(until=124)
# As we have been processing the current observation, we are also
# ingestting the next one.
self.assertEqual(250, self.buffer.cold[0].current_capacity)
class TestTelescopeIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=self.config)
self.scheduler = Scheduler(env=self.env,
buffer=self.buffer,
cluster=self.cluster,
algorithm=None)
self.planner = Planner(self.env, 'heft', self.cluster,
SHADOWPlanning('heft'))
def testIngest(self):
telescope = Telescope(env=self.env,
config=self.config,
planner=self.planner,
scheduler=self.scheduler)
self.assertEqual(0, telescope.telescope_use)
self.env.process(telescope.run())
self.env.process(self.cluster.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.buffer.run())
self.env.run(until=2)
self.assertEqual(36, telescope.telescope_use)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=11)
self.assertEqual(len([self.buffer.hot[0].observations["transfer"]]), 1)
self.assertEqual(462, self.buffer.hot[0].current_capacity)
self.assertEqual(248, self.buffer.cold[0].current_capacity)
self.env.run(until=12)
self.assertEqual(0, telescope.telescope_use)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
class TestSchedulerDelayHelpers(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env,
PLANNING_ALGORITHM,
self.cluster,
SHADOWPlanning('heft'),
delay_model=DelayModel(0.3, "normal"))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_propogate_delay_returns_updated_workflow(self):
"""
class TestSchedulerIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
# planner = None
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def testSchdulerCheckIngestReady(self):
"""
Check the return status of check_ingest_capacity is correct
"""
pipelines = self.telescope.pipelines
observation = self.telescope.observations[0]
max_ingest = self.telescope.max_ingest
# There should be capacity
self.assertEqual(0.0, self.env.now)
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertTrue(ret)
# Let's remove capacity to check it returns false
tmp = self.cluster._resources['available']
self.cluster._resources['available'] = self.cluster._resources[
'available'][:3]
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertFalse(ret)
self.cluster._resources['available'] = tmp
self.assertEqual(10, len(self.cluster._resources['available']))
def testSchedulerProvisionsIngest(self):
"""
Ensure that the scheduler correcly coordinates ingest onto the Cluster
and into the Buffer
Returns
-------
"""
pipelines = self.telescope.pipelines
max_ingest = self.telescope.max_ingest
observation = self.telescope.observations[0]
ready_status = self.scheduler.check_ingest_capacity(
observation, pipelines, max_ingest)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
observation.status = RunStatus.WAITING
status = self.env.process(
self.scheduler.allocate_ingest(observation, pipelines,
self.planner))
self.env.run(until=1)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=30)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
class TestDelaysInActors(unittest.TestCase):
def setUp(self):
"""
Repeating above test cases but with delays to determine that delay
flags reach us.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
dm = DelayModel(0.9, "normal",
DelayModel.DelayDegree.HIGH)
self.planner = Planner(
self.env, PLANNING_ALGORITHM,
self.cluster, SHADOWPlanning('heft',delay_model=dm), delay_model=dm
)
self.scheduler = Scheduler(
self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan()
)
self.telescope = Telescope(
self.env, config, self.planner, self.scheduler
)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_scheduler_delay_detection(self):
"""
Nothing should change until we reach the workflow plan, as we are
testing TaskDelays
Returns
-------
"""
self.env.run(until=1)
# Remember - env starts at 0, we don't start until 1.
self.assertEqual(10, len(self.cluster._resources['available']))
self.env.run(until=2)
# After 1 timestep, data in the HotBuffer should be 4
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.env.run(until=44)
# We know that the schedule has been delayed - however, we don't
# report this to the telescope until we know how long we are delayed
# (that is, until the task has completely finished its duration).
# In this instance. we know that the first task is going to be
# delayed, and so wait until it's completed execution to trigger a
# delay.
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
self.env.run(until=45)
self.assertTrue(ScheduleStatus.DELAYED,self.scheduler.schedule_status)
self.env.run(until=124)
# Assert that we still have tasks running
# self.assertLess(
# 0, len(self.cluster.clusters['default']['tasks']['running'])
# )
self.assertNotEqual(250, self.buffer.cold[0].current_capacity)
def test_telescope_delay_detection(self):
"""
Returns
-------
"""
self.env.run(until=1)
# Remember - env starts at 0, we don't start until 1.
self.assertEqual(10, len(self.cluster._resources['available']))
self.env.run(until=2)
# After 1 timestep, data in the HotBuffer should be 4
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.env.run(until=32)
# Ensure the time
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
self.env.run(until=100)
self.assertEqual(ScheduleStatus.DELAYED,self.scheduler.schedule_status)
self.assertTrue(self.telescope.delayed)
def test_telescope_delay_greedy_decision(self):
"""
class Simulation:
"""
The Simulation class is a wrapper for all Actors; we start the simulation
through the simulation class, which in turn invokes the initial Actors and
monitoring, and provides the conditions for checking if the simulation has
finished.
Parameters
----------
env : :py:obj:`simpy.Environment` object
The discrete-event simulation environment. This is the way TOpSim
simulation maintains state across the different actors,
and interfaces with the simpy processes.
config : str
Path to the simulation JSOn configuration file
instrument : :py:obj:`~topsim.core.instrument.Instrument`
User-defined implementation of the Instrument class.
planning_model : :py:obj:`~topsim.algorithms.planning.Planning` object
User-defined implementation of the planning algorithm class
planning_algorithm: str
Reference to the specific algorithm implementated in `planning_model`
scheduling: :py:obj:`~topsim.algorithms.scheduling.Algorithm`
User-defined implementation of the scheduling algorithm
:py:obj:`abc.ABC`.
delay: :py:obj:`~topsim.core.delay.DelayModel`, optional
for the simulation.
timestamp: str, optional
Optional Simulation start-time; this is useful for testing, to ensure we
name the file and the tests match up. Also useful if you do not want to
use the time of the simulation as the name.
to_file : bool, optional
`True` if the simulation is to be written to a Pandas `pkl` file;
`False` will return pandas DataFrame objects at the completion of the
:py:meth:`~topsim.core.simulation.Simulation.run` function.
Notes
-----
If to_file left as `False`, simulation results and output will be returned
as Pandas DataFrames (see
:py:meth:`~topsim.core.simulation.Simulation.run`) . This is designed for
running multiple simulations, allowing for the appending of individual
simulation results to a 'global' :py:obj:`~pandas.DataFrame` . Current
support for output is limited to Panda's `.pkl` files.
Parsing in the option `delimiters` provides a way of differentiating
between multiple simulations within a single HDF5 store (for example,
in an experiment). A typical experimental loop may involve the following
structure:
>>> for heuristic in list_of_scheduling_heuristics
>>> for algorithm in list_of_planning_algorithms
>>> for cfg in list_of_system_configs
>>> ...
>>> delimiter = f'{heuristic}/{algorithm}/{cfg}'
This means when querying HDF5 output files, the results of each
simulation can be filtered nicely:
>>> store = pd.HDFStore('path/to/output.h5')
>>> # Returns a dataframe of simulation results
>>> store['heuristic_1/algorithm_3/cfg.json']
Examples
--------
Standard simulation with data frame output
>>> env = simpy.environment()
>>> config = Config('path/to/config')
>>> instrument = CustomInstrument()
>>> plan = PlanningModel()
>>> sched = SchedulingModel()
>>> simulation = Simulation(env, config, instrument,plan,sched)
If we want delays in the model:
>>> dm = DelayModel(prob=0.1, dist='normal', dm=DelayModel.DelayDegree.LOW)
>>> simulation = Simulation(
>>> env, config, instrument,plan,sched, delay=dm
>>> )
Running a simulation to completion:
>>> df = simulation.run()
Running a simulation for a specific time period, then resuming:
>>> df = simulation.run(runtime=100)
>>> ### Check current status of simulatiion
>>> df = simulation.resume(until=150)
Raises
------
"""
def __init__(self,
env,
config,
instrument,
planning_model,
planning_algorithm,
scheduling,
delay=None,
timestamp=None,
to_file=False,
hdf5_path=None,
**kwargs):
#: :py:obj:`simpy.Environment` object
self.env = env
if timestamp:
#: :py:obj:`~topsim.core.monitor.Monitor` instance
self.monitor = Monitor(self, timestamp)
self._timestamp = timestamp
else:
sim_start_time = f'{time.time()}'.split('.')[0]
self._timestamp = sim_start_time
self.monitor = Monitor(self, sim_start_time)
# Process necessary config files
self._cfg_path = config #: Configuration path
# Initiaise Actor and Resource objects
cfg = Config(config)
#: :py:obj:`~topsim.core.cluster.Cluster` instance
self.cluster = Cluster(env, cfg)
#: :py:obj:`~topsim.core.buffer.Buffer` instance
self.buffer = Buffer(env, self.cluster, cfg)
planning_algorithm = planning_algorithm
planning_model = planning_model
if not delay:
# TODO Have this approach replicated so we don't specify the
# model outside the simulation.
delay = DelayModel(0.0, "normal", DelayModel.DelayDegree.NONE)
self.planner = Planner(env, planning_algorithm, self.cluster,
planning_model, delay)
scheduling_algorithm = scheduling()
#: :py:obj:`~topsim.core.scheduler.Scheduler` instance
self.scheduler = Scheduler(env, self.buffer, self.cluster,
scheduling_algorithm)
#: User-defined :py:obj:`~topsim.core.instrument.Instrument` instance
self.instrument = instrument(env=self.env,
config=cfg,
planner=self.planner,
scheduler=self.scheduler)
#: :py:obj:`bool` Flag for producing simulation output in a `.pkl`
# file.
self.to_file = to_file
if self.to_file and hdf5_path:
try:
if os.path.exists(hdf5_path):
LOGGER.warning('Output HDF5 path already exists, '
'simulation appended to existing file')
self._hdf5_store = pd.HDFStore(hdf5_path)
self._hdf5_store.close()
except ValueError(
'Check pandas.HDFStore documentation for valid file path'):
raise
elif self.to_file and not hdf5_path:
raise ValueError(
'Attempted to initialise Simulation object that outputs'
'to file without providing file path')
else:
LOGGER.info(
'Simulation output will not be stored directly to file')
if 'delimiters' in kwargs:
#: Used to separate different simulations in HDF5 output
self._delimiters = kwargs['delimiters']
else:
self._delimiters = ''
self.running = False
def start(self, runtime=-1):
"""
Run the simulation, either for the specified runtime, OR until the
exit conditions are reached.
The exit conditions are:
* There are no more observations to process,
* There is nothing left in the Buffer
* The Scheduler is not waiting to allocate machines to resources
* There are not tasks still running on the cluster.
Parameters
----------
runtime : int
Nominiated runtime of the simulation. If the simulation length is
known, pass that as the argument. If not, passing in a negative
value (typically, just -1) will run the simulation until the
exit condition is reached.
Returns
-------
If `to_file` is True:
sim_data_path, task_data_path : str
Path names for the global simulation runtime and the
individual task data output.
If `to_file` is False:
Two pandas.DataFrame objects for global sim runtime and task data.
"""
if self.running:
raise RuntimeError(
"start() has already been called!"
"Use resume() to continue a simulation that is already in "
"progress.")
self.running = True
self.env.process(self.monitor.run())
self.env.process(self.instrument.run())
self.env.process(self.cluster.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.buffer.run())
if runtime > 0:
self.env.run(until=runtime)
else:
while not self.is_finished():
self.env.run(self.env.now + 1)
self.env.run(self.env.now + 1)
LOGGER.info("Simulation Finished @ %s", self.env.now)
if self.to_file and self._hdf5_store is not None:
global_df = self.monitor.df
task_df = self._generate_final_task_data()
self._hdf5_store.open()
self._compose_hdf5_output(global_df, task_df)
self._hdf5_store.close()
else:
return self.monitor.df, self._generate_final_task_data()
def resume(self, until):
"""
Resume a simulation for a period of time.
Useful for testing purposes, as we do not re-initialise the process
calls as we used to in
:py:obj:`~core.topsim.simulation.Simulation.start`
Parameters
----------
until : int
The (non-inclusive) :py:obj:`Simpy.env.now` timestep that we want to
continue to in the simulation
Returns
-------
self.env.now : float
The current time in the simulation
"""
if not self.running:
raise RuntimeError(
"Simulation has not been started! Call start() to initialise "
"the process stack.")
self.env.run(until=until)
def is_finished(self):
"""
Check if simulation is finished based on the following finish
conditions:
* The Instrument is idle (i.e. has no more observations left to run)
* The Cluster is idle (no tasks are running)
* The Buffer is empty (no data sits on the buffer)
* The Schedule is idle (there are no more workflows/tasks queued)
It is only when all of these return True that the simulation is
regarded as finished.
Returns
-------
True if the above requirements are met; False otherwise (i.e. the
simulation is still running).
"""
if (self.buffer.is_empty() and self.cluster.is_idle()
and self.scheduler.is_idle() and self.instrument.is_idle()):
return True
return False
@staticmethod
def _split_monolithic_config(self, json):
return json
def _generate_final_task_data(self):
"""
Generate a final data frame from the cluster's task dataframe output.
Returns
-------
"""
df = self.cluster.finished_task_time_data()
df = df.T
size = len(df)
df['scheduling'] = [self.planner.algorithm for x in range(size)]
df['planning'] = [repr(self.scheduler.algorithm) for x in range(size)]
df['config'] = [self._cfg_path for x in range(size)]
return df
def _compose_hdf5_output(self, global_df, tasks_df):
"""
Given a :py:obj:`pandas.HDFStore()` object, put global simulation,
task specific, and configuration data into HDF5 storage files.
Parameters
----------
global_df : :py:obj:pandas.DataFrame
The global, per-timestep overview of the simulation
tasks_df : :py:obj:pandas.DataFrame
Information on each tasks' execution throughout the simulation.
Returns
-------
"""
if self._timestamp:
ts = f'd{self._timestamp}'
else:
ts = f'd{datetime.datetime.today().strftime("%y_%m_%d_%H_%M_%S")}'
workflows = self._create_config_table(self._cfg_path)
sanitised_path = self._cfg_path.replace(".json", '').split('/')[-1]
final_key = f'{ts}/{self._delimiters}/{sanitised_path}'
self._hdf5_store.put(key=f"{final_key}/sim", value=global_df)
self._hdf5_store.put(key=f'{final_key}/tasks', value=tasks_df)
self._hdf5_store.put(key=f'{final_key}/config', value=workflows)
return self._hdf5_store
def _stringify_json_data(self, path):
"""
From a given file pointer, get a string representation of the data stored
Parameters
----------
fp : file pointer for the opened JSON file
Returns
-------
jstr : String representation of JSON-encoded data
Raises:
"""
try:
with open(path) as fp:
jdict = json.load(fp)
except json.JSONDecodeError:
raise
jstr = json.dumps(jdict) # , indent=2)
return jstr
def _create_config_table(self, path):
"""
From the simulation config files, find the paths for each observation
workflow and produce a table of this information
Parameters
----------
path
Returns
-------
"""
cfg_str = self._stringify_json_data(path)
jdict = json.loads(cfg_str)
pipelines = jdict['instrument']['telescope']['pipelines']
ds = [['simulation_config', path, cfg_str]]
for observation in pipelines:
p = pipelines[observation]['workflow']
p = p.replace('publications', 'archived_results')
wf_str = self._stringify_json_data(p)
tpl = [f'{observation}', p, wf_str]
ds.append(tpl)
df = pd.DataFrame(ds, columns=['entity', 'config_path', 'config_json'])
return df