class TestSchedulerIntegration(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_FIFO_with_buffer(self):
"""
Demonstrate that the scheduler accurately schedules when we have
other Actors working in tandem.
Expectations:
- After 1 timestep in the simualtion, we have 5 resources
available of the 10 that we start with.
-
Returns
-------
"""
self.env.run(until=1)
self.assertEqual(10, len(self.cluster._resources['available']))
# This takes timestep, data in the HotBuffer should be 4
self.env.run(until=2)
self.assertEqual(5, len(self.cluster._resources['available']))
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
# self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
self.env.run(until=32)
# Ensure the time
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
# 30 timesteps until we finish everything + 81 timesteps to complete
# workflow plan.
self.env.run(until=124)
# As we have been processing the current observation, we are also
# ingestting the next one.
self.assertEqual(250, self.buffer.cold[0].current_capacity)
class TestIngest(unittest.TestCase): def setUp(self) -> None: self.env = simpy.Environment() self.cluster = Cluster(env=self.env, spec=CLUSTER_CONFIG) self.observation = Observation( 'planner_observation', OBS_START_TME, OBS_DURATION, OBS_DEMAND, OBS_WORKFLOW, type=None, data_rate=None ) def testClusterCheckIngest(self): retval = self.cluster.check_ingest_capacity(pipeline_demand=5) self.assertTrue(retval) def testClusterProvisionIngest(self): duration = self.observation.duration pipeline_demand = 5 self.env.process(self.cluster.run()) peek = self.env.peek() self.env.process(self.cluster.provision_ingest_resources( pipeline_demand, duration) ) self.env.run(until=1) self.assertEqual(1, self.env.now) # self.process(self.run_ingest(duration,pipeline_demand)) # for task in self.cluster.running_tasks: # self.assertEqual(TaskStatus.RUNNING, task.task_status) self.assertEqual(5, len(self.cluster.available_resources)) self.assertEqual(5, len(self.cluster.running_tasks)) self.env.run(until=10) self.assertEqual(5, len(self.cluster.available_resources)) self.env.run(until=20) self.assertEqual(10,len(self.cluster.available_resources)) self.assertEqual(20,self.env.now) def run_ingest(self, duration,demand): retval = self.cluster.provision_ingest_resources( demand, duration ) for task in self.cluster.running_tasks: self.assertEqual(TaskStatus.SCHEDULED, task.task_status)
class TestTelescopeIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
self.config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=self.config)
self.buffer = Buffer(env=self.env,
cluster=self.cluster,
config=self.config)
self.scheduler = Scheduler(env=self.env,
buffer=self.buffer,
cluster=self.cluster,
algorithm=None)
self.planner = Planner(self.env, 'heft', self.cluster,
SHADOWPlanning('heft'))
def testIngest(self):
telescope = Telescope(env=self.env,
config=self.config,
planner=self.planner,
scheduler=self.scheduler)
self.assertEqual(0, telescope.telescope_use)
self.env.process(telescope.run())
self.env.process(self.cluster.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.buffer.run())
self.env.run(until=2)
self.assertEqual(36, telescope.telescope_use)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=11)
self.assertEqual(len([self.buffer.hot[0].observations["transfer"]]), 1)
self.assertEqual(462, self.buffer.hot[0].current_capacity)
self.assertEqual(248, self.buffer.cold[0].current_capacity)
self.env.run(until=12)
self.assertEqual(0, telescope.telescope_use)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
class TestSchedulerDelayHelpers(unittest.TestCase):
def setUp(self):
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.planner = Planner(self.env,
PLANNING_ALGORITHM,
self.cluster,
SHADOWPlanning('heft'),
delay_model=DelayModel(0.3, "normal"))
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan())
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_propogate_delay_returns_updated_workflow(self):
"""
class TestSchedulerIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
self.scheduler = Scheduler(self.env, self.buffer, self.cluster,
DynamicAlgorithmFromPlan)
self.planner = Planner(self.env, PLANNING_ALGORITHM, self.cluster,
SHADOWPlanning('heft'))
# planner = None
self.telescope = Telescope(self.env, config, self.planner,
self.scheduler)
def testSchdulerCheckIngestReady(self):
"""
Check the return status of check_ingest_capacity is correct
"""
pipelines = self.telescope.pipelines
observation = self.telescope.observations[0]
max_ingest = self.telescope.max_ingest
# There should be capacity
self.assertEqual(0.0, self.env.now)
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertTrue(ret)
# Let's remove capacity to check it returns false
tmp = self.cluster._resources['available']
self.cluster._resources['available'] = self.cluster._resources[
'available'][:3]
ret = self.scheduler.check_ingest_capacity(observation, pipelines,
max_ingest)
self.assertFalse(ret)
self.cluster._resources['available'] = tmp
self.assertEqual(10, len(self.cluster._resources['available']))
def testSchedulerProvisionsIngest(self):
"""
Ensure that the scheduler correcly coordinates ingest onto the Cluster
and into the Buffer
Returns
-------
"""
pipelines = self.telescope.pipelines
max_ingest = self.telescope.max_ingest
observation = self.telescope.observations[0]
ready_status = self.scheduler.check_ingest_capacity(
observation, pipelines, max_ingest)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
observation.status = RunStatus.WAITING
status = self.env.process(
self.scheduler.allocate_ingest(observation, pipelines,
self.planner))
self.env.run(until=1)
self.assertEqual(5, len(self.cluster._resources['available']))
# After 1 timestep, data in the HotBuffer should be 2
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=30)
self.assertEqual(10, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.assertEqual(210, self.buffer.cold[0].current_capacity)
class TestDelaysInActors(unittest.TestCase):
def setUp(self):
"""
Repeating above test cases but with delays to determine that delay
flags reach us.
Returns
-------
"""
self.env = simpy.Environment()
config = Config(INTEGRATION)
self.cluster = Cluster(self.env, config)
self.buffer = Buffer(self.env, self.cluster, config)
dm = DelayModel(0.9, "normal",
DelayModel.DelayDegree.HIGH)
self.planner = Planner(
self.env, PLANNING_ALGORITHM,
self.cluster, SHADOWPlanning('heft',delay_model=dm), delay_model=dm
)
self.scheduler = Scheduler(
self.env, self.buffer, self.cluster, DynamicAlgorithmFromPlan()
)
self.telescope = Telescope(
self.env, config, self.planner, self.scheduler
)
self.env.process(self.cluster.run())
self.env.process(self.buffer.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.telescope.run())
def test_scheduler_delay_detection(self):
"""
Nothing should change until we reach the workflow plan, as we are
testing TaskDelays
Returns
-------
"""
self.env.run(until=1)
# Remember - env starts at 0, we don't start until 1.
self.assertEqual(10, len(self.cluster._resources['available']))
self.env.run(until=2)
# After 1 timestep, data in the HotBuffer should be 4
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.env.run(until=44)
# We know that the schedule has been delayed - however, we don't
# report this to the telescope until we know how long we are delayed
# (that is, until the task has completely finished its duration).
# In this instance. we know that the first task is going to be
# delayed, and so wait until it's completed execution to trigger a
# delay.
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
self.env.run(until=45)
self.assertTrue(ScheduleStatus.DELAYED,self.scheduler.schedule_status)
self.env.run(until=124)
# Assert that we still have tasks running
# self.assertLess(
# 0, len(self.cluster.clusters['default']['tasks']['running'])
# )
self.assertNotEqual(250, self.buffer.cold[0].current_capacity)
def test_telescope_delay_detection(self):
"""
Returns
-------
"""
self.env.run(until=1)
# Remember - env starts at 0, we don't start until 1.
self.assertEqual(10, len(self.cluster._resources['available']))
self.env.run(until=2)
# After 1 timestep, data in the HotBuffer should be 4
self.assertEqual(496, self.buffer.hot[0].current_capacity)
self.env.run(until=31)
self.assertEqual(5, len(self.cluster._tasks['finished']))
self.assertEqual(500, self.buffer.hot[0].current_capacity)
self.env.run(until=32)
# Ensure the time
self.assertEqual(ScheduleStatus.ONTIME, self.scheduler.schedule_status)
self.env.run(until=100)
self.assertEqual(ScheduleStatus.DELAYED,self.scheduler.schedule_status)
self.assertTrue(self.telescope.delayed)
def test_telescope_delay_greedy_decision(self):
"""
class TestIngest(unittest.TestCase):
def setUp(self) -> None:
"""
Setup a cluster and a simulation environment to test ingest pipelines
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
def testClusterCheckIngest(self):
"""
Need to determine that the cluster is able to deal with the pipeline
requirement
in addition, need to check that we are not going over the maximum
number of resources that can be feasibly allocated to ingest.
Returns
-------
"""
max_ingest = 5
retval = self.cluster.check_ingest_capacity(5, max_ingest)
self.assertTrue(retval)
def testClusterProvisionIngest(self):
"""
Notes:
When we run this test, we have the simulation run 'until time 10'.
AS Far as the simulation is concerned, we this occurs 'before' 10
seconds in simtime; however, the rest of the simulation is
operating AT time = 10, which is when the tasks are "Due" to
finish. Here we are running the tasks for "1 -timestep less" than
is necessary, but within the simulation time this allows us to run
"on time". I.e. at time = 10 (in the simulation), there _will_ be
10 resources available, because it's the time period in which all
ingest tasks will finish. However, in real terms, these tasks
technically finished one timestep earlier.
Returns
-------
"""
pipeline_demand = 5
self.env.process(self.cluster.run())
peek = self.env.peek()
self.env.process(
self.cluster.provision_ingest_resources(pipeline_demand,
self.observation))
self.env.run(until=1)
self.assertEqual(1, self.env.now)
self.assertEqual(5, len(self.cluster._resources['available']))
self.assertEqual(5, len(self.cluster._tasks['running']))
self.assertEqual(5, len(self.cluster._resources['available']))
self.env.run(until=11)
self.assertEqual(10, len(self.cluster._resources['available']))
def test_ingest_capacity_check(self):
"""
Given a pipeline demand that is too great, return 'false' to ensure
that we do not attempt to run observations.
"""
pipeline_demand = 11
max_ingest = 5
self.assertFalse(
self.cluster.check_ingest_capacity(pipeline_demand, max_ingest))
def test_ingest_demand_exceeds_cluster_capacity_runtime(self):
"""
If we have a pipeline that requests too many ingest pipelines then we
need to reject its observation. If - for some unknown reasons - we
do not call this check, make sure the simulation fails at runtime.
Returns
-------
"""
pipeline_demand = 11
self.env.process(
self.cluster.provision_ingest_resources(pipeline_demand,
self.observation))
self.assertRaises(RuntimeError, self.env.run, until=1)
class Simulation:
"""
The Simulation class is a wrapper for all Actors; we start the simulation
through the simulation class, which in turn invokes the initial Actors and
monitoring, and provides the conditions for checking if the simulation has
finished.
Parameters
----------
env : :py:obj:`simpy.Environment` object
The discrete-event simulation environment. This is the way TOpSim
simulation maintains state across the different actors,
and interfaces with the simpy processes.
config : str
Path to the simulation JSOn configuration file
instrument : :py:obj:`~topsim.core.instrument.Instrument`
User-defined implementation of the Instrument class.
planning_model : :py:obj:`~topsim.algorithms.planning.Planning` object
User-defined implementation of the planning algorithm class
planning_algorithm: str
Reference to the specific algorithm implementated in `planning_model`
scheduling: :py:obj:`~topsim.algorithms.scheduling.Algorithm`
User-defined implementation of the scheduling algorithm
:py:obj:`abc.ABC`.
delay: :py:obj:`~topsim.core.delay.DelayModel`, optional
for the simulation.
timestamp: str, optional
Optional Simulation start-time; this is useful for testing, to ensure we
name the file and the tests match up. Also useful if you do not want to
use the time of the simulation as the name.
to_file : bool, optional
`True` if the simulation is to be written to a Pandas `pkl` file;
`False` will return pandas DataFrame objects at the completion of the
:py:meth:`~topsim.core.simulation.Simulation.run` function.
Notes
-----
If to_file left as `False`, simulation results and output will be returned
as Pandas DataFrames (see
:py:meth:`~topsim.core.simulation.Simulation.run`) . This is designed for
running multiple simulations, allowing for the appending of individual
simulation results to a 'global' :py:obj:`~pandas.DataFrame` . Current
support for output is limited to Panda's `.pkl` files.
Parsing in the option `delimiters` provides a way of differentiating
between multiple simulations within a single HDF5 store (for example,
in an experiment). A typical experimental loop may involve the following
structure:
>>> for heuristic in list_of_scheduling_heuristics
>>> for algorithm in list_of_planning_algorithms
>>> for cfg in list_of_system_configs
>>> ...
>>> delimiter = f'{heuristic}/{algorithm}/{cfg}'
This means when querying HDF5 output files, the results of each
simulation can be filtered nicely:
>>> store = pd.HDFStore('path/to/output.h5')
>>> # Returns a dataframe of simulation results
>>> store['heuristic_1/algorithm_3/cfg.json']
Examples
--------
Standard simulation with data frame output
>>> env = simpy.environment()
>>> config = Config('path/to/config')
>>> instrument = CustomInstrument()
>>> plan = PlanningModel()
>>> sched = SchedulingModel()
>>> simulation = Simulation(env, config, instrument,plan,sched)
If we want delays in the model:
>>> dm = DelayModel(prob=0.1, dist='normal', dm=DelayModel.DelayDegree.LOW)
>>> simulation = Simulation(
>>> env, config, instrument,plan,sched, delay=dm
>>> )
Running a simulation to completion:
>>> df = simulation.run()
Running a simulation for a specific time period, then resuming:
>>> df = simulation.run(runtime=100)
>>> ### Check current status of simulatiion
>>> df = simulation.resume(until=150)
Raises
------
"""
def __init__(self,
env,
config,
instrument,
planning_model,
planning_algorithm,
scheduling,
delay=None,
timestamp=None,
to_file=False,
hdf5_path=None,
**kwargs):
#: :py:obj:`simpy.Environment` object
self.env = env
if timestamp:
#: :py:obj:`~topsim.core.monitor.Monitor` instance
self.monitor = Monitor(self, timestamp)
self._timestamp = timestamp
else:
sim_start_time = f'{time.time()}'.split('.')[0]
self._timestamp = sim_start_time
self.monitor = Monitor(self, sim_start_time)
# Process necessary config files
self._cfg_path = config #: Configuration path
# Initiaise Actor and Resource objects
cfg = Config(config)
#: :py:obj:`~topsim.core.cluster.Cluster` instance
self.cluster = Cluster(env, cfg)
#: :py:obj:`~topsim.core.buffer.Buffer` instance
self.buffer = Buffer(env, self.cluster, cfg)
planning_algorithm = planning_algorithm
planning_model = planning_model
if not delay:
# TODO Have this approach replicated so we don't specify the
# model outside the simulation.
delay = DelayModel(0.0, "normal", DelayModel.DelayDegree.NONE)
self.planner = Planner(env, planning_algorithm, self.cluster,
planning_model, delay)
scheduling_algorithm = scheduling()
#: :py:obj:`~topsim.core.scheduler.Scheduler` instance
self.scheduler = Scheduler(env, self.buffer, self.cluster,
scheduling_algorithm)
#: User-defined :py:obj:`~topsim.core.instrument.Instrument` instance
self.instrument = instrument(env=self.env,
config=cfg,
planner=self.planner,
scheduler=self.scheduler)
#: :py:obj:`bool` Flag for producing simulation output in a `.pkl`
# file.
self.to_file = to_file
if self.to_file and hdf5_path:
try:
if os.path.exists(hdf5_path):
LOGGER.warning('Output HDF5 path already exists, '
'simulation appended to existing file')
self._hdf5_store = pd.HDFStore(hdf5_path)
self._hdf5_store.close()
except ValueError(
'Check pandas.HDFStore documentation for valid file path'):
raise
elif self.to_file and not hdf5_path:
raise ValueError(
'Attempted to initialise Simulation object that outputs'
'to file without providing file path')
else:
LOGGER.info(
'Simulation output will not be stored directly to file')
if 'delimiters' in kwargs:
#: Used to separate different simulations in HDF5 output
self._delimiters = kwargs['delimiters']
else:
self._delimiters = ''
self.running = False
def start(self, runtime=-1):
"""
Run the simulation, either for the specified runtime, OR until the
exit conditions are reached.
The exit conditions are:
* There are no more observations to process,
* There is nothing left in the Buffer
* The Scheduler is not waiting to allocate machines to resources
* There are not tasks still running on the cluster.
Parameters
----------
runtime : int
Nominiated runtime of the simulation. If the simulation length is
known, pass that as the argument. If not, passing in a negative
value (typically, just -1) will run the simulation until the
exit condition is reached.
Returns
-------
If `to_file` is True:
sim_data_path, task_data_path : str
Path names for the global simulation runtime and the
individual task data output.
If `to_file` is False:
Two pandas.DataFrame objects for global sim runtime and task data.
"""
if self.running:
raise RuntimeError(
"start() has already been called!"
"Use resume() to continue a simulation that is already in "
"progress.")
self.running = True
self.env.process(self.monitor.run())
self.env.process(self.instrument.run())
self.env.process(self.cluster.run())
self.scheduler.start()
self.env.process(self.scheduler.run())
self.env.process(self.buffer.run())
if runtime > 0:
self.env.run(until=runtime)
else:
while not self.is_finished():
self.env.run(self.env.now + 1)
self.env.run(self.env.now + 1)
LOGGER.info("Simulation Finished @ %s", self.env.now)
if self.to_file and self._hdf5_store is not None:
global_df = self.monitor.df
task_df = self._generate_final_task_data()
self._hdf5_store.open()
self._compose_hdf5_output(global_df, task_df)
self._hdf5_store.close()
else:
return self.monitor.df, self._generate_final_task_data()
def resume(self, until):
"""
Resume a simulation for a period of time.
Useful for testing purposes, as we do not re-initialise the process
calls as we used to in
:py:obj:`~core.topsim.simulation.Simulation.start`
Parameters
----------
until : int
The (non-inclusive) :py:obj:`Simpy.env.now` timestep that we want to
continue to in the simulation
Returns
-------
self.env.now : float
The current time in the simulation
"""
if not self.running:
raise RuntimeError(
"Simulation has not been started! Call start() to initialise "
"the process stack.")
self.env.run(until=until)
def is_finished(self):
"""
Check if simulation is finished based on the following finish
conditions:
* The Instrument is idle (i.e. has no more observations left to run)
* The Cluster is idle (no tasks are running)
* The Buffer is empty (no data sits on the buffer)
* The Schedule is idle (there are no more workflows/tasks queued)
It is only when all of these return True that the simulation is
regarded as finished.
Returns
-------
True if the above requirements are met; False otherwise (i.e. the
simulation is still running).
"""
if (self.buffer.is_empty() and self.cluster.is_idle()
and self.scheduler.is_idle() and self.instrument.is_idle()):
return True
return False
@staticmethod
def _split_monolithic_config(self, json):
return json
def _generate_final_task_data(self):
"""
Generate a final data frame from the cluster's task dataframe output.
Returns
-------
"""
df = self.cluster.finished_task_time_data()
df = df.T
size = len(df)
df['scheduling'] = [self.planner.algorithm for x in range(size)]
df['planning'] = [repr(self.scheduler.algorithm) for x in range(size)]
df['config'] = [self._cfg_path for x in range(size)]
return df
def _compose_hdf5_output(self, global_df, tasks_df):
"""
Given a :py:obj:`pandas.HDFStore()` object, put global simulation,
task specific, and configuration data into HDF5 storage files.
Parameters
----------
global_df : :py:obj:pandas.DataFrame
The global, per-timestep overview of the simulation
tasks_df : :py:obj:pandas.DataFrame
Information on each tasks' execution throughout the simulation.
Returns
-------
"""
if self._timestamp:
ts = f'd{self._timestamp}'
else:
ts = f'd{datetime.datetime.today().strftime("%y_%m_%d_%H_%M_%S")}'
workflows = self._create_config_table(self._cfg_path)
sanitised_path = self._cfg_path.replace(".json", '').split('/')[-1]
final_key = f'{ts}/{self._delimiters}/{sanitised_path}'
self._hdf5_store.put(key=f"{final_key}/sim", value=global_df)
self._hdf5_store.put(key=f'{final_key}/tasks', value=tasks_df)
self._hdf5_store.put(key=f'{final_key}/config', value=workflows)
return self._hdf5_store
def _stringify_json_data(self, path):
"""
From a given file pointer, get a string representation of the data stored
Parameters
----------
fp : file pointer for the opened JSON file
Returns
-------
jstr : String representation of JSON-encoded data
Raises:
"""
try:
with open(path) as fp:
jdict = json.load(fp)
except json.JSONDecodeError:
raise
jstr = json.dumps(jdict) # , indent=2)
return jstr
def _create_config_table(self, path):
"""
From the simulation config files, find the paths for each observation
workflow and produce a table of this information
Parameters
----------
path
Returns
-------
"""
cfg_str = self._stringify_json_data(path)
jdict = json.loads(cfg_str)
pipelines = jdict['instrument']['telescope']['pipelines']
ds = [['simulation_config', path, cfg_str]]
for observation in pipelines:
p = pipelines[observation]['workflow']
p = p.replace('publications', 'archived_results')
wf_str = self._stringify_json_data(p)
tpl = [f'{observation}', p, wf_str]
ds.append(tpl)
df = pd.DataFrame(ds, columns=['entity', 'config_path', 'config_json'])
return df
class Simulation(object):
"""
The Simulation class is a wrapper for all Actors; we start the simulation
through the simulation class, which in turn invokes the initial Actors and
monitoring, and provides the conditions for checking if the simulation has
finished.
Parameters
----------
env : simpy.Environment bject
This is how the TOpSim simulation maintains state across the
different actors, and interfaces with the simpy processes.
telescope_config: str
This is a path to the telescope config that follows the TOpSim config
specification (JSON). This file will be parsed in the Telescope class
constructure
cluster_config: str
Path to the HPC cluster config that forms the computing component of
the SDP
buffer_config: str
Path to the buffer configuration
planning_algorithm: Object
instance of the planning algorithm class interface as defined in
algorithms.examples/
scheduling_algorithm: object
instance of the core.algorithm interface
event_file: str
Path to the output file that stores execution of simulation.
visualisation: bool
If visualisation is required, True; else, False
Methods
-------
Raises
------
"""
def __init__(
self,
env,
telescope_config,
cluster_config,
buffer_config,
planning_algorithm,
scheduling_algorithm,
event_file,
visualisation=False
):
self.env = env
# Event file setup
self.event_file = event_file
self.visualisation = visualisation
if event_file is not None:
self.monitor = Monitor(self)
if visualisation:
self.visualiser = Visualiser(self)
# Process necessary config files
# Initiaise Actor and Resource objects
self.cluster = Cluster(env, cluster_config)
self.buffer = Buffer(env, self.cluster, config=buffer_config)
self.planner = Planner(env, planning_algorithm, cluster_config)
self.scheduler = Scheduler(
env, self.buffer, self.cluster, scheduling_algorithm
)
self.telescope = Telescope(
env=self.env,
config=telescope_config,
planner=self.planner,
scheduler=self.scheduler
)
def start(self, runtime=150):
"""
Run the simulation, either for the specified runtime, OR until the
exit condition is reached:
* There are no more observations to process,
* There is nothing left in the Buffer
* The Scheduler is not waiting to allocate machines to resources
* There are not tasks still running on the cluster.
Parameters
----------
runtime : int
Nominiated runtime of the simulation. If the simulation length is
known, pass that as the argument. If not, passing in a negative
value (typically, just -1) will run the simulation until the
exit condition is reached.
Returns
-------
"""
if self.event_file is not None:
self.env.process(self.monitor.run())
if self.visualisation:
self.env.process(self.visualiser.run())
self.env.process(self.telescope.run())
self.env.process(self.cluster.run())
self.scheduler.init()
self.env.process(self.scheduler.run())
# Calling env.run() invokes the processes passed in init_process()
if runtime > 0:
self.env.run(until=runtime)
else:
if not self.is_finished():
self.env.run()
logger.info("Simulation Finished @ %s", self.env.now)
def is_finished(self):
status = (
len(self.telescope.observations) == 0
and self.buffer.observations_for_processing.empty()
and len(self.scheduler.waiting_observations) == 0
and len(self.cluster.running_tasks) == 0
)
if status:
# Using compound 'or' doesn't give us a True/False
return True
else:
return False