def test_multi_simulation_data_merge(self):
# global_sim_df = pd.DataFrame()
# global_task_df = pd.DataFrame()
for algorithm in ['heft', 'fcfs']:
env = simpy.Environment()
simulation = Simulation(
env,
CONFIG,
self.instrument,
planning_algorithm=algorithm,
planning_model=SHADOWPlanning(algorithm),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
hdf5_path='test/simulation_data/test_hdf5.h5',
to_file=True,
delimiters=f'{algorithm}')
simulation.start()
self.assertTrue(os.path.exists('test/simulation_data/test_hdf5.h5'))
heft_key = '/dunittest/heft/heft_single_observation_simulation/sim/'
fcfs_key = '/dunittest/fcfs/heft_single_observation_simulation/sim/'
heft_sim = pd.read_hdf('test/simulation_data/test_hdf5.h5',
key=heft_key)
self.assertEqual(120, len(heft_sim))
self.assertEqual(3, heft_sim.iloc[-1]['available_resources'])
def test_run_batchmodel(self):
simulation = Simulation(self.env,
CONFIG,
Telescope,
planning_model=BatchPlanning('batch'),
planning_algorithm='batch',
scheduling=BatchProcessing,
delay=None,
timestamp=f'{cwd}/test/data/output/{0}')
sim, task = simulation.start()
self.assertGreater(len(sim), 0)
def setUp(self) -> None:
env = simpy.Environment()
self.simulation = Simulation(
env,
LARGE_CONFIG,
Telescope,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=f'test/basic-workflow-data/output/{0}')
def __init__(self, machine_configs, task_configs, algorithm, event_file):
self.env = simpy.Environment()
cluster = Cluster()
cluster.add_machines(machine_configs)
task_broker = Broker(self.env, task_configs)
scheduler = Scheduler(self.env, algorithm)
self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
event_file)
def setUp(self) -> None:
self.env = simpy.Environment()
self.simulation = Simulation(
self.env,
BASIC_CONFIG,
Telescope,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=SIM_TIMESTAMP
)
def setUp(self) -> None:
event_file = EVENT_FILE
env = simpy.Environment()
planning_algorithm = 'heft'
scheduling_algorithm = FifoAlgorithm()
self.simulation = Simulation(env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
planning_algorithm,
scheduling_algorithm,
EVENT_FILE,
visualisation=False)
class Episode(object):
def __init__(self, machine_configs, task_configs, algorithm, event_file):
self.env = simpy.Environment()
cluster = Cluster()
cluster.add_machines(machine_configs)
task_broker = Broker(self.env, task_configs)
scheduler = Scheduler(self.env, algorithm)
self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
event_file)
def run(self):
self.simulation.start()
self.env.run()
def run_simulation(cfg, timestamp):
"""
Given the specified, construct the simulation object and run it
"""
env = simpy.Environment()
simulation = Simulation(env=env,
config=cfg,
instrument=Telescope,
planning_algorithm='batch',
planning_model=BatchPlanning('batch'),
scheduling=BatchProcessing,
delay=None,
timestamp=timestamp,
to_file=True)
sim, tasks = simulation.start()
return sim, tasks
class TestSimulationRuntime(unittest.TestCase):
def setUp(self) -> None:
event_file = EVENT_FILE
env = simpy.Environment()
planning_algorithm = 'heft'
scheduling_algorithm = FifoAlgorithm()
self.simulation = Simulation(env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
planning_algorithm,
scheduling_algorithm,
EVENT_FILE,
visualisation=False)
def testLimitedRuntime(self):
self.simulation.start(runtime=60)
def testBasicConfig(self):
simulation = Simulation(self.env,
CONFIG,
self.instrument,
planning_model=SHADOWPlanning('heft'),
planning_algorithm='heft',
scheduling=DynamicAlgorithmFromPlan,
timestamp=self.timestamp)
self.assertTrue(36, simulation.instrument.total_arrays)
def test_simulation_produces_file(self):
ts = f'{datetime.datetime(2021,1,1).strftime("%y_%m_%d_%H_%M_%S")}'
simulation = Simulation(self.env,
CONFIG,
Telescope,
planning_model=SHADOWPlanning('heft'),
planning_algorithm='heft',
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=ts,
to_file=True,
hdf5_path=self.output)
simulation.start(runtime=60)
self.assertTrue(os.path.exists(self.output))
store = pd.HDFStore(self.output)
store[f'd{ts}/standard_simulation/sim']
def testBasicConfig(self):
simulation = Simulation(self.env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
self.planning_algorithm,
self.scheduling_algorithm,
EVENT_FILE,
visualisation=False)
self.assertTrue(36, simulation.telescope.total_arrays)
def setUp(self) -> None:
"""
Basic simulation using a single observation + heft workflow +
homogenous system configuration.
Returns
-------
"""
env = simpy.Environment()
self.instrument = Telescope
self.simulation = Simulation(
env=env,
config=CONFIG,
instrument=self.instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
to_file=True,
hdf5_path='test/simulation_data/test_hdf5.h5',
delimiters=f'test/')
def test_simulation_nofile_option(self):
simulation = Simulation(
self.env,
CONFIG,
self.instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=None,
)
simdf, taskdf = simulation.start()
self.assertEqual(120, len(simdf))
self.env = simpy.Environment()
simulation = Simulation(
self.env,
CONFIG,
self.instrument,
planning_algorithm='fcfs',
planning_model=SHADOWPlanning('fcfs'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=None,
# delimiters=f'test/{algorithm}'
)
simdf, taskdf = simulation.start()
self.assertEqual(128, len(simdf))
class TestSimulationLargeExperiment(unittest.TestCase):
def setUp(self) -> None:
env = simpy.Environment()
self.simulation = Simulation(
env,
LARGE_CONFIG,
Telescope,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=f'test/basic-workflow-data/output/{0}')
def testSimulationRuns(self):
"""
This is a quick test to make sure that we can simulate a large
complex simulation where all the moving pieces exist.
There are a couple of conditions that require us to simply ignore an
allocation; there are circumstances that exist when an observation
provisions resources for ingest at the same timestep as allocations
are made by the scheduler. The observation/ingest is provision first,
but the resources are not 'consumed' (moved from 'available to
unavailable') until after the scheduler has seen that the resources are
available and assigned tasks to them.
This may be fixed in future approaches by using a 'shadow' allocation
scheme, which acts as a way of maintaining more global state with the
actors still 'theoretically' blind. The current approach is useful in
that it prototypes the use of simpy returns in a useful way.
Returns
-------
"""
self.simulation.start()
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Basic simulation, with minimal Observation Plan and dummy observation workflows
"""
import simpy
from topsim.algorithms.scheduling import FifoAlgorithm
from topsim.core.simulation import Simulation
from topsim import common as test_data
workflow_file = 'test/data/daliuge_pipeline_test.json'
event_file = 'sim.trace'
planning_algorithm = 'heft'
# env = simpy.RealtimeEnvironment(factor=0.5, strict=False)
env = simpy.Environment()
tmax = 36 # for starters, we will define telescope configuration as simply number of arrays that exist
salgorithm = FifoAlgorithm()
vis = False
# TODO move things like 'heft' into a 'common' file which has SchedulingAlgorithm.HEFT = 'heft' etc.
simulation = Simulation(env, test_data.telescope_config, tmax,
test_data.machine_config, salgorithm, 'heft',
event_file, vis)
simulation.start(-1)
from topsim.user.schedule.dynamic_plan import DynamicAlgorithmFromPlan
from topsim.user.telescope import Telescope
from topsim.core.simulation import Simulation
from topsim.core.delay import DelayModel
logging.basicConfig(level="WARNING")
LOGGER = logging.getLogger(__name__)
EVENT_FILE = 'simulations/heft_sim/output/heft_sim_delay_low.trace'
CONFIG = 'simulations/heft_sim/input/heft_single_observation_simulation.json'
env = simpy.Environment()
planning_algorithm = 'heft'
scheduling_algorithm = DynamicAlgorithmFromPlan
instrument = Telescope
dm = DelayModel(0.5, 'normal', DelayModel.DelayDegree.LOW)
simulation = Simulation(env=env,
config=CONFIG,
instrument=instrument,
algorithm_map={
'pheft': 'pheft',
'heft': 'heft',
'fifo': DynamicAlgorithmFromPlan
},
event_file=EVENT_FILE,
delay=dm)
simulation.start(-1)
# simulation.resume(200)
# User defined models
from user.telescope import Telescope # Instrument
from user.schedule.dynamic_plan import DynamicAlgorithmFromPlan # Scheduling
from user.schedule.greedy import GreedyAlgorithmFromPlan # Scheduling
from user.schedule.batch_allocation import BatchProcessing
from user.plan.batch_planning import BatchPlanning # Planning
from user.plan.static_planning import SHADOWPlanning
if __name__ == '__main__':
LOGGER.info(f"Running experiment from {RUN_PATH}/{FOLDER_PATH}")
env = simpy.Environment()
instrument = Telescope
# timestamp = f'{time.time()}'.split('.')[0]
simulation = Simulation(env=env,
config=cfg_path,
instrument=instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='skaworkflows_test',
to_file=True,
hdf5_path=f'{RUN_PATH}/{FOLDER_PATH}/results.h5'
# hdf5_path='',
# delimiters=f'test/'
)
simulation.start()
LOGGER.info(f"Experiment finished, exiting script...")
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Basic simulation, with minimal Observation Plan and dummy observation workflows
"""
import simpy
import logging
from topsim.user.schedule.dynamic_plan import DynamicAlgorithmFromPlan
from topsim.user.telescope import Telescope
from topsim.core.simulation import Simulation
logging.basicConfig(level="DEBUG")
LOGGER = logging.getLogger(__name__)
BASIC_CONFIG = 'simulations/basic_sim/input/basic_simulation.json'
EVENT_FILE = 'simulations/basic_sim/output/sim.trace'
# env = simpy.RealtimeEnvironment(factor=0.5, strict=False)
env = simpy.Environment()
event_file = EVENT_FILE
planning_algorithm = 'heft'
scheduling_algorithm = DynamicAlgorithmFromPlan
instrument = Telescope
algorith_map = {'heft': planning_algorithm, 'fifo': scheduling_algorithm}
simulation = Simulation(env, BASIC_CONFIG, instrument, algorith_map,
EVENT_FILE)
simulation.start(-1)
class TestMonitorPandasPickle(unittest.TestCase):
def setUp(self) -> None:
"""
Basic simulation using a single observation + heft workflow +
homogenous system configuration.
Returns
-------
"""
env = simpy.Environment()
self.instrument = Telescope
self.simulation = Simulation(
env=env,
config=CONFIG,
instrument=self.instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
to_file=True,
hdf5_path='test/simulation_data/test_hdf5.h5',
delimiters=f'test/')
def tearDown(self):
output = f'{cwd}/test/simulation_pickles/{0}'
os.remove('test/simulation_data/test_hdf5.h5')
# os.remove(f'{output}-sim.pkl')
# os.remove(f'{output}-tasks.pkl')
def testHDF5GeneratedAfterSimulation(self):
"""
Test that after a simulation, a HDF5 storage file is generated
"""
self.simulation.start()
self.assertTrue(os.path.exists('test/simulation_data/test_hdf5.h5'))
def testHDF5KeysAndDataFramesExist(self):
"""
Ensure that the generated HDF5 contains the correct results in the
keys
"""
def test_multi_simulation_data_merge(self):
# global_sim_df = pd.DataFrame()
# global_task_df = pd.DataFrame()
for algorithm in ['heft', 'fcfs']:
env = simpy.Environment()
simulation = Simulation(
env,
CONFIG,
self.instrument,
planning_algorithm=algorithm,
planning_model=SHADOWPlanning(algorithm),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
hdf5_path='test/simulation_data/test_hdf5.h5',
to_file=True,
delimiters=f'{algorithm}')
simulation.start()
self.assertTrue(os.path.exists('test/simulation_data/test_hdf5.h5'))
heft_key = '/dunittest/heft/heft_single_observation_simulation/sim/'
fcfs_key = '/dunittest/fcfs/heft_single_observation_simulation/sim/'
heft_sim = pd.read_hdf('test/simulation_data/test_hdf5.h5',
key=heft_key)
self.assertEqual(120, len(heft_sim))
self.assertEqual(3, heft_sim.iloc[-1]['available_resources'])
from topsim.core.simulation import Simulation
from topsim.core.delay import DelayModel
from user.telescope import Telescope
from user.scheduling import GreedyAlgorithmFromPlan
logging.basicConfig(level="DEBUG")
LOGGER = logging.getLogger(__name__)
CONFIG = f'visualisation_playground/series_observations.json'
env = simpy.Environment()
dm = DelayModel(0.20, 'normal', DelayModel.DelayDegree.HIGH)
instrument = Telescope
timestamp = f'{time.time()}'.split('.')[0]
cfg = CONFIG.split('/')[-1]
simulation = Simulation(
env=env,
config=CONFIG,
instrument=instrument,
algorithm_map={
'heft': 'heft',
'fifo': GreedyAlgorithmFromPlan
},
delay=dm,
timestamp=f'visualisation_playground/results/{cfg}-{timestamp}')
# LOGGER.info("Simulation Starting")
simulation.start(-1)
# simulation.resume(10000)
from topsim.core.simulation import Simulation
from topsim.algorithms.scheduling import FifoAlgorithm
# We need a number of configuration files for the basic set-up
TELESCOPE_CONFIG = 'test/data/config/observations.json'
CLUSTER_CONFIG = 'test/data/config/basic_spec-10.json'
BUFFER_CONFIG = 'test/data/config/buffer.json'
EVENT_FILE = 'recipes/output/sim.trace'
# Planning Algorithm - needs to be in our external library
planning_algorithm = 'heft'
# Scheduling Algorithm - these are implemented in a specific format to work
# in conjunction with the Scheduler Actor
scheduling_algorithm = FifoAlgorithm()
# Inititalise Simpy environment
env = simpy.Environment()
simulation = Simulation(env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
planning_algorithm,
scheduling_algorithm,
EVENT_FILE,
visualisation=False)
# RUNNING SIMULATION AND GENERATING DATA
for algorithm in algorithms:
for config in sorted(os.listdir(
'publications/2021_isc-hpc/config/single_size/40cluster')):
if '.json' in config:
CONFIG = f'publications/2021_isc-hpc/config/single_size/40cluster/{config}'
env = simpy.Environment()
instrument = Telescope
timestamp = f'{time.time()}'.split('.')[0]
simulation = Simulation(
env=env,
config=CONFIG,
instrument=instrument,
planning=algorithm,
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp={timestamp}
)
sim, tasks = simulation.start()
global_sim = global_sim.append(sim)
global_tasks = global_tasks.append(tasks)
print(algorithm, config, len(sim))
global_tasks.to_pickle('tasks_output.pkl')
global_sim.to_pickle('simulation_output.pkl')
# PLOTTING SIMULATION DATA - originally produced in a Jupyter Notebook
# Group by planning, delay, and config to get the simulation time for each
# simulation.
class TestBasicIngest(unittest.TestCase):
def setUp(self) -> None:
self.env = simpy.Environment()
self.simulation = Simulation(
self.env,
BASIC_CONFIG,
Telescope,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=SIM_TIMESTAMP
)
# def tearDown(self):
# output = 'test/basic-workflow-data/output/{0}'
# os.remove(f'{output}-sim.pkl')
# os.remove(f'{output}-tasks.pkl')
def testClusterIngest(self):
"""
The basic ingest represents the edge cases for timing and scheduling
within the simulation, as demonstrated in this test.
There are a couple of edge cases that occur here, especially when we
consider that we have only 2 resources; one of these will be taken by
ingest, meaning that we cannot start an observation until 1 timestep
AFTER an ingest has finished, because the telescope will check before
that task is successfully removed from the cluster.
This is why we run for 6 seconds and only process 2 observations.
After we've observed 2 observations, we reach capacity on the
cold-buffer so we are unable to observe any more.
Returns
-------
"""
self.assertEqual(0, self.env.now)
self.simulation.start(runtime=7)
self.assertEqual(
2, self.simulation.cluster._ingest['completed']
)
self.assertEqual(
RunStatus.FINISHED,
self.simulation.instrument.observations[1].status
)
def testBufferIngest(self):
self.assertEqual(0, self.simulation.env.now)
self.simulation.start(runtime=1)
self.assertEqual(
5, self.simulation.buffer.hot[0].current_capacity
)
self.simulation.resume(until=2)
self.assertEqual(
10, self.simulation.buffer.hot[0].current_capacity
)
self.assertEqual(
5, self.simulation.buffer.cold[0].current_capacity
)
self.assertEqual(
1,
len(self.simulation.buffer.cold[0].observations["stored"])
)
self.simulation.resume(until=4)
self.assertEqual(10, self.simulation.buffer.hot[0].current_capacity)
self.assertEqual(0, self.simulation.buffer.cold[0].current_capacity)
self.assertEqual(
2,
len(self.simulation.buffer.cold[0].observations["stored"])
)
def testSchedulerRunTime(self):
self.assertEqual(0, self.simulation.env.now)
self.simulation.start(runtime=2)
self.assertEqual(
1, len(self.simulation.buffer.cold[0].observations['stored'])
)
self.simulation.resume(until=8)
self.simulation.resume(until=11)
self.simulation.resume(until=12)
# self.assertEqual(0, len(self.simulation.cluster.tasks['running']))
# We've finished processing one of the workflows so one observation
# is finished.
self.assertEqual(
2, len(self.simulation.buffer.cold[0].observations['stored'])
)