def test_simulation_nofile_option(self):
simulation = Simulation(
self.env,
CONFIG,
self.instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=None,
)
simdf, taskdf = simulation.start()
self.assertEqual(120, len(simdf))
self.env = simpy.Environment()
simulation = Simulation(
self.env,
CONFIG,
self.instrument,
planning_algorithm='fcfs',
planning_model=SHADOWPlanning('fcfs'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=None,
# delimiters=f'test/{algorithm}'
)
simdf, taskdf = simulation.start()
self.assertEqual(128, len(simdf))
def test_multi_simulation_data_merge(self):
# global_sim_df = pd.DataFrame()
# global_task_df = pd.DataFrame()
for algorithm in ['heft', 'fcfs']:
env = simpy.Environment()
simulation = Simulation(
env,
CONFIG,
self.instrument,
planning_algorithm=algorithm,
planning_model=SHADOWPlanning(algorithm),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
hdf5_path='test/simulation_data/test_hdf5.h5',
to_file=True,
delimiters=f'{algorithm}')
simulation.start()
self.assertTrue(os.path.exists('test/simulation_data/test_hdf5.h5'))
heft_key = '/dunittest/heft/heft_single_observation_simulation/sim/'
fcfs_key = '/dunittest/fcfs/heft_single_observation_simulation/sim/'
heft_sim = pd.read_hdf('test/simulation_data/test_hdf5.h5',
key=heft_key)
self.assertEqual(120, len(heft_sim))
self.assertEqual(3, heft_sim.iloc[-1]['available_resources'])
def testBasicConfig(self):
simulation = Simulation(self.env,
CONFIG,
self.instrument,
planning_model=SHADOWPlanning('heft'),
planning_algorithm='heft',
scheduling=DynamicAlgorithmFromPlan,
timestamp=self.timestamp)
self.assertTrue(36, simulation.instrument.total_arrays)
def setUp(self) -> None:
env = simpy.Environment()
self.simulation = Simulation(
env,
BASIC_CONFIG,
Telescope,
planning='heft',
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=f'test/basic-workflow-data/output/{0}')
def testBasicConfig(self):
simulation = Simulation(self.env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
self.planning_algorithm,
self.scheduling_algorithm,
EVENT_FILE,
visualisation=False)
self.assertTrue(36, simulation.telescope.total_arrays)
def __init__(self, machine_configs, task_configs, algorithm, event_file):
self.env = simpy.Environment()
cluster = Cluster()
cluster.add_machines(machine_configs)
task_broker = Broker(self.env, task_configs)
scheduler = Scheduler(self.env, algorithm)
self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
event_file)
def test_run_batchmodel(self):
simulation = Simulation(self.env,
CONFIG,
Telescope,
planning_model=BatchPlanning('batch'),
planning_algorithm='batch',
scheduling=BatchProcessing,
delay=None,
timestamp=f'{cwd}/test/data/output/{0}')
sim, task = simulation.start()
self.assertGreater(len(sim), 0)
def setUp(self) -> None:
self.env = simpy.Environment()
self.simulation = Simulation(
self.env,
BASIC_CONFIG,
Telescope,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=SIM_TIMESTAMP
)
def setUp(self) -> None:
event_file = EVENT_FILE
env = simpy.Environment()
planning_algorithm = 'heft'
scheduling_algorithm = FifoAlgorithm()
self.simulation = Simulation(env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
planning_algorithm,
scheduling_algorithm,
EVENT_FILE,
visualisation=False)
def run_simulation(cfg, timestamp):
"""
Given the specified, construct the simulation object and run it
"""
env = simpy.Environment()
simulation = Simulation(env=env,
config=cfg,
instrument=Telescope,
planning_algorithm='batch',
planning_model=BatchPlanning('batch'),
scheduling=BatchProcessing,
delay=None,
timestamp=timestamp,
to_file=True)
sim, tasks = simulation.start()
return sim, tasks
def test_simulation_produces_file(self):
ts = f'{datetime.datetime(2021,1,1).strftime("%y_%m_%d_%H_%M_%S")}'
simulation = Simulation(self.env,
CONFIG,
Telescope,
planning_model=SHADOWPlanning('heft'),
planning_algorithm='heft',
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp=ts,
to_file=True,
hdf5_path=self.output)
simulation.start(runtime=60)
self.assertTrue(os.path.exists(self.output))
store = pd.HDFStore(self.output)
store[f'd{ts}/standard_simulation/sim']
def setUp(self) -> None:
"""
Basic simulation using a single observation + heft workflow +
homogenous system configuration.
Returns
-------
"""
env = simpy.Environment()
self.instrument = Telescope
self.simulation = Simulation(
env=env,
config=CONFIG,
instrument=self.instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='unittest',
to_file=True,
hdf5_path='test/simulation_data/test_hdf5.h5',
delimiters=f'test/')
from topsim.core.simulation import Simulation
from topsim.core.delay import DelayModel
from user.telescope import Telescope
from user.scheduling import GreedyAlgorithmFromPlan
logging.basicConfig(level="DEBUG")
LOGGER = logging.getLogger(__name__)
CONFIG = f'visualisation_playground/series_observations.json'
env = simpy.Environment()
dm = DelayModel(0.20, 'normal', DelayModel.DelayDegree.HIGH)
instrument = Telescope
timestamp = f'{time.time()}'.split('.')[0]
cfg = CONFIG.split('/')[-1]
simulation = Simulation(
env=env,
config=CONFIG,
instrument=instrument,
algorithm_map={
'heft': 'heft',
'fifo': GreedyAlgorithmFromPlan
},
delay=dm,
timestamp=f'visualisation_playground/results/{cfg}-{timestamp}')
# LOGGER.info("Simulation Starting")
simulation.start(-1)
# simulation.resume(10000)
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Basic simulation, with minimal Observation Plan and dummy observation workflows
"""
import simpy
from topsim.algorithms.scheduling import FifoAlgorithm
from topsim.core.simulation import Simulation
from topsim import common as test_data
workflow_file = 'test/data/daliuge_pipeline_test.json'
event_file = 'sim.trace'
planning_algorithm = 'heft'
# env = simpy.RealtimeEnvironment(factor=0.5, strict=False)
env = simpy.Environment()
tmax = 36 # for starters, we will define telescope configuration as simply number of arrays that exist
salgorithm = FifoAlgorithm()
vis = False
# TODO move things like 'heft' into a 'common' file which has SchedulingAlgorithm.HEFT = 'heft' etc.
simulation = Simulation(env, test_data.telescope_config, tmax,
test_data.machine_config, salgorithm, 'heft',
event_file, vis)
simulation.start(-1)
from topsim.user.schedule.dynamic_plan import DynamicAlgorithmFromPlan
from topsim.user.telescope import Telescope
from topsim.core.simulation import Simulation
from topsim.core.delay import DelayModel
logging.basicConfig(level="WARNING")
LOGGER = logging.getLogger(__name__)
EVENT_FILE = 'simulations/heft_sim/output/heft_sim_delay_low.trace'
CONFIG = 'simulations/heft_sim/input/heft_single_observation_simulation.json'
env = simpy.Environment()
planning_algorithm = 'heft'
scheduling_algorithm = DynamicAlgorithmFromPlan
instrument = Telescope
dm = DelayModel(0.5, 'normal', DelayModel.DelayDegree.LOW)
simulation = Simulation(env=env,
config=CONFIG,
instrument=instrument,
algorithm_map={
'pheft': 'pheft',
'heft': 'heft',
'fifo': DynamicAlgorithmFromPlan
},
event_file=EVENT_FILE,
delay=dm)
simulation.start(-1)
# simulation.resume(200)
# User defined models
from user.telescope import Telescope # Instrument
from user.schedule.dynamic_plan import DynamicAlgorithmFromPlan # Scheduling
from user.schedule.greedy import GreedyAlgorithmFromPlan # Scheduling
from user.schedule.batch_allocation import BatchProcessing
from user.plan.batch_planning import BatchPlanning # Planning
from user.plan.static_planning import SHADOWPlanning
if __name__ == '__main__':
LOGGER.info(f"Running experiment from {RUN_PATH}/{FOLDER_PATH}")
env = simpy.Environment()
instrument = Telescope
# timestamp = f'{time.time()}'.split('.')[0]
simulation = Simulation(env=env,
config=cfg_path,
instrument=instrument,
planning_algorithm='heft',
planning_model=SHADOWPlanning('heft'),
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp='skaworkflows_test',
to_file=True,
hdf5_path=f'{RUN_PATH}/{FOLDER_PATH}/results.h5'
# hdf5_path='',
# delimiters=f'test/'
)
simulation.start()
LOGGER.info(f"Experiment finished, exiting script...")
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
Basic simulation, with minimal Observation Plan and dummy observation workflows
"""
import simpy
import logging
from topsim.user.schedule.dynamic_plan import DynamicAlgorithmFromPlan
from topsim.user.telescope import Telescope
from topsim.core.simulation import Simulation
logging.basicConfig(level="DEBUG")
LOGGER = logging.getLogger(__name__)
BASIC_CONFIG = 'simulations/basic_sim/input/basic_simulation.json'
EVENT_FILE = 'simulations/basic_sim/output/sim.trace'
# env = simpy.RealtimeEnvironment(factor=0.5, strict=False)
env = simpy.Environment()
event_file = EVENT_FILE
planning_algorithm = 'heft'
scheduling_algorithm = DynamicAlgorithmFromPlan
instrument = Telescope
algorith_map = {'heft': planning_algorithm, 'fifo': scheduling_algorithm}
simulation = Simulation(env, BASIC_CONFIG, instrument, algorith_map,
EVENT_FILE)
simulation.start(-1)
# RUNNING SIMULATION AND GENERATING DATA
for algorithm in algorithms:
for config in sorted(os.listdir(
'publications/2021_isc-hpc/config/single_size/40cluster')):
if '.json' in config:
CONFIG = f'publications/2021_isc-hpc/config/single_size/40cluster/{config}'
env = simpy.Environment()
instrument = Telescope
timestamp = f'{time.time()}'.split('.')[0]
simulation = Simulation(
env=env,
config=CONFIG,
instrument=instrument,
planning=algorithm,
scheduling=DynamicAlgorithmFromPlan,
delay=None,
timestamp={timestamp}
)
sim, tasks = simulation.start()
global_sim = global_sim.append(sim)
global_tasks = global_tasks.append(tasks)
print(algorithm, config, len(sim))
global_tasks.to_pickle('tasks_output.pkl')
global_sim.to_pickle('simulation_output.pkl')
# PLOTTING SIMULATION DATA - originally produced in a Jupyter Notebook
# Group by planning, delay, and config to get the simulation time for each
# simulation.
from topsim.core.simulation import Simulation
from topsim.algorithms.scheduling import FifoAlgorithm
# We need a number of configuration files for the basic set-up
TELESCOPE_CONFIG = 'test/data/config/observations.json'
CLUSTER_CONFIG = 'test/data/config/basic_spec-10.json'
BUFFER_CONFIG = 'test/data/config/buffer.json'
EVENT_FILE = 'recipes/output/sim.trace'
# Planning Algorithm - needs to be in our external library
planning_algorithm = 'heft'
# Scheduling Algorithm - these are implemented in a specific format to work
# in conjunction with the Scheduler Actor
scheduling_algorithm = FifoAlgorithm()
# Inititalise Simpy environment
env = simpy.Environment()
simulation = Simulation(env,
TELESCOPE_CONFIG,
CLUSTER_CONFIG,
BUFFER_CONFIG,
planning_algorithm,
scheduling_algorithm,
EVENT_FILE,
visualisation=False)