def test_flatten_energy_bills(grid):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid, True)
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills._update_market_fees(epb.area_result_dict, epb.flattened_area_core_stats_dict)
bills = m_bills._energy_bills(epb.area_result_dict, epb.flattened_area_core_stats_dict)
flattened = {}
m_bills._flatten_energy_bills(bills, flattened)
assert all("children" not in v for _, v in flattened.items())
name_list = ['house1', 'house2', 'pv', 'fridge', 'e-car', 'commercial']
uuid_list = [grid.name_uuid_mapping[k] for k in name_list]
assert_lists_contain_same_elements(uuid_list, flattened.keys())
house1_uuid = grid.name_uuid_mapping["house1"]
_compare_bills(flattened[house1_uuid], bills[house1_uuid])
house2_uuid = grid.name_uuid_mapping["house2"]
_compare_bills(flattened[house2_uuid], bills[house2_uuid])
commercial_uuid = grid.name_uuid_mapping["commercial"]
_compare_bills(flattened[commercial_uuid], bills[commercial_uuid])
pv_uuid = grid.name_uuid_mapping["pv"]
pv = [v for k, v in bills[house1_uuid]["children"].items() if k == pv_uuid][0]
_compare_bills(flattened[pv_uuid], pv)
fridge_uuid = grid.name_uuid_mapping["fridge"]
fridge = [v for k, v in bills[house1_uuid]["children"].items() if k == fridge_uuid][0]
_compare_bills(flattened[fridge_uuid], fridge)
ecar_uuid = grid.name_uuid_mapping["e-car"]
ecar = [v for k, v in bills[house2_uuid]["children"].items() if k == ecar_uuid][0]
_compare_bills(flattened[ecar_uuid], ecar)
def _init(self, slowdown, seed, paused, pause_after, redis_job_id):
self.paused = paused
self.pause_after = pause_after
self.slowdown = slowdown
if seed is not None:
random.seed(int(seed))
else:
random_seed = random.randint(0, RANDOM_SEED_MAX_VALUE)
random.seed(random_seed)
self.initial_params["seed"] = random_seed
log.info("Random seed: {}".format(random_seed))
self.area = self.setup_module.get_setup(self.simulation_config)
self.endpoint_buffer = SimulationEndpointBuffer(
redis_job_id,
self.initial_params,
self.area,
export_plots=self.should_export_plots)
self._update_and_send_results()
if GlobalConfig.POWER_FLOW:
self.power_flow = PandaPowerFlow(self.area)
self.power_flow.run_power_flow()
self.bc = None
if self.use_bc:
self.bc = BlockChainInterface()
log.debug("Starting simulation with config %s", self.simulation_config)
self._set_traversal_length()
are_all_areas_unique(self.area, set())
self.area.activate(self.bc)
def test_export_something_if_loads_in_setup(self):
house1 = Area("House1", [self.area1, self.area3])
self.grid = Area("Grid", [house1])
epb = SimulationEndpointBuffer("1", {"seed": 0}, self.grid)
epb._update_unmatched_loads(self.grid)
unmatched_loads = epb.unmatched_loads
assert unmatched_loads["House1"] is not None
assert unmatched_loads["Grid"] is not None
def test_energy_bills_finds_iaas(grid2):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid2, True)
epb.current_market_time_slot_str = grid2.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid2)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result['house1']['bought'] == result['house2']['sold'] == 3
def test_energy_bills_ensure_device_types_are_populated(grid2):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid2, True)
epb.current_market_time_slot_str = grid2.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid2)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result["house1"]["type"] == "Area"
assert result["house2"]["type"] == "Area"
def __init__(self,
setup_module_name: str,
simulation_config: SimulationConfig = None,
simulation_events: str = None,
slowdown: int = 0,
seed=None,
paused: bool = False,
pause_after: duration = None,
repl: bool = False,
no_export: bool = False,
export_path: str = None,
export_subdir: str = None,
redis_job_id=None,
enable_bc=False):
self.initial_params = dict(slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after)
self.simulation_config = simulation_config
self.use_repl = repl
self.export_on_finish = not no_export
self.export_path = export_path
self.sim_status = "initialized"
if export_subdir is None:
self.export_subdir = \
DateTime.now(tz=TIME_ZONE).format(f"{DATE_TIME_FORMAT}:ss")
else:
self.export_subdir = export_subdir
self.setup_module_name = setup_module_name
self.use_bc = enable_bc
self.is_stopped = False
self.redis_connection = RedisSimulationCommunication(
self, redis_job_id)
self._started_from_cli = redis_job_id is None
self.run_start = None
self.paused_time = None
self._load_setup_module()
self._init(**self.initial_params)
deserialize_events_to_areas(simulation_events, self.area)
validate_const_settings_for_simulation()
self.endpoint_buffer = SimulationEndpointBuffer(
redis_job_id, self.initial_params, self.area)
if self.export_on_finish or self.redis_connection.is_enabled():
self.export = ExportAndPlot(self.area, self.export_path,
self.export_subdir,
self.endpoint_buffer)
def test_energy_bills_use_only_last_market_if_not_keep_past_markets(grid_fees):
constants.RETAIN_PAST_MARKET_STRATEGIES_STATE = False
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid_fees, True)
epb.current_market_time_slot_str = grid_fees.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid_fees)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills._update_market_fees(epb.area_result_dict, epb.flattened_area_core_stats_dict)
assert m_bills.market_fees[grid_fees.name_uuid_mapping['house2']] == 0.03
assert m_bills.market_fees[grid_fees.name_uuid_mapping['street']] == 0.01
assert m_bills.market_fees[grid_fees.name_uuid_mapping['house1']] == 0.06
def test_energy_bills_redis(grid):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid, True)
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
result_redis = m_bills.bills_redis_results
for house in grid.children:
assert_dicts_identical(result[house.name], result_redis[house.uuid])
for device in house.children:
assert_dicts_identical(result[device.name], result_redis[device.uuid])
def __init__(self,
setup_module_name: str,
simulation_config: SimulationConfig = None,
slowdown: int = 0,
seed=None,
paused: bool = False,
pause_after: duration = None,
use_repl: bool = False,
export: bool = False,
export_path: str = None,
reset_on_finish: bool = False,
reset_on_finish_wait: duration = duration(minutes=1),
exit_on_finish: bool = False,
exit_on_finish_wait: duration = duration(seconds=1),
api_url=None,
redis_job_id=None,
use_bc=False):
self.initial_params = dict(slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after)
self.simulation_config = simulation_config
self.use_repl = use_repl
self.export_on_finish = export
self.export_path = export_path
self.reset_on_finish = reset_on_finish
self.reset_on_finish_wait = reset_on_finish_wait
self.exit_on_finish = exit_on_finish
self.exit_on_finish_wait = exit_on_finish_wait
self.api_url = api_url
self.setup_module_name = setup_module_name
self.use_bc = use_bc
self.is_stopped = False
self.endpoint_buffer = SimulationEndpointBuffer(
redis_job_id, self.initial_params)
self.redis_connection = RedisSimulationCommunication(
self, redis_job_id)
if sum([reset_on_finish, exit_on_finish, use_repl]) > 1:
raise D3AException(
"Can only specify one of '--reset-on-finish', '--exit-on-finish' and '--use-repl' "
"simultaneously.")
self.run_start = None
self.paused_time = None
self._load_setup_module()
self._init(**self.initial_params)
self._init_events()
def test_export_none_if_no_loads_in_setup(self):
house1 = Area("House1", [])
self.grid = Area("Grid", [house1])
epb = SimulationEndpointBuffer("1", {"seed": 0}, self.grid, True)
epb.market_unmatched_loads.update_unmatched_loads(self.grid)
unmatched_loads = epb.market_unmatched_loads.unmatched_loads
assert unmatched_loads["House1"] is None
assert unmatched_loads["Grid"] is None
def test_calculate_raw_energy_bills(grid):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid, True)
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills._update_market_fees(epb.area_result_dict, epb.flattened_area_core_stats_dict)
bills = m_bills._energy_bills(epb.area_result_dict, epb.flattened_area_core_stats_dict)
grid_children_uuids = [c.uuid for c in grid.children]
assert all(h in bills for h in grid_children_uuids)
commercial_uuid = grid.name_uuid_mapping["commercial"]
assert 'children' not in bills[commercial_uuid]
house1_uuid = grid.name_uuid_mapping["house1"]
assert grid.name_uuid_mapping["pv"] in bills[house1_uuid]["children"]
pv_bills = [v for k, v in bills[house1_uuid]["children"].items()
if k == grid.name_uuid_mapping["pv"]][0]
assert pv_bills['sold'] == 2.0 and isclose(pv_bills['earned'], 0.01)
assert grid.name_uuid_mapping["fridge"] in bills[house1_uuid]["children"]
house2_uuid = grid.name_uuid_mapping["house2"]
assert grid.name_uuid_mapping["e-car"] in bills[house2_uuid]["children"]
def test_energy_bills_last_past_market(grid):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid, True)
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result['house2']['Accumulated Trades']['bought'] == result['commercial']['sold'] == 1
assert result['house2']['Accumulated Trades']['spent'] == \
result['commercial']['earned'] == \
0.01
external_trades = result['house2']['External Trades']
assert external_trades['total_energy'] == external_trades['bought'] - external_trades['sold']
assert external_trades['total_cost'] == external_trades['spent'] - external_trades['earned']
assert result['commercial']['spent'] == result['commercial']['bought'] == 0
assert result['fridge']['bought'] == 2 and isclose(result['fridge']['spent'], 0.01)
assert result['pv']['sold'] == 2 and isclose(result['pv']['earned'], 0.01)
assert 'children' not in result
def test_energy_bills_accumulate_fees(grid_fees):
constants.RETAIN_PAST_MARKET_STRATEGIES_STATE = True
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid_fees, True)
epb.current_market_time_slot_str = grid_fees.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid_fees)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills._update_market_fees(epb.area_result_dict, epb.flattened_area_core_stats_dict)
grid_fees.children[0].past_markets = [FakeMarket([], name='house1', fees=2.0)]
grid_fees.children[1].past_markets = []
grid_fees.past_markets = [FakeMarket((_trade(2, make_iaa_name(grid_fees.children[0]), 3,
make_iaa_name(grid_fees.children[0]),
fee_price=4.0),), 'street', fees=4.0)]
epb.current_market_time_slot_str = grid_fees.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid_fees)
m_bills._update_market_fees(epb.area_result_dict, epb.flattened_area_core_stats_dict)
assert m_bills.market_fees[grid_fees.name_uuid_mapping['house2']] == 0.03
assert m_bills.market_fees[grid_fees.name_uuid_mapping['street']] == 0.05
assert m_bills.market_fees[grid_fees.name_uuid_mapping['house1']] == 0.08
def test_energy_bills_report_correctly_market_fees(grid_fees):
constants.RETAIN_PAST_MARKET_STRATEGIES_STATE = True
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid_fees, True)
epb.current_market_time_slot_str = grid_fees.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid_fees)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
grid_fees.children[0].past_markets = [FakeMarket([], name='house1', fees=2.0)]
grid_fees.children[1].past_markets = []
grid_fees.past_markets = [FakeMarket((_trade(2, make_iaa_name(grid_fees.children[0]), 3,
make_iaa_name(grid_fees.children[0]),
fee_price=4.0),), 'street', fees=4.0)]
epb.current_market_time_slot_str = grid_fees.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid_fees)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result["street"]["house1"]["market_fee"] == 0.04
assert result["street"]["house2"]["market_fee"] == 0.01
assert result["street"]['Accumulated Trades']["market_fee"] == 0.05
assert result["house1"]['External Trades']["market_fee"] == 0.0
assert result["house2"]['External Trades']["market_fee"] == 0.0
def test_energy_bills(grid):
epb = SimulationEndpointBuffer("1", {"seed": 0}, grid, True)
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills = MarketEnergyBills(should_export_plots=True)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result['house2']['Accumulated Trades']['bought'] == result['commercial']['sold'] == 1
assert result['house2']['Accumulated Trades']['spent'] == result['commercial']['earned'] == \
0.01
assert result['commercial']['spent'] == result['commercial']['bought'] == 0
assert result['fridge']['bought'] == 2 and isclose(result['fridge']['spent'], 0.01)
assert result['pv']['sold'] == 2 and isclose(result['pv']['earned'], 0.01)
assert 'children' not in result
grid.children[0].past_markets = [FakeMarket((_trade(2, 'fridge', 2, 'pv'),
_trade(3, 'fridge', 1, 'iaa')), 'house1')]
grid.children[1].past_markets = [FakeMarket((_trade(1, 'e-car', 4, 'iaa'),
_trade(1, 'e-car', 8, 'iaa'),
_trade(3, 'iaa', 5, 'e-car')), 'house2')]
grid.past_markets = [FakeMarket((_trade(2, 'house2', 12, 'commercial'),), 'grid')]
epb.current_market_time_slot_str = grid.current_market.time_slot_str
epb._populate_core_stats_and_sim_state(grid)
m_bills.update(epb.area_result_dict, epb.flattened_area_core_stats_dict,
epb.current_market_time_slot_str)
result = m_bills.bills_results
assert result['house2']['Accumulated Trades']['bought'] == result['commercial']['sold'] == 13
assert result['house2']['Accumulated Trades']['spent'] == \
result['commercial']['earned'] == \
0.03
assert result['commercial']['spent'] == result['commercial']['bought'] == 0
assert result['fridge']['bought'] == 5 and isclose(result['fridge']['spent'], 0.06)
assert result['pv']['sold'] == 4 and isclose(result['pv']['earned'], 0.03)
assert 'children' not in result
class Simulation:
def __init__(self,
setup_module_name: str,
simulation_config: SimulationConfig = None,
slowdown: int = 0,
seed=None,
paused: bool = False,
pause_after: duration = None,
use_repl: bool = False,
export: bool = False,
export_path: str = None,
reset_on_finish: bool = False,
reset_on_finish_wait: duration = duration(minutes=1),
exit_on_finish: bool = False,
exit_on_finish_wait: duration = duration(seconds=1),
api_url=None,
redis_job_id=None,
use_bc=False):
self.initial_params = dict(slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after)
self.simulation_config = simulation_config
self.use_repl = use_repl
self.export_on_finish = export
self.export_path = export_path
self.reset_on_finish = reset_on_finish
self.reset_on_finish_wait = reset_on_finish_wait
self.exit_on_finish = exit_on_finish
self.exit_on_finish_wait = exit_on_finish_wait
self.api_url = api_url
self.setup_module_name = setup_module_name
self.use_bc = use_bc
self.is_stopped = False
self.endpoint_buffer = SimulationEndpointBuffer(
redis_job_id, self.initial_params)
self.redis_connection = RedisSimulationCommunication(
self, redis_job_id)
if sum([reset_on_finish, exit_on_finish, use_repl]) > 1:
raise D3AException(
"Can only specify one of '--reset-on-finish', '--exit-on-finish' and '--use-repl' "
"simultaneously.")
self.run_start = None
self.paused_time = None
self._load_setup_module()
self._init(**self.initial_params)
self._init_events()
def _set_traversal_length(self):
if ConstSettings.GeneralSettings.MAX_OFFER_TRAVERSAL_LENGTH is None:
no_of_levels = self._get_setup_levels(self.area) + 1
num_ticks_to_propagate = no_of_levels * 2
ConstSettings.GeneralSettings.MAX_OFFER_TRAVERSAL_LENGTH = int(
num_ticks_to_propagate)
time_to_propagate_minutes = num_ticks_to_propagate * \
self.simulation_config.tick_length.seconds / 60.
log.error(
"Setup has {} levels, offers/bids need at least {} minutes "
"({} ticks) to propagate.".format(
no_of_levels,
time_to_propagate_minutes,
ConstSettings.GeneralSettings.MAX_OFFER_TRAVERSAL_LENGTH,
))
def _get_setup_levels(self, area, level_count=0):
level_count += 1
count_list = [
self._get_setup_levels(child, level_count)
for child in area.children if child.children
]
return max(count_list) if len(count_list) > 0 else level_count
def _load_setup_module(self):
try:
self.setup_module = import_module(
".{}".format(self.setup_module_name), 'd3a.setup')
log.info("Using setup module '%s'", self.setup_module_name)
except ImportError as ex:
raise SimulationException("Invalid setup module '{}'".format(
self.setup_module_name)) from ex
def _init_events(self):
self.interrupt = Event()
self.interrupted = Event()
self.ready = Event()
self.ready.set()
def _init(self, slowdown, seed, paused, pause_after):
self.paused = paused
self.pause_after = pause_after
self.slowdown = slowdown
if seed:
random.seed(seed)
else:
random_seed = random.randint(0, 1000000)
random.seed(random_seed)
log.error("Random seed: {}".format(random_seed))
self.area = self.setup_module.get_setup(self.simulation_config)
self.bc = None # type: BlockChainInterface
if self.use_bc:
self.bc = BlockChainInterface()
log.info("Starting simulation with config %s", self.simulation_config)
self._set_traversal_length()
are_all_areas_unique(self.area, set())
self.area.activate(self.bc)
@property
def finished(self):
return self.area.current_tick >= self.area.config.total_ticks
@property
def time_since_start(self):
return self.area.current_tick * self.simulation_config.tick_length
def reset(self, sync=True):
"""
Reset simulation to initial values and restart the run.
*IMPORTANT*: This method *MUST* be called from another thread, otherwise a deadlock will
occur!
"""
log.error("=" * 15 + " Simulation reset requested " + "=" * 15)
if sync:
self.interrupted.clear()
self.interrupt.set()
self.interrupted.wait()
self.interrupt.clear()
self._init(**self.initial_params)
self.ready.set()
def stop(self):
self.is_stopped = True
def run(self, resume=False) -> (Period, duration):
if resume:
log.critical("Resuming simulation")
self._info()
self.is_stopped = False
config = self.simulation_config
tick_lengths_s = config.tick_length.total_seconds()
slot_count = int(config.duration / config.slot_length) + 1
while True:
self.ready.wait()
self.ready.clear()
if resume:
# FIXME: Fix resume time calculation
if self.run_start is None or self.paused_time is None:
raise RuntimeError("Can't resume without saved state")
slot_resume, tick_resume = divmod(self.area.current_tick,
config.ticks_per_slot)
else:
self.run_start = DateTime.now(tz=TIME_ZONE)
self.paused_time = 0
slot_resume = tick_resume = 0
try:
with NonBlockingConsole() as console:
for slot_no in range(slot_resume, slot_count - 1):
run_duration = (DateTime.now(tz=TIME_ZONE) -
self.run_start -
duration(seconds=self.paused_time))
log.error(
"Slot %d of %d (%2.0f%%) - %s elapsed, ETA: %s",
slot_no + 1, slot_count,
(slot_no + 1) / slot_count * 100, run_duration,
run_duration / (slot_no + 1) * slot_count)
if self.is_stopped:
log.error("Received stop command.")
sleep(5)
break
for tick_no in range(tick_resume,
config.ticks_per_slot):
# reset tick_resume after possible resume
tick_resume = 0
self._handle_input(console)
self.paused_time += self._handle_paused(console)
tick_start = time.monotonic()
log.debug(
"Tick %d of %d in slot %d (%2.0f%%)",
tick_no + 1,
config.ticks_per_slot,
slot_no + 1,
(tick_no + 1) / config.ticks_per_slot * 100,
)
with page_lock:
self.area.tick(is_root_area=True)
tick_length = time.monotonic() - tick_start
if self.slowdown and tick_length < tick_lengths_s:
# Simulation runs faster than real time but a slowdown was
# requested
tick_diff = tick_lengths_s - tick_length
diff_slowdown = tick_diff * self.slowdown / 10000
log.debug("Slowdown: %.4f", diff_slowdown)
self._handle_input(console, diff_slowdown)
with page_lock:
self.endpoint_buffer.update_stats(
self.area, self.status)
self.redis_connection.publish_intermediate_results(
self.endpoint_buffer)
run_duration = (DateTime.now(tz=TIME_ZONE) -
self.run_start -
duration(seconds=self.paused_time))
paused_duration = duration(seconds=self.paused_time)
self.redis_connection.publish_results(self.endpoint_buffer)
if not self.is_stopped:
log.error(
"Run finished in %s%s / %.2fx real time",
run_duration,
" ({} paused)".format(paused_duration)
if paused_duration else "",
config.duration / (run_duration - paused_duration))
if not self.exit_on_finish:
log.error("REST-API still running at %s", self.api_url)
if self.export_on_finish:
export = ExportAndPlot(
self.area, self.export_path,
DateTime.now(tz=TIME_ZONE).isoformat())
json_dir = os.path.join(export.directory,
"aggregated_results")
mkdir_from_str(json_dir)
for key, value in self.endpoint_buffer.generate_result_report(
).items():
json_file = os.path.join(json_dir, key)
with open(json_file, 'w') as outfile:
json.dump(value, outfile)
if self.use_repl:
self._start_repl()
elif self.reset_on_finish:
log.error("Automatically restarting simulation in %s",
format_interval(self.reset_on_finish_wait))
self._handle_input(
console, self.reset_on_finish_wait.in_seconds())
def _reset():
self.reset(sync=False)
self.paused = False
t = Thread(target=_reset)
t.start()
t.join()
continue
elif self.exit_on_finish:
self._handle_input(
console, self.exit_on_finish_wait.in_seconds())
log.error("Terminating. (--exit-on-finish set.)")
break
else:
log.info("Ctrl-C to quit")
while True:
self._handle_input(console, 0.5)
break
except _SimulationInterruped:
self.interrupted.set()
except KeyboardInterrupt:
break
def toggle_pause(self):
if self.finished:
return False
self.paused = not self.paused
return True
def _handle_input(self, console, sleep: float = 0):
timeout = 0
start = 0
if sleep > 0:
timeout = sleep / 100
start = time.monotonic()
while True:
if self.interrupt.is_set():
raise _SimulationInterruped()
cmd = console.get_char(timeout)
if cmd:
if cmd not in {'i', 'p', 'q', 'r', 'S', 'R', 's', '+', '-'}:
log.critical("Invalid command. Valid commands:\n"
" [i] info\n"
" [p] pause\n"
" [q] quit\n"
" [r] reset\n"
" [S] stop\n"
" [R] start REPL\n"
" [s] save state\n"
" [+] increase slowdown\n"
" [-] decrease slowdown")
continue
if self.finished and cmd in {'p', '+', '-'}:
log.error(
"Simulation has finished. The commands [p, +, -] are unavailable."
)
continue
if cmd == 'r':
Thread(target=lambda: self.reset()).start()
elif cmd == 'R':
self._start_repl()
elif cmd == 'i':
self._info()
elif cmd == 'p':
self.paused = not self.paused
break
elif cmd == 'q':
raise KeyboardInterrupt()
elif cmd == 's':
self.save_state()
elif cmd == 'S':
self.stop()
elif cmd == '+':
v = 5
if self.slowdown <= 95:
self.slowdown += v
log.critical("Simulation slowdown changed to %d",
self.slowdown)
elif cmd == '-':
if self.slowdown >= 5:
self.slowdown -= 5
log.critical("Simulation slowdown changed to %d",
self.slowdown)
if sleep == 0 or time.monotonic() - start >= sleep:
break
def _handle_paused(self, console):
if self.pause_after and self.time_since_start >= self.pause_after:
self.paused = True
self.pause_after = None
if self.paused:
start = time.monotonic()
log.critical(
"Simulation paused. Press 'p' to resume or resume from API.")
self.endpoint_buffer.update(self.area, self.status)
self.redis_connection.publish_intermediate_results(
self.endpoint_buffer)
while self.paused and not self.interrupt.is_set():
self._handle_input(console, 0.1)
log.critical("Simulation resumed")
return time.monotonic() - start
return 0
def _info(self):
info = self.simulation_config.as_dict()
slot, tick = divmod(self.area.current_tick,
self.simulation_config.ticks_per_slot)
percent = self.area.current_tick / self.simulation_config.total_ticks * 100
slot_count = self.simulation_config.duration // self.simulation_config.slot_length
info.update(slot=slot + 1,
tick=tick + 1,
slot_count=slot_count,
percent=percent)
log.critical(
"\n"
"Simulation configuration:\n"
" Duration: %(duration)s\n"
" Slot length: %(slot_length)s\n"
" Tick length: %(tick_length)s\n"
" Market count: %(market_count)d\n"
" Ticks per slot: %(ticks_per_slot)d\n"
"Status:\n"
" Slot: %(slot)d / %(slot_count)d\n"
" Tick: %(tick)d / %(ticks_per_slot)d\n"
" Completed: %(percent).1f%%", info)
def _start_repl(self):
log.info(
"An interactive REPL has been started. The root Area is available as "
"`root_area`.")
log.info("Ctrl-D to quit.")
embed({'root_area': self.area})
def save_state(self):
save_dir = Path('.d3a')
save_dir.mkdir(exist_ok=True)
save_file_name = save_dir.joinpath(
"saved-state_{:%Y%m%dT%H%M%S}.pickle".format(
DateTime.now(tz=TIME_ZONE)))
with save_file_name.open('wb') as save_file:
dill.dump(self, save_file, protocol=HIGHEST_PROTOCOL)
log.critical("Saved state to %s", save_file_name.resolve())
return save_file_name
@property
def status(self):
if self.is_stopped:
return "stopped"
elif self.finished:
return "finished"
elif self.paused:
return "paused"
elif self.ready.is_set():
return "ready"
else:
return "running"
def __getstate__(self):
state = self.__dict__.copy()
state['_random_state'] = random.getstate()
del state['interrupt']
del state['interrupted']
del state['ready']
del state['setup_module']
return state
def __setstate__(self, state):
random.setstate(state.pop('_random_state'))
self.__dict__.update(state)
self._load_setup_module()
self._init_events()
class Simulation:
def __init__(self,
setup_module_name: str,
simulation_config: SimulationConfig = None,
simulation_events: str = None,
slowdown: int = 0,
seed=None,
paused: bool = False,
pause_after: duration = None,
repl: bool = False,
no_export: bool = False,
export_path: str = None,
export_subdir: str = None,
redis_job_id=None,
enable_bc=False):
self.initial_params = dict(slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after)
self.progress_info = SimulationProgressInfo()
self.simulation_config = simulation_config
self.use_repl = repl
self.export_on_finish = not no_export
self.export_path = export_path
self.sim_status = "initializing"
self.is_timed_out = False
if export_subdir is None:
self.export_subdir = \
DateTime.now(tz=TIME_ZONE).format(f"{DATE_TIME_FORMAT}:ss")
else:
self.export_subdir = export_subdir
self.setup_module_name = setup_module_name
self.use_bc = enable_bc
self.is_stopped = False
self.live_events = LiveEvents(self.simulation_config)
self.redis_connection = RedisSimulationCommunication(
self, redis_job_id, self.live_events)
self._simulation_id = redis_job_id
self._started_from_cli = redis_job_id is None
self.run_start = None
self.paused_time = None
self._load_setup_module()
self._init(**self.initial_params, redis_job_id=redis_job_id)
deserialize_events_to_areas(simulation_events, self.area)
validate_const_settings_for_simulation()
if self.export_on_finish and not self.redis_connection.is_enabled():
self.export = ExportAndPlot(self.area, self.export_path,
self.export_subdir,
self.endpoint_buffer)
def _set_traversal_length(self):
no_of_levels = self._get_setup_levels(self.area) + 1
num_ticks_to_propagate = no_of_levels * 2
ConstSettings.GeneralSettings.MAX_OFFER_TRAVERSAL_LENGTH = 2
time_to_propagate_minutes = num_ticks_to_propagate * \
self.simulation_config.tick_length.seconds / 60.
log.info("Setup has {} levels, offers/bids need at least {} minutes "
"({} ticks) to propagate.".format(
no_of_levels,
time_to_propagate_minutes,
ConstSettings.GeneralSettings.MAX_OFFER_TRAVERSAL_LENGTH,
))
def _get_setup_levels(self, area, level_count=0):
level_count += 1
count_list = [
self._get_setup_levels(child, level_count)
for child in area.children if child.children
]
return max(count_list) if len(count_list) > 0 else level_count
def _load_setup_module(self):
try:
if ConstSettings.GeneralSettings.SETUP_FILE_PATH is None:
self.setup_module = import_module(
".{}".format(self.setup_module_name), 'd3a.setup')
else:
import sys
sys.path.append(ConstSettings.GeneralSettings.SETUP_FILE_PATH)
self.setup_module = import_module("{}".format(
self.setup_module_name))
log.debug("Using setup module '%s'", self.setup_module_name)
except ImportError as ex:
raise SimulationException("Invalid setup module '{}'".format(
self.setup_module_name)) from ex
def _init(self, slowdown, seed, paused, pause_after, redis_job_id):
self.paused = paused
self.pause_after = pause_after
self.slowdown = slowdown
if seed is not None:
random.seed(int(seed))
else:
random_seed = random.randint(0, RANDOM_SEED_MAX_VALUE)
random.seed(random_seed)
self.initial_params["seed"] = random_seed
log.info("Random seed: {}".format(random_seed))
self.area = self.setup_module.get_setup(self.simulation_config)
self.endpoint_buffer = SimulationEndpointBuffer(
redis_job_id,
self.initial_params,
self.area,
export_plots=self.should_export_plots)
self._update_and_send_results()
if GlobalConfig.POWER_FLOW:
self.power_flow = PandaPowerFlow(self.area)
self.power_flow.run_power_flow()
self.bc = None
if self.use_bc:
self.bc = BlockChainInterface()
log.debug("Starting simulation with config %s", self.simulation_config)
self._set_traversal_length()
are_all_areas_unique(self.area, set())
self.area.activate(self.bc)
@property
def finished(self):
return self.area.current_tick >= self.area.config.total_ticks
@property
def time_since_start(self):
return self.area.current_tick * self.simulation_config.tick_length
def reset(self):
"""
Reset simulation to initial values and restart the run.
"""
log.info("=" * 15 + " Simulation reset requested " + "=" * 15)
self._init(**self.initial_params)
self.run()
raise SimulationResetException
def stop(self):
self.is_stopped = True
def deactivate_areas(self, area):
"""
For putting the last market into area.past_markets
"""
area.deactivate()
for child in area.children:
self.deactivate_areas(child)
def run(self, resume=False) -> (Period, duration):
self.sim_status = "running"
if resume:
log.critical("Resuming simulation")
self._info()
self.is_stopped = False
while True:
if resume:
# FIXME: Fix resume time calculation
if self.run_start is None or self.paused_time is None:
raise RuntimeError("Can't resume without saved state")
slot_resume, tick_resume = divmod(
self.area.current_tick,
self.simulation_config.ticks_per_slot)
else:
self.run_start = DateTime.now(tz=TIME_ZONE)
self.paused_time = 0
slot_resume = tick_resume = 0
try:
self._run_cli_execute_cycle(slot_resume, tick_resume) \
if self._started_from_cli \
else self._execute_simulation(slot_resume, tick_resume)
except KeyboardInterrupt:
break
except SimulationResetException:
break
else:
break
def _run_cli_execute_cycle(self, slot_resume, tick_resume):
with NonBlockingConsole() as console:
self._execute_simulation(slot_resume, tick_resume, console)
def _update_and_send_results(self, is_final=False):
self.endpoint_buffer.update_stats(self.area, self.status,
self.progress_info)
if not self.redis_connection.is_enabled():
return
if is_final:
self.redis_connection.publish_results(self.endpoint_buffer)
if hasattr(self.redis_connection, 'heartbeat'):
self.redis_connection.heartbeat.cancel()
else:
self.redis_connection.publish_intermediate_results(
self.endpoint_buffer)
def _update_progress_info(self, slot_no, slot_count):
run_duration = (DateTime.now(tz=TIME_ZONE) - self.run_start -
duration(seconds=self.paused_time))
self.progress_info.eta = (run_duration /
(slot_no + 1) * slot_count) - run_duration
self.progress_info.elapsed_time = run_duration
self.progress_info.percentage_completed = (slot_no +
1) / slot_count * 100
self.progress_info.current_slot_str = get_market_slot_time_str(
slot_no, self.simulation_config)
self.progress_info.next_slot_str = get_market_slot_time_str(
slot_no + 1, self.simulation_config)
def _execute_simulation(self, slot_resume, tick_resume, console=None):
config = self.simulation_config
tick_lengths_s = config.tick_length.total_seconds()
slot_count = int(config.sim_duration / config.slot_length)
self.simulation_config.external_redis_communicator.sub_to_aggregator()
self.simulation_config.external_redis_communicator.start_communication(
)
self._update_and_send_results()
for slot_no in range(slot_resume, slot_count):
self._update_progress_info(slot_no, slot_count)
log.warning("Slot %d of %d (%2.0f%%) - %s elapsed, ETA: %s",
slot_no + 1, slot_count,
self.progress_info.percentage_completed,
self.progress_info.elapsed_time,
self.progress_info.eta)
if self.is_stopped:
log.info("Received stop command.")
sleep(5)
break
self.live_events.handle_all_events(self.area)
self.area._cycle_markets()
gc.collect()
process = psutil.Process(os.getpid())
mbs_used = process.memory_info().rss / 1000000.0
log.debug(f"Used {mbs_used} MBs.")
for tick_no in range(tick_resume, config.ticks_per_slot):
tick_start = time.time()
self._handle_paused(console, tick_start)
# reset tick_resume after possible resume
tick_resume = 0
log.trace(
"Tick %d of %d in slot %d (%2.0f%%)",
tick_no + 1,
config.ticks_per_slot,
slot_no + 1,
(tick_no + 1) / config.ticks_per_slot * 100,
)
self.simulation_config.external_redis_communicator.\
approve_aggregator_commands()
self.area.tick_and_dispatch()
self.simulation_config.external_redis_communicator.\
publish_aggregator_commands_responses_events()
realtime_tick_length = time.time() - tick_start
if self.slowdown and realtime_tick_length < tick_lengths_s:
# Simulation runs faster than real time but a slowdown was
# requested
tick_diff = tick_lengths_s - realtime_tick_length
diff_slowdown = tick_diff * self.slowdown / SLOWDOWN_FACTOR
log.trace("Slowdown: %.4f", diff_slowdown)
if console is not None:
self._handle_input(console, diff_slowdown)
else:
sleep(diff_slowdown)
if ConstSettings.GeneralSettings.RUN_REAL_TIME:
sleep(abs(tick_lengths_s - realtime_tick_length))
self._update_and_send_results()
if self.export_on_finish and not self.redis_connection.is_enabled(
):
self.export.data_to_csv(self.area,
True if slot_no == 0 else False)
self.sim_status = "finished"
self.deactivate_areas(self.area)
if not self.is_stopped:
self._update_progress_info(slot_count - 1, slot_count)
paused_duration = duration(seconds=self.paused_time)
log.info(
"Run finished in %s%s / %.2fx real time",
self.progress_info.elapsed_time,
" ({} paused)".format(paused_duration)
if paused_duration else "", config.sim_duration /
(self.progress_info.elapsed_time - paused_duration))
self._update_and_send_results(is_final=True)
if self.export_on_finish and not self.redis_connection.is_enabled():
log.info("Exporting simulation data.")
if GlobalConfig.POWER_FLOW:
self.export.export(export_plots=self.should_export_plots,
power_flow=self.power_flow)
else:
self.export.export(self.should_export_plots)
if self.use_repl:
self._start_repl()
@property
def should_export_plots(self):
return not self.redis_connection.is_enabled()
def toggle_pause(self):
if self.finished:
return False
self.paused = not self.paused
return True
def _handle_input(self, console, sleep: float = 0):
timeout = 0
start = 0
if sleep > 0:
timeout = sleep / 100
start = time.time()
while True:
cmd = console.get_char(timeout)
if cmd:
if cmd not in {'i', 'p', 'q', 'r', 'S', 'R', 's', '+', '-'}:
log.critical("Invalid command. Valid commands:\n"
" [i] info\n"
" [p] pause\n"
" [q] quit\n"
" [r] reset\n"
" [S] stop\n"
" [R] start REPL\n"
" [s] save state\n"
" [+] increase slowdown\n"
" [-] decrease slowdown")
continue
if self.finished and cmd in {'p', '+', '-'}:
log.info(
"Simulation has finished. The commands [p, +, -] are unavailable."
)
continue
if cmd == 'r':
self.reset()
elif cmd == 'R':
self._start_repl()
elif cmd == 'i':
self._info()
elif cmd == 'p':
self.paused = not self.paused
break
elif cmd == 'q':
raise KeyboardInterrupt()
elif cmd == 's':
self.save_state()
elif cmd == 'S':
self.stop()
elif cmd == '+':
if self.slowdown <= SLOWDOWN_FACTOR - SLOWDOWN_STEP:
self.slowdown += SLOWDOWN_STEP
log.critical("Simulation slowdown changed to %d",
self.slowdown)
elif cmd == '-':
if self.slowdown >= SLOWDOWN_STEP:
self.slowdown -= SLOWDOWN_STEP
log.critical("Simulation slowdown changed to %d",
self.slowdown)
if sleep == 0 or time.time() - start >= sleep:
break
def _handle_paused(self, console, tick_start):
if console is not None:
self._handle_input(console)
if self.pause_after and self.time_since_start >= self.pause_after:
self.paused = True
self.pause_after = None
paused_flag = False
if self.paused:
if console:
log.critical(
"Simulation paused. Press 'p' to resume or resume from API."
)
else:
self._update_and_send_results()
start = time.time()
while self.paused:
paused_flag = True
if console:
self._handle_input(console, 0.1)
if time.time() - tick_start > SIMULATION_PAUSE_TIMEOUT:
self.is_timed_out = True
self.is_stopped = True
self.paused = False
sleep(0.5)
if console and paused_flag:
log.critical("Simulation resumed")
self.paused_time += time.time() - start
def _info(self):
info = self.simulation_config.as_dict()
slot, tick = divmod(self.area.current_tick,
self.simulation_config.ticks_per_slot)
percent = self.area.current_tick / self.simulation_config.total_ticks * 100
slot_count = self.simulation_config.sim_duration // self.simulation_config.slot_length
info.update(slot=slot + 1,
tick=tick + 1,
slot_count=slot_count,
percent=percent)
log.critical(
"\n"
"Simulation configuration:\n"
" Duration: %(sim_duration)s\n"
" Slot length: %(slot_length)s\n"
" Tick length: %(tick_length)s\n"
" Market count: %(market_count)d\n"
" Ticks per slot: %(ticks_per_slot)d\n"
"Status:\n"
" Slot: %(slot)d / %(slot_count)d\n"
" Tick: %(tick)d / %(ticks_per_slot)d\n"
" Completed: %(percent).1f%%", info)
def _start_repl(self):
log.debug(
"An interactive REPL has been started. The root Area is available as "
"`root_area`.")
log.debug("Ctrl-D to quit.")
embed({'root_area': self.area})
def save_state(self):
save_dir = Path('.d3a')
save_dir.mkdir(exist_ok=True)
save_file_name = save_dir.joinpath(
"saved-state_{:%Y%m%dT%H%M%S}.pickle".format(
DateTime.now(tz=TIME_ZONE)))
with save_file_name.open('wb') as save_file:
dill.dump(self, save_file, protocol=HIGHEST_PROTOCOL)
log.critical("Saved state to %s", save_file_name.resolve())
return save_file_name
@property
def status(self):
if self.is_timed_out:
return "timed-out"
elif self.is_stopped:
return "stopped"
elif self.paused:
return "paused"
else:
return self.sim_status
def __getstate__(self):
state = self.__dict__.copy()
state['_random_state'] = random.getstate()
del state['setup_module']
return state
def __setstate__(self, state):
random.setstate(state.pop('_random_state'))
self.__dict__.update(state)
self._load_setup_module()