def after_scenario(context, scenario):
shutil.rmtree(context.simdir)
update_advanced_settings(context.default_const_settings)
context.resource_manager.close()
DeviceRegistry.REGISTRY = {}
GlobalConfig.market_maker_rate = 30
def run(interface, port, setup_module_name, settings_file, slowdown, seed,
paused, pause_after, repl, export, export_path, reset_on_finish,
reset_on_finish_wait, exit_on_finish, exit_on_finish_wait, enable_bc,
enable_bm, **config_params):
try:
if settings_file is not None:
simulation_settings, advanced_settings = read_settings_from_file(
settings_file)
update_advanced_settings(advanced_settings)
simulation_config = SimulationConfig(**simulation_settings)
else:
simulation_config = SimulationConfig(**config_params)
api_url = "http://{}:{}/api".format(interface, port)
ConstSettings.BalancingSettings.ENABLE_BALANCING_MARKET = enable_bm
simulation = Simulation(setup_module_name=setup_module_name,
simulation_config=simulation_config,
slowdown=slowdown,
seed=seed,
paused=paused,
pause_after=pause_after,
use_repl=repl,
export=export,
export_path=export_path,
reset_on_finish=reset_on_finish,
reset_on_finish_wait=reset_on_finish_wait,
exit_on_finish=exit_on_finish,
exit_on_finish_wait=exit_on_finish_wait,
api_url=api_url,
redis_job_id=None,
use_bc=enable_bc)
except D3AException as ex:
raise click.BadOptionUsage(ex.args[0])
start_web(interface, port, simulation)
simulation.run()
def run(setup_module_name, settings_file, duration, slot_length, tick_length,
market_count, cloud_coverage, compare_alt_pricing,
enable_external_connection, start_date, pause_at, slot_length_realtime,
**kwargs):
# Force the multiprocessing start method to be 'fork' on macOS.
if platform.system() == 'Darwin':
multiprocessing.set_start_method('fork')
try:
if settings_file is not None:
simulation_settings, advanced_settings = read_settings_from_file(
settings_file)
update_advanced_settings(advanced_settings)
validate_global_settings(simulation_settings)
simulation_settings["external_connection_enabled"] = False
simulation_config = SimulationConfig(**simulation_settings)
else:
global_settings = {
"sim_duration": duration,
"slot_length": slot_length,
"tick_length": tick_length,
"cloud_coverage": cloud_coverage,
"market_count": market_count
}
validate_global_settings(global_settings)
simulation_config = \
SimulationConfig(duration, slot_length, tick_length, market_count,
cloud_coverage, start_date=start_date,
external_connection_enabled=enable_external_connection)
if compare_alt_pricing is True:
ConstSettings.IAASettings.AlternativePricing.COMPARE_PRICING_SCHEMES = True
# we need the seconds in the export dir name
kwargs["export_subdir"] = DateTime.now(
tz=TIME_ZONE).format(f"{DATE_TIME_FORMAT}:ss")
processes = []
for pricing_scheme in range(0, 4):
kwargs["pricing_scheme"] = pricing_scheme
p = Process(target=run_simulation,
args=(setup_module_name, simulation_config, None,
None, None, slot_length_realtime, kwargs))
p.start()
processes.append(p)
for p in processes:
p.join()
else:
if pause_at is not None:
kwargs["pause_after"] = convert_str_to_pause_after_interval(
start_date, pause_at)
run_simulation(setup_module_name, simulation_config, None, None,
None, slot_length_realtime, kwargs)
except D3AException as ex:
raise click.BadOptionUsage(ex.args[0])
def start(scenario, settings):
logging.getLogger().setLevel(logging.ERROR)
interface = environ.get('WORKER_INTERFACE', "0.0.0.0")
port = int(environ.get('WORKER_PORT', 5000))
api_host = environ.get('WORKER_HOST', interface)
api_url = "http://{}:{}/api".format(api_host, port)
job = get_current_job()
job.meta['api_url'] = api_url
job.save_meta()
if settings is None:
settings = {}
advanced_settings = settings.get('advanced_settings', None)
if advanced_settings is not None:
update_advanced_settings(ast.literal_eval(advanced_settings))
config = SimulationConfig(
duration=pendulum.duration(days=1 if 'duration' not in
settings else settings['duration'].days),
slot_length=pendulum.duration(
seconds=15 * 60 if 'slot_length' not in
settings else settings['slot_length'].seconds),
tick_length=pendulum.duration(
seconds=15 if 'tick_length' not in
settings else settings['tick_length'].seconds),
market_count=settings.get('market_count', 1),
cloud_coverage=settings.get(
'cloud_coverage', ConstSettings.PVSettings.DEFAULT_POWER_PROFILE),
pv_user_profile=settings.get('pv_user_profile', None),
market_maker_rate=settings.get(
'market_maker_rate',
str(ConstSettings.GeneralSettings.DEFAULT_MARKET_MAKER_RATE)),
iaa_fee=settings.get('iaa_fee',
ConstSettings.IAASettings.FEE_PERCENTAGE))
if scenario is None:
scenario_name = "default"
elif scenario in available_simulation_scenarios:
scenario_name = scenario
else:
scenario_name = 'json_arg'
config.area = scenario
simulation = Simulation(scenario_name,
config,
slowdown=settings.get('slowdown', 0),
exit_on_finish=True,
exit_on_finish_wait=pendulum.duration(seconds=10),
api_url=api_url,
redis_job_id=job.id)
start_web(interface, port, simulation)
simulation.run()
def test_parse_settings_file(self):
simulation_settings, advanced_settings = read_settings_from_file(
os.path.join(d3a_path, "setup", "d3a-settings.json"))
update_advanced_settings(advanced_settings)
simulation_config = SimulationConfig(**simulation_settings)
assert simulation_config.__getattribute__(
"sim_duration") == IntervalType('H:M')("24h")
try:
for setting in advanced_settings.keys():
getattr(ConstSettings, setting)
except AttributeError:
self.fail(
"The settings file is not consistent with the selection of variables in "
"const.py")
def start(scenario, settings, events, aggregator_device_mapping):
logging.getLogger().setLevel(logging.ERROR)
scenario = decompress_and_decode_queued_strings(scenario)
job = get_current_job()
job.save_meta()
try:
if settings is None:
settings = {}
else:
settings = {
k: v
for k, v in settings.items() if v is not None and v != "None"
}
advanced_settings = settings.get('advanced_settings', None)
if advanced_settings is not None:
update_advanced_settings(ast.literal_eval(advanced_settings))
aggregator_device_mapping = json.loads(aggregator_device_mapping)
if events is not None:
events = ast.literal_eval(events)
config_settings = {
"start_date":
instance(
datetime.combine(settings.get('start_date'),
datetime.min.time()))
if 'start_date' in settings else GlobalConfig.start_date,
"sim_duration":
duration(days=settings['duration'].days)
if 'duration' in settings else GlobalConfig.sim_duration,
"slot_length":
duration(seconds=settings['slot_length'].seconds)
if 'slot_length' in settings else GlobalConfig.slot_length,
"tick_length":
duration(seconds=settings['tick_length'].seconds)
if 'tick_length' in settings else GlobalConfig.tick_length,
"market_maker_rate":
settings.get(
'market_maker_rate',
str(ConstSettings.GeneralSettings.DEFAULT_MARKET_MAKER_RATE)),
"market_count":
settings.get('market_count', GlobalConfig.market_count),
"cloud_coverage":
settings.get('cloud_coverage', GlobalConfig.cloud_coverage),
"pv_user_profile":
settings.get('pv_user_profile', None),
"max_panel_power_W":
settings.get('max_panel_power_W',
ConstSettings.PVSettings.MAX_PANEL_OUTPUT_W),
"grid_fee_type":
settings.get('grid_fee_type', GlobalConfig.grid_fee_type),
"external_connection_enabled":
settings.get('external_connection_enabled', False),
"aggregator_device_mapping":
aggregator_device_mapping
}
validate_global_settings(config_settings)
config = SimulationConfig(**config_settings)
spot_market_type = settings.get('spot_market_type', None)
if spot_market_type is not None:
ConstSettings.IAASettings.MARKET_TYPE = spot_market_type
if scenario is None:
scenario_name = "default_2a"
elif scenario in available_simulation_scenarios:
scenario_name = scenario
else:
scenario_name = 'json_arg'
config.area = scenario
kwargs = {
"no_export": True,
"pricing_scheme": 0,
"seed": settings.get('random_seed', 0)
}
slowdown_factor = environ.get('D3A_SLOWDOWN_FACTOR', None)
if slowdown_factor is None:
slowdown_factor = settings.get('slowdown', 0)
else:
slowdown_factor = float(slowdown_factor)
run_simulation(setup_module_name=scenario_name,
simulation_config=config,
simulation_events=events,
slowdown=slowdown_factor,
redis_job_id=job.id,
kwargs=kwargs)
except Exception:
import traceback
from d3a.d3a_core.redis_connections.redis_communication import publish_job_error_output
publish_job_error_output(job.id, traceback.format_exc())
logging.getLogger().error(
f"Error on jobId {job.id}: {traceback.format_exc()}")
def launch_simulation_from_rq_job(scenario, settings, events,
aggregator_device_mapping, saved_state,
job_id):
logging.getLogger().setLevel(logging.ERROR)
scenario = decompress_and_decode_queued_strings(scenario)
if "collaboration_uuid" in scenario:
d3a.constants.COLLABORATION_ID = scenario.pop("collaboration_uuid")
d3a.constants.EXTERNAL_CONNECTION_WEB = True
GlobalConfig.IS_CANARY_NETWORK = scenario.pop("is_canary_network",
False)
d3a.constants.RUN_IN_REALTIME = GlobalConfig.IS_CANARY_NETWORK
saved_state = decompress_and_decode_queued_strings(saved_state)
log.error(f"Starting simulation with job_id: {job_id}")
try:
if settings is None:
settings = {}
else:
settings = {
k: v
for k, v in settings.items() if v is not None and v != "None"
}
advanced_settings = settings.get('advanced_settings', None)
if advanced_settings is not None:
update_advanced_settings(ast.literal_eval(advanced_settings))
aggregator_device_mapping = json.loads(aggregator_device_mapping)
if events is not None:
events = ast.literal_eval(events)
config_settings = {
"start_date":
instance(
datetime.combine(settings.get('start_date'),
datetime.min.time()))
if 'start_date' in settings else GlobalConfig.start_date,
"sim_duration":
duration(days=settings['duration'].days)
if 'duration' in settings else GlobalConfig.sim_duration,
"slot_length":
duration(seconds=settings['slot_length'].seconds)
if 'slot_length' in settings else GlobalConfig.slot_length,
"tick_length":
duration(seconds=settings['tick_length'].seconds)
if 'tick_length' in settings else GlobalConfig.tick_length,
"market_maker_rate":
settings.get(
'market_maker_rate',
str(ConstSettings.GeneralSettings.DEFAULT_MARKET_MAKER_RATE)),
"market_count":
settings.get('market_count', GlobalConfig.market_count),
"cloud_coverage":
settings.get('cloud_coverage', GlobalConfig.cloud_coverage),
"pv_user_profile":
settings.get('pv_user_profile', None),
"max_panel_power_W":
settings.get('max_panel_power_W',
ConstSettings.PVSettings.MAX_PANEL_OUTPUT_W),
"grid_fee_type":
settings.get('grid_fee_type', GlobalConfig.grid_fee_type),
"external_connection_enabled":
settings.get('external_connection_enabled', False),
"aggregator_device_mapping":
aggregator_device_mapping
}
if GlobalConfig.IS_CANARY_NETWORK:
config_settings['start_date'] = \
instance((datetime.combine(date.today(), datetime.min.time())))
validate_global_settings(config_settings)
slot_length_realtime = duration(seconds=settings['slot_length_realtime'].seconds) \
if 'slot_length_realtime' in settings else None
config = SimulationConfig(**config_settings)
spot_market_type = settings.get('spot_market_type', None)
if spot_market_type is not None:
if spot_market_type == SpotMarketTypeEnum.ONE_SIDED.value:
ConstSettings.IAASettings.MARKET_TYPE = SpotMarketTypeEnum.ONE_SIDED.value
if spot_market_type == SpotMarketTypeEnum.TWO_SIDED_PAY_AS_BID.value:
ConstSettings.IAASettings.MARKET_TYPE = (
SpotMarketTypeEnum.TWO_SIDED_PAY_AS_BID.value)
ConstSettings.IAASettings.BID_OFFER_MATCH_TYPE = (
BidOfferMatchAlgoEnum.PAY_AS_BID.value)
if spot_market_type == SpotMarketTypeEnum.TWO_SIDED_PAY_AS_CLEAR.value:
ConstSettings.IAASettings.MARKET_TYPE = (
SpotMarketTypeEnum.TWO_SIDED_PAY_AS_BID.value)
ConstSettings.IAASettings.BID_OFFER_MATCH_TYPE = \
BidOfferMatchAlgoEnum.PAY_AS_CLEAR.value
if spot_market_type == SpotMarketTypeEnum.TWO_SIDED_EXTERNAL.value:
ConstSettings.IAASettings.MARKET_TYPE = (
SpotMarketTypeEnum.TWO_SIDED_PAY_AS_BID.value)
ConstSettings.IAASettings.BID_OFFER_MATCH_TYPE = \
BidOfferMatchAlgoEnum.EXTERNAL.value
if scenario is None:
scenario_name = "default_2a"
elif scenario in available_simulation_scenarios:
scenario_name = scenario
else:
scenario_name = 'json_arg'
config.area = scenario
kwargs = {
"no_export": True,
"pricing_scheme": 0,
"seed": settings.get('random_seed', 0)
}
run_simulation(setup_module_name=scenario_name,
simulation_config=config,
simulation_events=events,
redis_job_id=job_id,
saved_sim_state=saved_state,
slot_length_realtime=slot_length_realtime,
kwargs=kwargs)
except Exception:
import traceback
from d3a.d3a_core.redis_connections.redis_communication import publish_job_error_output
publish_job_error_output(job_id, traceback.format_exc())
logging.getLogger().error(
f"Error on jobId {job_id}: {traceback.format_exc()}")
