def test_energy_bills_accumulate_fees(grid_fees):
ConstSettings.GeneralSettings.KEEP_PAST_MARKETS = True
m_bills = MarketEnergyBills()
m_bills._update_market_fees(grid_fees, 'past_markets')
assert m_bills.market_fees['house2'] == 0.03
assert m_bills.market_fees['street'] == 0.05
assert m_bills.market_fees['house1'] == 0.08
def __init__(self, job_id, initial_params, area):
self.job_id = job_id
self.random_seed = initial_params["seed"] if initial_params[
"seed"] is not None else ''
self.status = {}
self.eta = duration(seconds=0)
self.unmatched_loads = {}
self.unmatched_loads_redis = {}
self.export_unmatched_loads = ExportUnmatchedLoads(area)
self.market_unmatched_loads = MarketUnmatchedLoads()
self.cumulative_loads = {}
self.price_energy_day = MarketPriceEnergyDay()
self.cumulative_grid_trades = {}
self.accumulated_trades = {}
self.accumulated_trades_redis = {}
self.accumulated_balancing_trades = {}
self.cumulative_grid_trades_redis = {}
self.cumulative_grid_balancing_trades = {}
self.tree_summary = {}
self.tree_summary_redis = {}
self.market_bills = MarketEnergyBills()
self.balancing_bills = MarketEnergyBills(is_spot_market=False)
self.trade_details = {}
self.device_statistics = DeviceStatistics()
self.energy_trade_profile = {}
self.energy_trade_profile_redis = {}
self.file_export_endpoints = FileExportEndpoints()
def test_energy_bills_use_only_last_market_if_not_keep_past_markets(grid_fees):
ConstSettings.GeneralSettings.KEEP_PAST_MARKETS = False
m_bills = MarketEnergyBills()
m_bills._update_market_fees(grid_fees, 'past_markets')
assert m_bills.market_fees['house2'] == 0.03
assert m_bills.market_fees['street'] == 0.01
assert m_bills.market_fees['house1'] == 0.06
def __init__(self, job_id, initial_params, area, export_plots=True):
self.job_id = job_id
self.current_market = ""
self.random_seed = initial_params["seed"] if initial_params[
"seed"] is not None else ''
self.status = {}
self.simulation_progress = {
"eta_seconds": 0,
"elapsed_time_seconds": 0,
"percentage_completed": 0
}
self.market_unmatched_loads = MarketUnmatchedLoads(area)
self.price_energy_day = MarketPriceEnergyDay()
self.market_bills = MarketEnergyBills()
self.cumulative_bills = CumulativeBills()
self.balancing_bills = MarketEnergyBills(is_spot_market=False)
self.cumulative_grid_trades = CumulativeGridTrades()
self.device_statistics = DeviceStatistics()
self.file_export_endpoints = FileExportEndpoints(export_plots)
self.kpi = KPI()
self.area_throughput_stats = AreaThroughputStats()
self.last_unmatched_loads = {}
self.bids_offers_trades = {}
self.export_plots = export_plots
if export_plots:
self.area_market_stocks_stats = OfferBidTradeGraphStats()
def test_energy_bills_report_correctly_market_fees(grid_fees):
ConstSettings.GeneralSettings.KEEP_PAST_MARKETS = True
m_bills = MarketEnergyBills()
m_bills.update(grid_fees)
result = m_bills.bills_results
assert result["street"]["house1"]["market_fee"] == 0.04
assert result["street"]["house2"]["market_fee"] == 0.01
assert result["street"]['Accumulated Trades']["market_fee"] == 0.05
assert result["house1"]['External Trades']["market_fee"] == 0.0
assert result["house2"]['External Trades']["market_fee"] == 0.0
def test_energy_bills_last_past_market(grid):
ConstSettings.GeneralSettings.KEEP_PAST_MARKETS = False
m_bills = MarketEnergyBills()
m_bills.update(grid)
result = m_bills.bills_results
assert result['house2']['Accumulated Trades']['bought'] == result['commercial']['sold'] == 1
assert result['house2']['Accumulated Trades']['spent'] == \
result['commercial']['earned'] == \
0.01
assert result['commercial']['spent'] == result['commercial']['bought'] == 0
assert result['fridge']['bought'] == 2 and isclose(result['fridge']['spent'], 0.01)
assert result['pv']['sold'] == 2 and isclose(result['pv']['earned'], 0.01)
assert 'children' not in result
class SimulationEndpointBuffer:
def __init__(self, job_id, initial_params, area):
self.job_id = job_id
self.random_seed = initial_params["seed"] if initial_params[
"seed"] is not None else ''
self.status = {}
self.eta = duration(seconds=0)
self.unmatched_loads = {}
self.unmatched_loads_redis = {}
self.export_unmatched_loads = ExportUnmatchedLoads(area)
self.market_unmatched_loads = MarketUnmatchedLoads()
self.cumulative_loads = {}
self.price_energy_day = MarketPriceEnergyDay()
self.cumulative_grid_trades = {}
self.accumulated_trades = {}
self.accumulated_trades_redis = {}
self.accumulated_balancing_trades = {}
self.cumulative_grid_trades_redis = {}
self.cumulative_grid_balancing_trades = {}
self.tree_summary = {}
self.tree_summary_redis = {}
self.market_bills = MarketEnergyBills()
self.balancing_bills = MarketEnergyBills(is_spot_market=False)
self.trade_details = {}
self.device_statistics = DeviceStatistics()
self.energy_trade_profile = {}
self.energy_trade_profile_redis = {}
self.file_export_endpoints = FileExportEndpoints()
def generate_result_report(self):
return {
"job_id": self.job_id,
"random_seed": self.random_seed,
"unmatched_loads": self.unmatched_loads_redis,
"cumulative_loads": self.cumulative_loads,
"price_energy_day": self.price_energy_day.redis_output,
"cumulative_grid_trades": self.cumulative_grid_trades_redis,
"bills": self.market_bills.bills_redis_results,
"tree_summary": self.tree_summary_redis,
"status": self.status,
"eta_seconds": self.eta.seconds,
"device_statistics": self.device_statistics.flat_results_time_str,
"energy_trade_profile": self.energy_trade_profile_redis
}
def generate_json_report(self):
return {
"job_id": self.job_id,
"random_seed": self.random_seed,
"unmatched_loads": self.unmatched_loads,
"cumulative_loads": self.cumulative_loads,
"price_energy_day": self.price_energy_day.csv_output,
"cumulative_grid_trades": self.cumulative_grid_trades,
"bills": self.market_bills.bills_results,
"tree_summary": self.tree_summary,
"status": self.status,
"device_statistics": self.device_statistics.device_stats_time_str,
"energy_trade_profile": self.energy_trade_profile
}
def _update_unmatched_loads(self, area):
if self.export_unmatched_loads.load_count == 0:
self.unmatched_loads, self.unmatched_loads_redis =\
self.market_unmatched_loads.write_none_to_unmatched_loads(area, {}, {})
else:
current_results, current_results_uuid = \
self.export_unmatched_loads.get_current_market_results(
all_past_markets=ConstSettings.GeneralSettings.KEEP_PAST_MARKETS)
if ConstSettings.GeneralSettings.KEEP_PAST_MARKETS:
self.unmatched_loads = current_results
self.unmatched_loads_redis = current_results_uuid
else:
self.unmatched_loads, self.unmatched_loads_redis = \
self.market_unmatched_loads.update_and_get_unmatched_loads(
current_results, current_results_uuid)
def _update_cumulative_grid_trades(self, area):
market_type = \
"past_markets" if ConstSettings.GeneralSettings.KEEP_PAST_MARKETS else "current_market"
balancing_market_type = "past_balancing_markets" \
if ConstSettings.GeneralSettings.KEEP_PAST_MARKETS \
else "current_balancing_market"
if ConstSettings.GeneralSettings.KEEP_PAST_MARKETS:
self.accumulated_trades = {}
self.accumulated_trades_redis = {}
self.accumulated_balancing_trades = {}
self.accumulated_trades_redis, self.cumulative_grid_trades_redis = \
export_cumulative_grid_trades_redis(area, self.accumulated_trades_redis,
market_type)
self.accumulated_trades, self.cumulative_grid_trades = \
export_cumulative_grid_trades(area, self.accumulated_trades,
market_type, all_devices=True)
self.accumulated_balancing_trades, self.cumulative_grid_balancing_trades = \
export_cumulative_grid_trades(area, self.accumulated_balancing_trades,
balancing_market_type)
def update_stats(self, area, simulation_status, eta):
self.status = simulation_status
self.eta = eta
self._update_unmatched_loads(area)
# Should always precede tree-summary update
self.price_energy_day.update(area)
self.cumulative_loads = {
"price-currency": "Euros",
"load-unit": "kWh",
"cumulative-load-price": export_cumulative_loads(area)
}
self._update_cumulative_grid_trades(area)
self.market_bills.update(area)
self.balancing_bills.update(area)
self._update_tree_summary(area)
self.trade_details = generate_inter_area_trade_details(
area, "past_markets")
self.device_statistics.update(area)
self.file_export_endpoints(area)
self.energy_trade_profile = self.file_export_endpoints.traded_energy_profile
self.energy_trade_profile_redis = self._round_energy_trade_profile(
self.file_export_endpoints.traded_energy_profile_redis)
def _update_tree_summary(self, area):
price_energy_list = self.price_energy_day.csv_output
def calculate_prices(key, functor):
if area.name not in price_energy_list:
return 0.
energy_prices = [
price_energy[key] for price_energy in price_energy_list[
area.name]["price-energy-day"]
]
return round(functor(energy_prices),
2) if len(energy_prices) > 0 else 0.0
self.tree_summary[area.slug] = {
"min_trade_price": calculate_prices("min_price", min),
"max_trade_price": calculate_prices("max_price", max),
"avg_trade_price": calculate_prices("av_price", mean),
}
self.tree_summary_redis[area.uuid] = self.tree_summary[area.slug]
for child in area.children:
if child.children:
self._update_tree_summary(child)
@classmethod
def _round_energy_trade_profile(cls, profile):
for k in profile.keys():
for sold_bought in ['sold_energy', 'bought_energy']:
for dev in profile[k][sold_bought].keys():
for target in profile[k][sold_bought][dev].keys():
for timestamp in profile[k][sold_bought][dev][
target].keys():
profile[k][sold_bought][dev][target][
timestamp] = round_floats_for_ui(
profile[k][sold_bought][dev][target]
[timestamp])
return profile
class SimulationEndpointBuffer:
def __init__(self, job_id, initial_params, area):
self.job_id = job_id
self.current_market = ""
self.random_seed = initial_params["seed"] if initial_params[
"seed"] is not None else ''
self.status = {}
self.eta = duration(seconds=0)
self.market_unmatched_loads = MarketUnmatchedLoads(area)
self.cumulative_loads = {}
self.price_energy_day = MarketPriceEnergyDay()
self.tree_summary = TreeSummary()
self.market_bills = MarketEnergyBills()
self.balancing_bills = MarketEnergyBills(is_spot_market=False)
self.cumulative_grid_trades = CumulativeGridTrades()
self.trade_details = {}
self.device_statistics = DeviceStatistics()
self.file_export_endpoints = FileExportEndpoints()
self.kpi = KPI()
self.last_unmatched_loads = {}
def generate_result_report(self):
redis_results = {
"job_id": self.job_id,
"current_market": self.current_market,
"random_seed": self.random_seed,
"cumulative_loads": self.cumulative_loads,
"cumulative_grid_trades":
self.cumulative_grid_trades.current_trades_redis,
"bills": self.market_bills.bills_redis_results,
"tree_summary": self.tree_summary.current_results_redis,
"status": self.status,
"eta_seconds": self.eta.seconds,
"kpi": self.kpi.performance_indices
}
if ConstSettings.GeneralSettings.REDIS_PUBLISH_FULL_RESULTS:
redis_results.update({
"unmatched_loads":
self.market_unmatched_loads.unmatched_loads_uuid,
"price_energy_day":
self.price_energy_day.redis_output,
"device_statistics":
self.device_statistics.flat_stats_time_str,
"energy_trade_profile":
self.file_export_endpoints.traded_energy_profile_redis,
})
else:
redis_results.update({
"last_unmatched_loads":
self.market_unmatched_loads.last_unmatched_loads,
"last_energy_trade_profile":
self.file_export_endpoints.traded_energy_current,
"last_price_energy_day":
self.price_energy_day.redis_output,
"last_device_statistics":
self.device_statistics.current_stats_time_str
})
return redis_results
def generate_json_report(self):
return {
"job_id": self.job_id,
"random_seed": self.random_seed,
"unmatched_loads": self.market_unmatched_loads.unmatched_loads,
"cumulative_loads": self.cumulative_loads,
"price_energy_day": self.price_energy_day.csv_output,
"cumulative_grid_trades":
self.cumulative_grid_trades.current_trades_redis,
"bills": self.market_bills.bills_results,
"tree_summary": self.tree_summary.current_results,
"status": self.status,
"device_statistics": self.device_statistics.device_stats_time_str,
"energy_trade_profile":
self.file_export_endpoints.traded_energy_profile,
"kpi": self.kpi.performance_indices
}
def update_stats(self, area, simulation_status, eta):
self.status = simulation_status
if area.current_market is not None:
self.current_market = area.current_market.time_slot_str
self.eta = eta
self.cumulative_loads = export_cumulative_loads(area)
self.cumulative_grid_trades.update(area)
self.market_bills.update(area)
self.balancing_bills.update(area)
self.trade_details = generate_inter_area_trade_details(
area, "past_markets")
self.file_export_endpoints(area)
self.market_unmatched_loads.update_unmatched_loads(area)
self.device_statistics.update(area)
self._update_price_energy_day_tree_summary(area)
self.generate_result_report()
self.kpi.update_kpis_from_area(area)
self.update_area_aggregated_stats(area)
def update_area_aggregated_stats(self, area):
self._update_area_stats(area)
for child in area.children:
self.update_area_aggregated_stats(child)
def _update_area_stats(self, area):
area.stats.update_aggregated_stats({
"simulation_id":
self.job_id,
"status":
self.status,
"bills":
self.market_bills.bills_redis_results[area.uuid],
"cumulative_grid_trades":
self.cumulative_grid_trades.accumulated_trades_redis.get(
area.uuid, None),
"tree_summary":
self.tree_summary.current_results.get(area.slug, None),
"unmatched_loads":
self.market_unmatched_loads.unmatched_loads.get(area.name, None),
"price_energy_day":
self.price_energy_day.csv_output.get(area.name, None),
"device_statistics":
self.device_statistics.device_stats_time_str.get(area.uuid, None),
"energy_trade_profile":
self.file_export_endpoints.traded_energy_profile.get(
area.slug, None),
"kpi":
self.kpi.performance_indices.get(area.name, None)
})
def _update_price_energy_day_tree_summary(self, area):
# Update of the price_energy_day endpoint should always precede tree-summary.
# The reason is that the price_energy_day data are used when calculating the
# tree-summary data.
self.price_energy_day.update(area)
self.tree_summary.update(area, self.price_energy_day.csv_output)
def test_energy_bills_ensure_device_types_are_populated(grid2):
m_bills = MarketEnergyBills()
m_bills.update(grid2)
result = m_bills.bills_results
assert result["house1"]["type"] == "House 1 type"
assert result["house2"]["type"] == "House 2 type"
def test_energy_bills_finds_iaas(grid2):
m_bills = MarketEnergyBills()
m_bills.update(grid2)
result = m_bills.bills_results
assert result['house1']['bought'] == result['house2']['sold'] == 3
class SimulationEndpointBuffer:
def __init__(self, job_id, initial_params, area, export_plots=True):
self.job_id = job_id
self.current_market = ""
self.random_seed = initial_params["seed"] if initial_params[
"seed"] is not None else ''
self.status = {}
self.simulation_progress = {
"eta_seconds": 0,
"elapsed_time_seconds": 0,
"percentage_completed": 0
}
self.market_unmatched_loads = MarketUnmatchedLoads(area)
self.price_energy_day = MarketPriceEnergyDay()
self.market_bills = MarketEnergyBills()
self.cumulative_bills = CumulativeBills()
self.balancing_bills = MarketEnergyBills(is_spot_market=False)
self.cumulative_grid_trades = CumulativeGridTrades()
self.device_statistics = DeviceStatistics()
self.file_export_endpoints = FileExportEndpoints(export_plots)
self.kpi = KPI()
self.area_throughput_stats = AreaThroughputStats()
self.last_unmatched_loads = {}
self.bids_offers_trades = {}
self.export_plots = export_plots
if export_plots:
self.area_market_stocks_stats = OfferBidTradeGraphStats()
def generate_result_report(self):
# TODO: In D3ASIM-2288, add unix_time=True to convert_pendulum_to_str_in_dict
return {
"job_id":
self.job_id,
"current_market":
self.current_market,
"random_seed":
self.random_seed,
"cumulative_grid_trades":
self.cumulative_grid_trades.current_trades_redis,
"bills":
self.market_bills.bills_redis_results,
"cumulative_bills":
self.cumulative_bills.cumulative_bills,
"status":
self.status,
"progress_info":
self.simulation_progress,
"kpi":
self.kpi.performance_indices_redis,
"last_unmatched_loads":
convert_pendulum_to_str_in_dict(
self.market_unmatched_loads.last_unmatched_loads, {}),
"last_energy_trade_profile":
convert_pendulum_to_str_in_dict(
self.file_export_endpoints.traded_energy_current, {},
ui_format=True),
"last_price_energy_day":
convert_pendulum_to_str_in_dict(self.price_energy_day.redis_output,
{}),
"last_device_statistics":
convert_pendulum_to_str_in_dict(
self.device_statistics.current_stats_dict, {}),
"area_throughput":
self.area_throughput_stats.results_redis,
"last_energy_trades_high_resolution":
convert_pendulum_to_str_in_dict(
self.file_export_endpoints.last_energy_trades_high_resolution,
{}),
"bids_offers_trades":
self.bids_offers_trades
}
def generate_json_report(self):
return {
"job_id":
self.job_id,
"random_seed":
self.random_seed,
"unmatched_loads":
convert_pendulum_to_str_in_dict(
self.market_unmatched_loads.unmatched_loads, {}),
"price_energy_day":
convert_pendulum_to_str_in_dict(self.price_energy_day.csv_output,
{}),
"cumulative_grid_trades":
self.cumulative_grid_trades.current_trades_redis,
"bills":
self.market_bills.bills_results,
"cumulative_bills":
self.cumulative_bills.cumulative_bills,
"status":
self.status,
"progress_info":
self.simulation_progress,
"device_statistics":
convert_pendulum_to_str_in_dict(
self.device_statistics.device_stats_dict, {}),
"energy_trade_profile":
convert_pendulum_to_str_in_dict(
self.file_export_endpoints.traded_energy_profile, {},
ui_format=True),
"kpi":
self.kpi.performance_indices,
"area_throughput":
self.area_throughput_stats.results,
}
def update_stats(self, area, simulation_status, progress_info):
self.status = simulation_status
if area.current_market is not None:
self.current_market = area.current_market.time_slot_str
self.simulation_progress = {
"eta_seconds": progress_info.eta.seconds,
"elapsed_time_seconds": progress_info.elapsed_time.seconds,
"percentage_completed": int(progress_info.percentage_completed)
}
self.cumulative_grid_trades.update(area)
self.market_bills.update(area)
if ConstSettings.BalancingSettings.ENABLE_BALANCING_MARKET:
self.balancing_bills.update(area)
self.cumulative_bills.update_cumulative_bills(area)
self.file_export_endpoints(area)
self.market_unmatched_loads.update_unmatched_loads(area)
self.device_statistics.update(area)
self.price_energy_day.update(area)
self.kpi.update_kpis_from_area(area)
self.area_throughput_stats.update(area)
self.generate_result_report()
self.bids_offers_trades.clear()
self.update_area_aggregated_stats(area)
if self.export_plots:
self.area_market_stocks_stats.update(area)
def _send_results_to_areas(self, area):
stats = {
"kpi": self.kpi.performance_indices_redis.get(area.uuid, None)
}
area.endpoint_stats.update(stats)
def update_area_aggregated_stats(self, area):
self._update_area_stats(area)
self._send_results_to_areas(area)
for child in area.children:
self.update_area_aggregated_stats(child)
def _update_area_stats(self, area):
if area.current_market is not None:
self.bids_offers_trades[
area.uuid] = area.current_market.get_bids_offers_trades()
bills = self.market_bills.bills_redis_results[area.uuid]
bills.update({
"penalty_cost":
self.cumulative_bills.cumulative_bills_results[
area.uuid]["penalties"],
"penalty_energy":
self.cumulative_bills.cumulative_bills_results[area.uuid]
["penalty_energy"]
})
area.stats.update_aggregated_stats({"bills": bills})
