def test_all_units_and_service_dispatch_historically_present_in_market():
con = sqlite3.connect(test_db)
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals(number=1000):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market = market_builder.get_market_object()
market_checker = historical_market_builder.MarketChecker(market=market,
mms_db=mms_database,
xml_cache=xml_cache,
interval=interval)
assert market_checker.all_dispatch_units_and_services_have_decision_variables()
def test_hist_dispatch_values_meet_demand():
con = sqlite3.connect(test_db)
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals(number=100):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.add_interconnectors_to_market()
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(market=market,
mms_db=mms_database,
interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_overrider.set_interconnector_flow_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(market=market,
mms_db=mms_database,
xml_cache=xml_cache,
interval=interval)
test_passed = market_checker.is_regional_demand_meet()
assert test_passed
con.close()
def ignore_test_find_fcas_trapezium_scaled_availability_erros():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db'
)
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020'
)
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
outputs = []
for interval in get_test_intervals_august_2020(number=100):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
unit_inputs.get_processed_bids()
unit_inputs.add_fcas_trapezium_constraints()
traps = unit_inputs.get_fcas_regulation_trapeziums()
traps = traps[traps['service'] == 'lower_reg']
avails = mms_database.DISPATCHLOAD.get_data(interval)
avails = avails.loc[:, [
'DUID', 'TOTALCLEARED', 'LOWERREG', 'LOWERREGACTUALAVAILABILITY'
]]
avails.columns = [
'unit', 'total_cleared', 'lower_reg',
'lower_reg_actual_availability'
]
avails = avails[avails['lower_reg'] >
avails['lower_reg_actual_availability'] + 0.1]
avails = pd.merge(avails, traps, on='unit')
avails['time'] = interval
outputs.append(avails)
pd.concat(outputs).to_csv('avails_august_2020.csv')
def test_capacity_constraint_where_constraints_violated():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db')
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache')
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
with open('interval_with_violations.pickle', 'rb') as f:
interval_with_violations = pickle.load(f)
tests_to_run = 10
tests_run = 0
for interval, types in interval_with_violations.items():
if tests_run == tests_to_run:
break
if 'unit_capacity' in types:
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(
raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(
unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.add_interconnectors_to_market()
market_builder.set_unit_limit_constraints()
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(
market=market, mms_db=mms_database, interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_overrider.set_interconnector_flow_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(
market=market,
mms_db=mms_database,
xml_cache=xml_cache_manager,
interval=interval)
assert market_checker.measured_violation_equals_historical_violation(
'unit_capacity', nempy_constraints=['unit_bid_capacity'])
tests_run += 1
assert tests_to_run == tests_run
def ignore_test_fcas_trapezium_scaled_availability():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db'
)
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020'
)
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals_august_2020(number=10):
if interval != '2020/08/21 13:00:00':
continue
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(
raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(
unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.set_unit_fcas_constraints()
market_builder.set_unit_limit_constraints()
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(
market=market, mms_db=mms_database, interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(
market=market,
mms_db=mms_database,
xml_cache=xml_cache,
interval=interval,
unit_inputs=unit_inputs)
avails = market_checker.do_fcas_availabilities_match_historical()
# I think NEMDE might be getting avail calcs wrong when units are operating on the slopes, and the slopes
# are vertical. They should be ignore 0 slope coefficients, maybe this is not happening because of floating
# point comparison.
if interval == '2019/01/29 18:10:00':
avails = avails[~(avails['unit'] == 'PPCCGT')]
if interval == '2019/01/07 19:35:00':
avails = avails[~(avails['unit'] == 'PPCCGT')]
def _test_setup():
import sqlite3
from nempy.historical_inputs import mms_db
from nempy.historical_inputs import xml_cache
from nempy.historical_inputs import loaders
con = sqlite3.connect('market_management_system.db')
mms_db_manager = mms_db.DBManager(connection=con)
xml_cache_manager = xml_cache.XMLCacheManager('test_nemde_cache')
inputs_loader = loaders.RawInputsLoader(xml_cache_manager, mms_db_manager)
inputs_loader.set_interval('2019/01/01 00:00:00')
return inputs_loader
def test_slack_in_generic_constraints_with_fcas_interface():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db')
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache')
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals(number=100):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(
raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(
unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.add_interconnectors_to_market()
market_builder.add_generic_constraints_with_fcas_requirements_interface(
)
market_builder.set_unit_fcas_constraints()
market_builder.set_unit_limit_constraints()
market_builder.set_region_demand_constraints()
market_builder.set_ramp_rate_limits()
market_builder.set_fast_start_constraints()
market_builder.set_solver('CBC')
market_builder.dispatch(calc_prices=True)
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(
market=market, mms_db=mms_database, interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_overrider.set_interconnector_flow_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(
market=market,
mms_db=mms_database,
xml_cache=xml_cache,
interval=interval)
assert market_checker.is_generic_constraint_slack_correct()
assert market_checker.is_fcas_constraint_slack_correct()
assert market_checker.is_regional_demand_meet()
def test_against_100_interval_benchmark():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db')
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache')
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
outputs = []
for interval in get_test_intervals(number=100):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(
raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(
unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.add_interconnectors_to_market()
market_builder.add_generic_constraints_with_fcas_requirements_interface(
)
market_builder.set_unit_fcas_constraints()
market_builder.set_unit_limit_constraints()
market_builder.set_region_demand_constraints()
market_builder.set_ramp_rate_limits()
market_builder.set_fast_start_constraints()
market_builder.set_solver('GUROBI')
market_builder.dispatch(calc_prices=True)
market = market_builder.get_market_object()
market_checker = historical_market_builder.MarketChecker(
market=market,
mms_db=mms_database,
xml_cache=xml_cache,
interval=interval)
price_comp = market_checker.get_price_comparison()
outputs.append(price_comp)
outputs = pd.concat(outputs)
outputs.to_csv('latest_100_interval_run.csv', index=False)
benchmark = pd.read_csv('100_interval_benchmark.csv')
assert_frame_equal(outputs.reset_index(drop=True),
benchmark,
check_exact=False,
atol=1e-2)
def test_capacity_constraints():
con = sqlite3.connect(
'/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db')
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(
'/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache')
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals(number=10):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(
raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(
unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.add_interconnectors_to_market()
market_builder.set_unit_limit_constraints()
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(
market=market, mms_db=mms_database, interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_overrider.set_interconnector_flow_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(
market=market,
mms_db=mms_database,
xml_cache=xml_cache_manager,
interval=interval)
assert market_checker.measured_violation_equals_historical_violation(
'unit_capacity', nempy_constraints=['unit_bid_capacity'])
def test_ramp_rate_constraints():
con = sqlite3.connect(test_db)
mms_database = mms_db.DBManager(con)
xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_database)
for interval in get_test_intervals(number=10):
raw_inputs_loader.set_interval(interval)
unit_inputs = units.UnitData(raw_inputs_loader)
interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
demand_inputs = demand.DemandData(raw_inputs_loader)
market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
interconnector_inputs=interconnector_inputs,
constraint_inputs=constraint_inputs,
demand_inputs=demand_inputs)
market_builder.add_unit_bids_to_market()
market_builder.set_ramp_rate_limits()
market = market_builder.get_market_object()
market_overrider = historical_market_builder.MarketOverrider(market=market,
mms_db=mms_database,
interval=interval)
market_overrider.set_unit_dispatch_to_historical_values()
market_builder.dispatch()
market_checker = historical_market_builder.MarketChecker(market=market,
mms_db=mms_database,
xml_cache=xml_cache_manager,
interval=interval)
assert market_checker.measured_violation_equals_historical_violation(historical_name='ramp_rate',
nempy_constraints=['ramp_up', 'ramp_down'])
if download_inputs:
# This requires approximately 5 GB of storage.
mms_db_manager.populate(start_year=2019,
start_month=1,
end_year=2019,
end_month=1)
# This requires approximately 60 GB of storage.
xml_cache_manager.populate(start_year=2019,
start_month=1,
end_year=2019,
end_month=1)
raw_inputs_loader = loaders.RawInputsLoader(
nemde_xml_cache_manager=xml_cache_manager,
market_management_system_database=mms_db_manager)
# A list of intervals we want to recreate historical dispatch for.
dispatch_intervals = [
'2019/01/01 12:00:00', '2019/01/01 12:05:00', '2019/01/01 12:10:00',
'2019/01/01 12:15:00', '2019/01/01 12:20:00', '2019/01/01 12:25:00',
'2019/01/01 12:30:00'
]
# List for saving outputs to.
outputs = []
# Create and dispatch the spot market for each dispatch interval.
for interval in dispatch_intervals:
raw_inputs_loader.set_interval(interval)
