def test_hist_dispatch_values_meet_demand(): con = sqlite3.connect(test_db) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache) raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals(number=100): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.add_interconnectors_to_market() market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider(market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_overrider.set_interconnector_flow_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker(market=market, mms_db=mms_database, xml_cache=xml_cache, interval=interval) test_passed = market_checker.is_regional_demand_meet() assert test_passed con.close()
def ignore_test_find_fcas_trapezium_scaled_availability_erros(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db' ) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020' ) raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) outputs = [] for interval in get_test_intervals_august_2020(number=100): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) unit_inputs.get_processed_bids() unit_inputs.add_fcas_trapezium_constraints() traps = unit_inputs.get_fcas_regulation_trapeziums() traps = traps[traps['service'] == 'lower_reg'] avails = mms_database.DISPATCHLOAD.get_data(interval) avails = avails.loc[:, [ 'DUID', 'TOTALCLEARED', 'LOWERREG', 'LOWERREGACTUALAVAILABILITY' ]] avails.columns = [ 'unit', 'total_cleared', 'lower_reg', 'lower_reg_actual_availability' ] avails = avails[avails['lower_reg'] > avails['lower_reg_actual_availability'] + 0.1] avails = pd.merge(avails, traps, on='unit') avails['time'] = interval outputs.append(avails) pd.concat(outputs).to_csv('avails_august_2020.csv')
def test_all_units_and_service_dispatch_historically_present_in_market(): con = sqlite3.connect(test_db) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache) raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals(number=1000): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market = market_builder.get_market_object() market_checker = historical_market_builder.MarketChecker(market=market, mms_db=mms_database, xml_cache=xml_cache, interval=interval) assert market_checker.all_dispatch_units_and_services_have_decision_variables()
def test_capacity_constraint_where_constraints_violated(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db') mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache') raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) with open('interval_with_violations.pickle', 'rb') as f: interval_with_violations = pickle.load(f) tests_to_run = 10 tests_run = 0 for interval, types in interval_with_violations.items(): if tests_run == tests_to_run: break if 'unit_capacity' in types: raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData( raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder( unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.add_interconnectors_to_market() market_builder.set_unit_limit_constraints() market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider( market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_overrider.set_interconnector_flow_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker( market=market, mms_db=mms_database, xml_cache=xml_cache_manager, interval=interval) assert market_checker.measured_violation_equals_historical_violation( 'unit_capacity', nempy_constraints=['unit_bid_capacity']) tests_run += 1 assert tests_to_run == tests_run
def ignore_test_fcas_trapezium_scaled_availability(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db' ) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020' ) raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals_august_2020(number=10): if interval != '2020/08/21 13:00:00': continue raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData( raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder( unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.set_unit_fcas_constraints() market_builder.set_unit_limit_constraints() market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider( market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker( market=market, mms_db=mms_database, xml_cache=xml_cache, interval=interval, unit_inputs=unit_inputs) avails = market_checker.do_fcas_availabilities_match_historical() # I think NEMDE might be getting avail calcs wrong when units are operating on the slopes, and the slopes # are vertical. They should be ignore 0 slope coefficients, maybe this is not happening because of floating # point comparison. if interval == '2019/01/29 18:10:00': avails = avails[~(avails['unit'] == 'PPCCGT')] if interval == '2019/01/07 19:35:00': avails = avails[~(avails['unit'] == 'PPCCGT')]
def _test_setup(): import sqlite3 from nempy.historical_inputs import mms_db from nempy.historical_inputs import xml_cache from nempy.historical_inputs import loaders con = sqlite3.connect('market_management_system.db') mms_db_manager = mms_db.DBManager(connection=con) xml_cache_manager = xml_cache.XMLCacheManager('test_nemde_cache') inputs_loader = loaders.RawInputsLoader(xml_cache_manager, mms_db_manager) inputs_loader.set_interval('2019/01/01 00:00:00') return inputs_loader
def test_slack_in_generic_constraints_with_fcas_interface(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db') mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache') raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals(number=100): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData( raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder( unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.add_interconnectors_to_market() market_builder.add_generic_constraints_with_fcas_requirements_interface( ) market_builder.set_unit_fcas_constraints() market_builder.set_unit_limit_constraints() market_builder.set_region_demand_constraints() market_builder.set_ramp_rate_limits() market_builder.set_fast_start_constraints() market_builder.set_solver('CBC') market_builder.dispatch(calc_prices=True) market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider( market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_overrider.set_interconnector_flow_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker( market=market, mms_db=mms_database, xml_cache=xml_cache, interval=interval) assert market_checker.is_generic_constraint_slack_correct() assert market_checker.is_fcas_constraint_slack_correct() assert market_checker.is_regional_demand_meet()
def test_against_100_interval_benchmark(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db') mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache') raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) outputs = [] for interval in get_test_intervals(number=100): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData( raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder( unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.add_interconnectors_to_market() market_builder.add_generic_constraints_with_fcas_requirements_interface( ) market_builder.set_unit_fcas_constraints() market_builder.set_unit_limit_constraints() market_builder.set_region_demand_constraints() market_builder.set_ramp_rate_limits() market_builder.set_fast_start_constraints() market_builder.set_solver('GUROBI') market_builder.dispatch(calc_prices=True) market = market_builder.get_market_object() market_checker = historical_market_builder.MarketChecker( market=market, mms_db=mms_database, xml_cache=xml_cache, interval=interval) price_comp = market_checker.get_price_comparison() outputs.append(price_comp) outputs = pd.concat(outputs) outputs.to_csv('latest_100_interval_run.csv', index=False) benchmark = pd.read_csv('100_interval_benchmark.csv') assert_frame_equal(outputs.reset_index(drop=True), benchmark, check_exact=False, atol=1e-2)
def test_xml_price_bid_loader_vs_mms_db_loader(): test_db = 'F:/nempy_test_files/historical_mms.db' test_xml_cache = 'F:/nempy_test_files/nemde_cache' con = sqlite3.connect(test_db) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache) xml_cache_manager.load_interval('2019/01/01 00:00:00') xml_bids = xml_cache_manager.get_unit_price_bids() mms_bids = mms_database.BIDDAYOFFER_D.get_data('2019/01/01 00:00:00') mms_bids = mms_bids.loc[:, ['DUID', 'BIDTYPE', 'PRICEBAND1', 'PRICEBAND2', 'PRICEBAND3', 'PRICEBAND4', 'PRICEBAND5', 'PRICEBAND6', 'PRICEBAND7', 'PRICEBAND8', 'PRICEBAND9', 'PRICEBAND10']] mms_bids = mms_bids.sort_values(['DUID', 'BIDTYPE']) xml_bids = xml_bids.sort_values(['DUID', 'BIDTYPE']) assert_frame_equal(xml_bids.reset_index(drop=True), mms_bids.reset_index(drop=True), check_less_precise=3)
def test_capacity_constraints(): con = sqlite3.connect( '/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms.db') mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager( '/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache') raw_inputs_loader = loaders.RawInputsLoader( nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals(number=10): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData( raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder( unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.add_interconnectors_to_market() market_builder.set_unit_limit_constraints() market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider( market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_overrider.set_interconnector_flow_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker( market=market, mms_db=mms_database, xml_cache=xml_cache_manager, interval=interval) assert market_checker.measured_violation_equals_historical_violation( 'unit_capacity', nempy_constraints=['unit_bid_capacity'])
def test_ramp_rate_constraints(): con = sqlite3.connect(test_db) mms_database = mms_db.DBManager(con) xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache) raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager, market_management_system_database=mms_database) for interval in get_test_intervals(number=10): raw_inputs_loader.set_interval(interval) unit_inputs = units.UnitData(raw_inputs_loader) interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader) constraint_inputs = constraints.ConstraintData(raw_inputs_loader) demand_inputs = demand.DemandData(raw_inputs_loader) market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs, interconnector_inputs=interconnector_inputs, constraint_inputs=constraint_inputs, demand_inputs=demand_inputs) market_builder.add_unit_bids_to_market() market_builder.set_ramp_rate_limits() market = market_builder.get_market_object() market_overrider = historical_market_builder.MarketOverrider(market=market, mms_db=mms_database, interval=interval) market_overrider.set_unit_dispatch_to_historical_values() market_builder.dispatch() market_checker = historical_market_builder.MarketChecker(market=market, mms_db=mms_database, xml_cache=xml_cache_manager, interval=interval) assert market_checker.measured_violation_equals_historical_violation(historical_name='ramp_rate', nempy_constraints=['ramp_up', 'ramp_down'])
# Notice: this script downloads large volumes of historical market data from AEMO's nemweb portal. import sqlite3 import pandas as pd from nempy import markets from nempy.historical_inputs import loaders, mms_db, \ xml_cache, units, demand, interconnectors, \ constraints con = sqlite3.connect('market_management_system.db') mms_db_manager = mms_db.DBManager(connection=con) xml_cache_manager = xml_cache.XMLCacheManager('cache_directory') # The second time this example is run on a machine this flag can # be set to false to save downloading the data again. download_inputs = True if download_inputs: # This requires approximately 5 GB of storage. mms_db_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=1) # This requires approximately 60 GB of storage. xml_cache_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=1)