def test_raise_error_if_fformat_not_in_expected_set(self):
with self.assertRaises(Exception) as context:
dynamic_data_compiler('2019/01/01 00:00:00',
'2019/02/01 00:00:00',
'DISPATCHPRICE',
defaults.raw_data_cache,
fformat='db')
self.assertTrue(
"Argument fformat must be 'csv', 'feather' or 'parquet'" in str(
context.exception))
def test_raise_error_for_no_data_returned(self):
with self.assertRaises(Exception) as context:
dynamic_data_compiler('2000/01/01 00:00:00', '2000/02/01 00:00:00',
'DISPATCHPRICE', defaults.raw_data_cache)
self.assertTrue((
f'Compiling data for table DISPATCHPRICE failed. ' +
'This probably because none of the requested data ' +
'could be download from AEMO. Check your internet ' +
'connection and that the requested data is archived on: ' +
'https://nemweb.com.au see nemosis.defaults for table specific urls.'
) in str(context.exception))
def test_raise_error_if_select_columns_not_in_data(self):
with self.assertRaises(Exception) as context:
dynamic_data_compiler('2019/01/01 00:00:00',
'2019/02/01 00:00:00',
'DISPATCHPRICE',
defaults.raw_data_cache,
select_columns=['NOTACOLUMN'])
self.assertTrue((
f'None of columns [\'NOTACOLUMN\'] are in raw_aemo_data\\PUBLIC_DVD_DISPATCHPRICE_201812010000.feather. '
"This may be caused by user input if the \'select_columns\' "
"argument is being used, or by changed AEMO data formats. "
"This error can be avoided by using the argument select_columns=\'all\'."
) in str(context.exception))
def test_raise_error_for_filter_column_not_in_default_columns(self):
with self.assertRaises(Exception) as context:
dynamic_data_compiler(
'2019/01/01 00:00:00',
'2019/02/01 00:00:00',
'DISPATCHPRICE',
defaults.raw_data_cache,
select_columns=['REGIONID', 'SETTLEMENTDATE', 'RRP'],
filter_cols=['NOTACOLUMN'],
filter_values=(['0'], ))
self.assertTrue((
'Filter columns not valid. They must be a part of ' +
'select_columns or the table defaults.') in str(context.exception))
def test_using_select_columns_all_does_not_raise_error(self):
price_data = dynamic_data_compiler('2019/01/01 00:00:00',
'2019/02/01 00:00:00',
'DISPATCHPRICE',
defaults.raw_data_cache,
select_columns='all',
fformat='csv')
expected_columns = [
'I', 'DISPATCH', 'PRICE', '1', 'SETTLEMENTDATE', 'RUNNO',
'REGIONID', 'DISPATCHINTERVAL', 'INTERVENTION', 'RRP', 'EEP',
'ROP', 'APCFLAG', 'MARKETSUSPENDEDFLAG', 'LASTCHANGED',
'RAISE6SECRRP', 'RAISE6SECROP', 'RAISE6SECAPCFLAG',
'RAISE60SECRRP', 'RAISE60SECROP', 'RAISE60SECAPCFLAG',
'RAISE5MINRRP', 'RAISE5MINROP', 'RAISE5MINAPCFLAG', 'RAISEREGRRP',
'RAISEREGROP', 'RAISEREGAPCFLAG', 'LOWER6SECRRP', 'LOWER6SECROP',
'LOWER6SECAPCFLAG', 'LOWER60SECRRP', 'LOWER60SECROP',
'LOWER60SECAPCFLAG', 'LOWER5MINRRP', 'LOWER5MINROP',
'LOWER5MINAPCFLAG', 'LOWERREGRRP', 'LOWERREGROP',
'LOWERREGAPCFLAG', 'PRICE_STATUS', 'PRE_AP_ENERGY_PRICE',
'PRE_AP_RAISE6_PRICE', 'PRE_AP_RAISE60_PRICE',
'PRE_AP_RAISE5MIN_PRICE', 'PRE_AP_RAISEREG_PRICE',
'PRE_AP_LOWER6_PRICE', 'PRE_AP_LOWER60_PRICE',
'PRE_AP_LOWER5MIN_PRICE', 'PRE_AP_LOWERREG_PRICE',
'CUMUL_PRE_AP_ENERGY_PRICE', 'CUMUL_PRE_AP_RAISE6_PRICE',
'CUMUL_PRE_AP_RAISE60_PRICE', 'CUMUL_PRE_AP_RAISE5MIN_PRICE',
'CUMUL_PRE_AP_RAISEREG_PRICE', 'CUMUL_PRE_AP_LOWER6_PRICE',
'CUMUL_PRE_AP_LOWER60_PRICE', 'CUMUL_PRE_AP_LOWER5MIN_PRICE',
'CUMUL_PRE_AP_LOWERREG_PRICE'
]
self.assertSequenceEqual(list(price_data.columns), expected_columns)
def test_raise_error_for_incorrect_table_name(self):
with self.assertRaises(Exception) as context:
dynamic_data_compiler('2000/01/01 00:00:00', '2000/02/01 00:00:00',
'NOTATABLE', defaults.raw_data_cache)
self.assertTrue("Table name provided is not a dynamic table." in str(
context.exception))
raw_data_location=raw_data_cache,
update_static_file=True)
# Using filtering and manual inspection find which fcas element numbers belong to Hornsdale Power Reserve.
elements_for_honsdale_power_reserve = \
fcas_causer_pays_elements[fcas_causer_pays_elements['EMSNAME'].str.contains('HPR')]
# Check which variable numbers we will need.
fcas_causer_pays_elements = static_table(table_name='ELEMENTS_FCAS_4_SECOND',
raw_data_location=raw_data_cache,
update_static_file=True)
scada_4s_resolution = dynamic_data_compiler(
start_time,
end_time,
table_name='FCAS_4_SECOND',
raw_data_location=raw_data_cache,
filter_cols=['ELEMENTNUMBER', 'VARIABLENUMBER'],
filter_values=([330, 331], [2, 5]),
fformat='parquet')
scada_5min_resolution = dynamic_data_compiler(
start_time,
end_time,
'DISPATCHLOAD',
raw_data_cache,
select_columns=['SETTLEMENTDATE', 'DUID', 'INITIALMW', 'TOTALCLEARED'],
filter_cols=['DUID'],
filter_values=(['HPRG1', 'HPRL1'], ))
elements = {330: 'HPRG1', 331: 'HPRL1'}