def test_get_load_forecast_dst_start(self, expected_requests):
self.nbpower_client = client_factory('NBP')
exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-03-12%2000.csv'
mocked_csv = read_fixture('nbpower',
'2017-03-12 00.csv').encode('utf8')
expected_requests.get(exp_forect_url, content=mocked_csv)
start_at = parse('2017-03-12T00:00:00-04:00')
end_at = parse('2017-03-12T04:00:00-03:00')
load_ts = self.nbpower_client.get_load(start_at=start_at,
end_at=end_at)
self.assertEqual(
len(load_ts), 3
) # 4 forecasts - 1 for quirky duplicate 2017-03-12T03:00-04:00 overwrite
self.assertEqual(load_ts[0].get('timestamp', None),
Timestamp('2017-03-12T04:00:00.000Z'))
self.assertEqual(load_ts[0].get('load_MW', None), 2020)
self.assertEqual(load_ts[1].get('timestamp', None),
Timestamp('2017-03-12T05:00:00.000Z'))
self.assertEqual(load_ts[1].get('load_MW', None), 1982)
self.assertEqual(load_ts[2].get('timestamp', None),
Timestamp('2017-03-12T06:00:00.000Z'))
self.assertEqual(
load_ts[2].get('load_MW', None),
1974) # second T03:00-03:00 load value overwrites first
def test_fetch_csv_load(self):
c = client_factory("NYISO")
now = pytz.utc.localize(datetime.utcnow())
today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
content_list = c.fetch_csvs(today, "pal")
self.assertEqual(len(content_list), 1)
self.assertEqual(content_list[0].split("\r\n")[0], '"Time Stamp","Time Zone","Name","PTID","Load"')
def test_get_load_dange_range_with_latest_and_forecast(
self, expected_requests):
self.nbpower_client = client_factory('NBP')
exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-07-16%2022.csv'
mocked_csv = read_fixture('nbpower',
'2017-07-16 22.csv').encode('utf8')
mocked_html = read_fixture(
'nbpower', 'SystemInformation_realtime.html').encode('utf8')
expected_requests.get(self.nbpower_client.LATEST_REPORT_URL,
content=mocked_html)
expected_requests.get(exp_forect_url, content=mocked_csv)
start_at = parse('2017-07-16T00:00:00-03:00')
end_at = parse('2017-07-17T01:00:00-03:00')
load_ts = self.nbpower_client.get_load(start_at=start_at,
end_at=end_at)
self.assertEqual(len(load_ts), 4) # latest + 3 forecasts.
# The first forecast is in the past, so the first timestamp should be from the latest report.
self.assertEqual(load_ts[0].get('timestamp', None),
Timestamp('2017-07-17T01:57:29.000Z'))
self.assertEqual(load_ts[0].get('load_MW', None), 1150)
self.assertEqual(load_ts[1].get('timestamp', None),
Timestamp('2017-07-17T02:00:00.000Z'))
self.assertEqual(load_ts[1].get('load_MW', None), 1160)
self.assertEqual(load_ts[2].get('timestamp', None),
Timestamp('2017-07-17T03:00:00.000Z'))
self.assertEqual(load_ts[2].get('load_MW', None), 1129)
self.assertEqual(load_ts[3].get('timestamp', None),
Timestamp('2017-07-17T04:00:00.000Z'))
self.assertEqual(load_ts[3].get('load_MW', None), 1089)
def test_get_load_forecast_standard_time_start(self, expected_requests):
self.nbpower_client = client_factory('NBP')
exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-11-05%2000.csv'
mocked_csv = read_fixture('nbpower',
'2017-11-05 00.csv').encode('utf8')
expected_requests.get(exp_forect_url, content=mocked_csv)
start_at = parse('2017-11-05T00:00:00-03:00')
end_at = parse('2017-11-05T03:00:00-04:00')
load_ts = self.nbpower_client.get_load(start_at=start_at,
end_at=end_at)
self.assertEqual(len(load_ts), 4) # 4 hours of load forecasts
self.assertEqual(load_ts[0].get('timestamp', None),
Timestamp('2017-11-05T03:00:00.000Z'))
self.assertEqual(load_ts[0].get('load_MW', None), 1293)
self.assertEqual(load_ts[1].get('timestamp', None),
Timestamp('2017-11-05T04:00:00.000Z'))
self.assertEqual(load_ts[1].get('load_MW', None), 1266)
self.assertEqual(load_ts[2].get('timestamp', None),
Timestamp('2017-11-05T05:00:00.000Z'))
self.assertEqual(load_ts[2].get('load_MW', None), 1261)
# CSV skips time 20171105020000AS (i.e. 2017-11-05T02:00:00-04:00)
self.assertEqual(load_ts[3].get('timestamp', None),
Timestamp('2017-11-05T07:00:00.000Z'))
self.assertEqual(load_ts[3].get('load_MW', None), 1262)
def setUp(self):
environ['EIA_KEY'] = 'test'
self.c = client_factory("EIA")
self.longMessage = True
self.BA_CHOICES = EIAClient.EIA_BAs
self.BALists = BALists()
def test_fetch_csv_genmix(self):
c = client_factory("NYISO")
now = pytz.utc.localize(datetime.utcnow())
today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
content_list = c.fetch_csvs(today, "rtfuelmix")
self.assertEqual(len(content_list), 1)
self.assertEqual(content_list[0].split("\r\n")[0], "Time Stamp,Time Zone,Fuel Category,Gen MWh")
def create_client(self, ba_name):
# set up client with logging
c = client_factory(ba_name)
handler = logging.StreamHandler()
c.logger.addHandler(handler)
c.logger.setLevel(logging.DEBUG)
return c
def create_client(self, ba_name):
# set up client with logging
c = client_factory(ba_name)
handler = logging.StreamHandler()
c.logger.addHandler(handler)
c.logger.setLevel(logging.INFO)
return c
def test_get_generation_valid_date_range_during_dst_returns_expected(
self, mocked_request):
frozen_client = client_factory('YUKON')
frozen_utcnow = datetime.utcnow().replace(tzinfo=pytz.utc)
start_at = frozen_utcnow - timedelta(hours=12)
end_at = frozen_utcnow
expected_url = 'http://www.yukonenergy.ca/consumption/chart.php?chart=hourly'
expected_response = read_fixture(frozen_client.__module__,
'hourly_2017-10-11.html')
mocked_request.get(expected_url,
content=expected_response.encode('utf-8'))
results = frozen_client.get_generation(start_at=start_at,
end_at=end_at)
expected_length = 22 # 2 fuels * 11 hours (1 hour is missing in this particular response)
self.assertEqual(len(results), expected_length)
# Spot check values at the start and end of the results
self.assertEqual(results[0]['timestamp'],
Timestamp('2017-10-10T23:00:00Z'))
self.assertEqual(results[0]['fuel_name'], 'hydro')
self.assertAlmostEqual(results[0]['gen_MW'], 51.36)
self.assertEqual(results[21]['timestamp'],
Timestamp('2017-10-11T09:00:00Z'))
self.assertEqual(results[21]['fuel_name'], 'thermal')
self.assertAlmostEqual(results[21]['gen_MW'], 0)
def _run_test(self, ba, kwargs):
expected = client_factory(ba).get_trade(**kwargs)
received = tasks.get_trade(ba, **kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['net_exp_MW'],
received[i]['net_exp_MW'])
def _run_test(self, ba, kwargs):
expected = client_factory(ba).get_generation(**kwargs)
received = tasks.get_generation(ba, **kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW'])
self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])
def test_parse_ren_report_both(self):
c = client_factory('CAISO')
# top half
top_df = c.parse_to_df(self.ren_report_tsv,
skiprows=1,
nrows=24,
header=0,
delimiter='\t+',
engine='python')
self.assertEqual(list(top_df.columns), [
'Hour', 'GEOTHERMAL', 'BIOMASS', 'BIOGAS', 'SMALL HYDRO',
'WIND TOTAL', 'SOLAR PV', 'SOLAR THERMAL'
])
self.assertEqual(len(top_df), 24)
# bottom half
bot_df = c.parse_to_df(self.ren_report_tsv,
skiprows=3,
nrows=24,
header=0,
delimiter='\t+',
engine='python')
self.assertEqual(
list(bot_df.columns),
['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO'])
self.assertEqual(len(bot_df), 24)
def setUp(self):
self.c = client_factory('AZPS')
handler = logging.StreamHandler()
self.c.logger.addHandler(handler)
self.c.logger.setLevel(logging.INFO)
self.TZ_NAME = 'America/Phoenix'
self.now = datetime.now(pytz.timezone(self.TZ_NAME))
self.today = datetime.combine(self.now.date(), time())
self.yesterday = self.today - timedelta(days=1)
self.tomorrow = self.today + timedelta(days=1)
self.last_month = self.today - timedelta(days=32)
self.next_month = self.today + timedelta(days=32)
self.sample = '"Time (MST)","Solar Aggregate (MW)","Wind Aggregate (MW)","Other Renewables Aggregate (MW)","Hydro Aggregate (MW)"\n\
"2015-07-18 00:00:05",0.028038,134.236,116.294,835.628\n\
"2015-07-18 00:00:15",0.079792,132.875,115.742,817.136\n\
"2015-07-18 00:00:25",0.101218,131.663,115.727,802.076\n\
"2015-07-18 00:00:35",0.119465,130.456,115.712,798.891\n\
"2015-07-18 00:00:45",0.180171,129.403,115.697,820.483\n\
"2015-07-18 00:00:55",0.268247,129.221,115.681,849.028\n\
"2015-07-18 00:01:05",0.29724,128.836,115.666,865.843\n\
"2015-07-18 00:01:15",0.281987,127.656,115.354,862.289\n\
'
self.sample_start = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 18, 0, 0))
self.sample_end = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 19, 0, 0))
def test_pjm_latest(self):
expected = client_factory("PJM").get_generation(**self.latest_kwargs)
received = tasks.get_generation("PJM", **self.latest_kwargs)
for i in range(len(expected)):
if expected[i]["timestamp"] == received[i]["timestamp"]:
self.assertEqual(expected[i]["gen_MW"], received[i]["gen_MW"])
self.assertEqual(expected[i]["fuel_name"], received[i]["fuel_name"])
def test_pjm_latest(self):
expected = client_factory('PJM').get_generation(**self.latest_kwargs)
received = tasks.get_generation('PJM', **self.latest_kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW'])
self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])
def lmp_nyiso(self, start_time, end_time, product):
if self.options['verbose']:
'Retrieving NYISO data'
end_time = end_time - datetime.timedelta(seconds=1)
nyiso = client_factory('NYISO')
data = nyiso.get_lmp(data='lmp',
node_id='ALL',
start_at=start_time,
end_at=end_time,
market=product)
data = pd.DataFrame(data)
data.drop(['ba_name', 'freq', 'lmp_type', 'market'],
axis=1,
inplace=True)
data.rename(columns={
'PTID': 'iso_node_id',
'node_id': 'name',
'timestamp': 'datatime'
},
inplace=True)
return data
def test_parse_ren_report_bot(self):
c = client_factory("CAISO")
# bottom half
bot_df = c.parse_to_df(self.ren_report_tsv, skiprows=29, nrows=24, header=0, delimiter="\t+", engine="python")
self.assertEqual(list(bot_df.columns), ["Hour", "RENEWABLES", "NUCLEAR", "THERMAL", "IMPORTS", "HYDRO"])
self.assertEqual(len(bot_df), 24)
def test_fetch_oasis_ren_dam(self):
c = client_factory("CAISO")
ts = c.utcify("2014-05-08 12:00")
payload = {
"queryname": "SLD_REN_FCST",
"market_run_id": "DAM",
"startdatetime": (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format),
"enddatetime": (ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format),
}
payload.update(c.base_payload)
data = c.fetch_oasis(payload=payload)
self.assertEqual(len(data), 4)
self.assertEqual(
str(data[0]),
"<report_data>\n\
<data_item>RENEW_FCST_DA_MW</data_item>\n\
<opr_date>2014-05-08</opr_date>\n\
<interval_num>13</interval_num>\n\
<interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\
<interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\
<value>813.7</value>\n\
<trading_hub>NP15</trading_hub>\n\
<renewable_type>Solar</renewable_type>\n\
</report_data>",
)
def _run_test(self, ba_name, **kwargs):
# set up
c = client_factory(ba_name)
# get data
data = c.get_generation(**kwargs)
# test number
self.assertGreater(len(data), 1)
# test contents
for dp in data:
# test key names
self.assertEqual(set(['gen_MW', 'ba_name', 'fuel_name',
'timestamp', 'freq', 'market']),
set(dp.keys()))
# test values
self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
self.assertIn(dp['fuel_name'], self.FUEL_CHOICES)
self.assertIn(dp['ba_name'], self.BA_CHOICES)
# test for numeric gen
self.assertGreaterEqual(dp['gen_MW']+1, dp['gen_MW'])
# test earlier than now
if c.options.get('forecast', False):
self.assertGreater(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))
else:
self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))
# return
return data
def test_fetch_oasis_demand_rtm(self):
c = client_factory('CAISO')
ts = c.utcify('2014-05-08 12:00')
payload = {
'queryname':
'SLD_FCST',
'market_run_id':
'RTM',
'startdatetime':
(ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format),
'enddatetime':
(ts + timedelta(minutes=20)).strftime(c.oasis_request_time_format),
}
payload.update(c.base_payload)
data = c.fetch_oasis(payload=payload)
self.assertEqual(len(data), 55)
self.assertEqual(
str(data[0]).lower(), '<report_data>\n\
<data_item>SYS_FCST_15MIN_MW</data_item>\n\
<resource_name>CA ISO-TAC</resource_name>\n\
<opr_date>2014-05-08</opr_date>\n\
<interval_num>50</interval_num>\n\
<interval_start_gmt>2014-05-08T19:15:00-00:00</interval_start_gmt>\n\
<interval_end_gmt>2014-05-08T19:30:00-00:00</interval_end_gmt>\n\
<value>26723</value>\n\
</report_data>'.lower())
def test_fetch_oasis_slrs_dam(self):
c = client_factory('CAISO')
ts = c.utcify('2014-05-08 12:00')
payload = {
'queryname':
'ENE_SLRS',
'market_run_id':
'DAM',
'startdatetime':
(ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format),
'enddatetime':
(ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format),
}
payload.update(c.base_payload)
data = c.fetch_oasis(payload=payload)
self.assertEqual(len(data), 17)
self.assertEqual(
str(data[0]).lower(), '<report_data>\n\
<data_item>ISO_TOT_EXP_MW</data_item>\n\
<resource_name>Caiso_Totals</resource_name>\n\
<opr_date>2014-05-08</opr_date>\n\
<interval_num>13</interval_num>\n\
<interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\
<interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\
<value>1044</value>\n\
</report_data>'.lower())
def test_fetch_csv_load_forecast(self):
c = client_factory('NYISO')
now = pytz.utc.localize(datetime.utcnow())
today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
content = c.fetch_csv(today, 'isolf')
self.assertEqual(content.split('\n')[0],
'"Time Stamp","Capitl","Centrl","Dunwod","Genese","Hud Vl","Longil","Mhk Vl","Millwd","N.Y.C.","North","West","NYISO"')
def test_fetch_oasis_ren_dam(self):
c = client_factory('CAISO')
ts = c.utcify('2014-05-08 12:00')
payload = {
'queryname':
'SLD_REN_FCST',
'market_run_id':
'DAM',
'startdatetime':
(ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format),
'enddatetime':
(ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format),
}
payload.update(c.base_payload)
data = c.fetch_oasis(payload=payload)
self.assertEqual(len(data), 4)
self.assertEqual(
str(data[0]).lower(), '<report_data>\n\
<data_item>RENEW_FCST_DA_MW</data_item>\n\
<opr_date>2014-05-08</opr_date>\n\
<interval_num>13</interval_num>\n\
<interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\
<interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\
<value>813.7</value>\n\
<trading_hub>NP15</trading_hub>\n\
<renewable_type>Solar</renewable_type>\n\
</report_data>'.lower())
def test_utcify_index(self):
c = client_factory('BPA')
df = c.parse_to_df(self.wind_tsv, skiprows=6, header=0, delimiter='\t',
index_col=0, parse_dates=True)
utc_index = c.utcify_index(df.index)
self.assertEqual(utc_index[0].to_pydatetime(), datetime(2014, 4, 15, 10+7, 10, tzinfo=pytz.utc))
self.assertEqual(len(df), len(utc_index))
def setUp(self):
self.c = client_factory('AZPS')
self.TZ_NAME = 'America/Phoenix'
self.now = datetime.now(pytz.timezone(self.TZ_NAME))
self.today = datetime.combine(self.now.date(), time())
self.yesterday = self.today - timedelta(days=1)
self.tomorrow = self.today + timedelta(days=1)
self.last_month = self.today - timedelta(days=32)
self.next_month = self.today + timedelta(days=32)
self.sample = '"Time (MST)","Solar Aggregate (MW)","Wind Aggregate (MW)","Other Renewables Aggregate (MW)","Hydro Aggregate (MW)"\n\
"2015-07-18 00:00:05",0.028038,134.236,116.294,835.628\n\
"2015-07-18 00:00:15",0.079792,132.875,115.742,817.136\n\
"2015-07-18 00:00:25",0.101218,131.663,115.727,802.076\n\
"2015-07-18 00:00:35",0.119465,130.456,115.712,798.891\n\
"2015-07-18 00:00:45",0.180171,129.403,115.697,820.483\n\
"2015-07-18 00:00:55",0.268247,129.221,115.681,849.028\n\
"2015-07-18 00:01:05",0.29724,128.836,115.666,865.843\n\
"2015-07-18 00:01:15",0.281987,127.656,115.354,862.289\n\
'
self.sample_start = pytz.timezone(self.TZ_NAME).localize(
datetime(2015, 7, 18, 0, 0))
self.sample_end = pytz.timezone(self.TZ_NAME).localize(
datetime(2015, 7, 19, 0, 0))
def _run_pairwise_test(self, ba_name, **kwargs):
# set up
c = client_factory(ba_name)
# get data
data = c.get_trade(**kwargs)
# test number
self.assertGreaterEqual(len(data), 1)
# test contents
for dp in data:
# test key names
for key in ['source_ba_name', 'dest_ba_name', 'timestamp', 'freq', 'market']:
self.assertIn(key, dp.keys())
self.assertEqual(len(dp.keys()), 6)
# test values
self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
self.assertIn(dp['source_ba_name'], self.BA_CHOICES)
# test for numeric value
self.assertGreaterEqual(dp['export_MW']+1, dp['export_MW'])
# test correct temporal relationship to now
if c.options['forecast']:
self.assertGreaterEqual(dp['timestamp'], datetime.now(pytz.utc))
else:
self.assertLess(dp['timestamp'], datetime.now(pytz.utc))
# return
return data
def test_get_ancillary_services(self):
c = client_factory('CAISO')
ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc)
start = ts - timedelta(days=2)
as_prc = c.get_ancillary_services('AS_CAISO_EXP', start_at=start, end_at=ts,
market_run_id='DAM')
self.assertEqual(len(as_prc), 48)
self.assertGreaterEqual(min([i['timestamp'] for i in as_prc]),
start - timedelta(minutes=5))
self.assertLessEqual(max([i['timestamp'] for i in as_prc]),
ts + timedelta(minutes=5))
means = {
'SR': 1.685417,
'RU': 3.074583,
'RMU': 5.729167e-02,
'RMD': 3.620833e-01,
'RD': 3.901667,
'NR': 9.000000e-02,
}
for anc_type in means:
dp = [i[anc_type] for i in as_prc]
self.assertAlmostEqual(numpy.mean(dp), means[anc_type], places=6)
def test_names(self):
for name in self.expected_names:
if 'env_vars' in name:
for var in name['env_vars']:
environ[var] = name['env_vars'][var]
c = client_factory(name['name'])
self.assertIsNotNone(c)
def test_pjm_latest(self):
expected = client_factory('PJM').get_generation(**self.latest_kwargs)
received = tasks.get_generation('PJM', **self.latest_kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW'])
self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])
def test_ercot_latest(self):
# before min 32, will not have wind data
if datetime.now().minute >= 32:
# basic test
data = self._run_test('ERCOT', latest=True)
# test all timestamps are equal
timestamps = [d['timestamp'] for d in data]
self.assertEqual(len(set(timestamps)), 1)
# test flags
for dp in data:
self.assertEqual(dp['market'], self.MARKET_CHOICES.hourly)
self.assertEqual(dp['freq'], self.FREQUENCY_CHOICES.hourly)
# test fuel names
fuels = set([d['fuel_name'] for d in data])
expected_fuels = ['wind', 'nonwind']
for expfuel in expected_fuels:
self.assertIn(expfuel, fuels)
else:
c = client_factory('ERCOT')
data = c.get_generation(latest=True)
self.assertEqual(len(data), 0)
def test_request_renewable_report(self):
c = client_factory('CAISO')
response = c.request(
'http://content.caiso.com/green/renewrpt/20140312_DailyRenewablesWatch.txt'
)
self.assertIn('Hourly Breakdown of Renewable Resources (MW)',
response.text)
def setUp(self):
self.edata_inst_load = open(
FIXTURES_DIR + '/pjm/InstantaneousLoad.html').read().encode('utf8')
self.edata_forecast_load = open(
FIXTURES_DIR +
'/pjm/ForecastedLoadHistory.html').read().encode('utf8')
self.c = client_factory('PJM')
def test_utcify_index(self):
c = client_factory('BPA')
df = c.parse_to_df(self.wind_tsv, skiprows=6, header=0, delimiter='\t',
index_col=0, parse_dates=True)
utc_index = c.utcify_index(df.index)
self.assertEqual(utc_index[0].to_pydatetime(), datetime(2014, 4, 15, 10+7, 10, tzinfo=pytz.utc))
self.assertEqual(len(df), len(utc_index))
def test_names(self):
for name in self.expected_names:
if 'env_vars' in name:
for var in name['env_vars']:
environ[var] = name['env_vars'][var];
c = client_factory(name['name'])
self.assertIsNotNone(c)
def setUp(self):
environ['EIA_KEY'] = 'test'
self.c = client_factory("EIA")
self.longMessage = True
self.BA_CHOICES = EIAClient.EIA_BAs
self.BALists = BALists()
def _run_test(self, ba, node_list, kwargs):
expected = client_factory(ba).get_lmp(node_list, **kwargs)
received = tasks.get_lmp(ba, node_list, **kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['lmp'], received[i]['lmp'])
self.assertEqual(expected[i]['lmp_type'], received[i]['lmp_type'])
def _run_net_test(self, ba_name, **kwargs):
# set up
c = client_factory(ba_name)
# get data
data = c.get_trade(**kwargs)
# test number
self.assertGreaterEqual(len(data), 1)
# test contents
for dp in data:
# test key names
for key in ['ba_name', 'timestamp', 'freq', 'market']:
self.assertIn(key, dp.keys())
self.assertEqual(len(dp.keys()), 5)
# test values
self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
self.assertIn(dp['ba_name'], self.BA_CHOICES)
# test for numeric value
self.assertGreaterEqual(dp['net_exp_MW'] + 1, dp['net_exp_MW'])
# test correct temporal relationship to now
if c.options['forecast']:
self.assertGreaterEqual(dp['timestamp'],
pytz.utc.localize(datetime.utcnow()))
else:
self.assertLess(dp['timestamp'],
pytz.utc.localize(datetime.utcnow()))
# return
return data
def test_get_AS_dataframe_latest(self):
c = client_factory('CAISO')
as_prc = c.get_AS_dataframe('AS_CAISO_EXP')
# Could be 1 or 2 prices in last 61 minutes
self.assertLessEqual(len(as_prc), 12)
self.assertGreaterEqual(len(as_prc), 6)
def _run_test(self, ba_name, expect_data=True, **kwargs):
# set up
c = client_factory(ba_name)
# get data
data = c.get_load(**kwargs)
# test number
if expect_data:
self.assertGreaterEqual(len(data), 1)
else:
self.assertEqual(data, [])
# test contents
for dp in data:
# test key names
self.assertEqual(set(['load_MW', 'ba_name',
'timestamp', 'freq', 'market']),
set(dp.keys()))
# test values
self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
self.assertIn(dp['ba_name'], self.BA_CHOICES)
# test for numeric gen
self.assertGreaterEqual(dp['load_MW']+1, dp['load_MW'])
# test correct temporal relationship to now
if c.options['forecast']:
self.assertGreaterEqual(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))
else:
self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))
# return
return data
def test_get_AS_dataframe_empty(self):
c = client_factory('CAISO')
st = pytz.utc.localize(datetime.now() + timedelta(days=2))
et = st + timedelta(days=1)
as_prc = c.get_AS_dataframe('AS_CAISO_EXP', start_at=st, end_at=et,
market_run_id='DAM', anc_type='RU')
self.assertTrue(as_prc.empty)
def test_get_AS_dataframe(self):
c = client_factory('CAISO')
ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc)
start = ts - timedelta(days=2)
as_prc = c.get_AS_dataframe(node_id='AS_CAISO_EXP', start_at=start, end_at=ts,
market_run_id='DAM')
self.assertEqual(len(as_prc), 288)
self.assertAlmostEqual(as_prc['MW'].mean(), 1.528506944444443)
grouped = as_prc.groupby('XML_DATA_ITEM')
self.assertEqual(len(grouped), 6)
means = {
'SP_CLR_PRC': 1.685417,
'RU_CLR_PRC': 3.074583,
'RMU_CLR_PRC': 5.729167e-02,
'RMD_CLR_PRC': 3.620833e-01,
'RD_CLR_PRC': 3.901667,
'NS_CLR_PRC': 9.000000e-02,
}
for group in means:
self.assertAlmostEqual(grouped.get_group(group)['MW'].mean(), means[group], places=6)
self.assertEqual(len(grouped.get_group(group)), 48)
def test_parse_systemconditions(self):
"""Test for a newly discovered edge case: an extra, empty `docdata` cell."""
c = client_factory('CAISO')
soup = BeautifulSoup(self.systemconditions_html, 'lxml')
c.handle_options()
parsed_data = c.todays_outlook_time(soup)
expected = datetime(2017, 1, 5, 21, 50, tzinfo=pytz.utc)
self.assertEqual(parsed_data, expected)
def test_dt_index(self):
c = client_factory('CAISO')
df = c.parse_to_df(self.ren_report_tsv,
skiprows=1, nrows=24, header=0,
delimiter='\t+', engine='python')
indexed = c.set_dt_index(df, date(2014, 3, 12), df['Hour'])
self.assertEqual(type(indexed.index), pd.tseries.index.DatetimeIndex)
self.assertEqual(indexed.index[0].hour, 7)
def test_get_AS_empty(self):
"""No AS data available 2 days in future"""
c = client_factory('CAISO')
st = pytz.utc.localize(datetime.now() + timedelta(days=2))
et = st + timedelta(days=1)
as_prc = c.get_ancillary_services(node_id='AS_CAISO_EXP', start_at=st, end_at=et,
market_run_id='DAM', anc_type='RU')
self.assertEqual(as_prc, {})
def _run_test(self, ba, kwargs):
expected = client_factory(ba).get_trade(**kwargs)
received = tasks.get_trade(ba, **kwargs)
self.assertEqual(len(expected), len(received))
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['net_exp_MW'], received[i]['net_exp_MW'])
def test_request_report(self):
# get data
c = client_factory('ERCOT')
result = c._request_report('gen_hrly')
# should be a list containing 1 dict
self.assertEqual(len(result), 1)
self.assertIn('SE_MW', result[0].keys())
def test_request_report(self):
# get data
c = client_factory('ERCOT')
result = c._request_report('gen_hrly')
# should be a list containing 1 dict
self.assertEqual(len(result), 1)
self.assertIn('SE_MW', result[0].keys())
def setUp(self):
self.c = client_factory('CAISO')
self.ren_report_tsv = read_fixture(self.c.__module__, 'ren_report.csv')
self.sld_fcst_xml = read_fixture(self.c.__module__, 'sld_forecast.xml')
self.ene_slrs_xml = read_fixture(self.c.__module__, 'ene_slrs.xml')
self.sld_ren_fcst_xml = read_fixture(self.c.__module__, 'sld_ren_forecast.xml')
self.systemconditions_html = read_fixture(self.c.__module__, 'systemconditions.html')
self.todays_outlook_renewables = read_fixture(self.c.__module__, 'todays_outlook_renewables.html')
def _run_test(self, ba, node_list, kwargs):
expected = client_factory(ba).get_lmp(node_list, **kwargs)
received = tasks.get_lmp(ba, node_list, **kwargs)
for i in range(len(expected)):
if expected[i]['timestamp'] == received[i]['timestamp']:
self.assertEqual(expected[i]['lmp'], received[i]['lmp'])
self.assertEqual(expected[i]['lmp_type'],
received[i]['lmp_type'])
def test_bad_data_lmp_only(self, mock_request):
mock_request.return_value = requests.get('https://httpbin.org/')
c = client_factory('CAISO')
ts = pytz.utc.localize(datetime(2015, 3, 1, 12))
start = ts - timedelta(hours=2)
df = c.get_lmp_as_dataframe('CAISO_AS', latest=False, start_at=start, end_at=ts,
lmp_only=False)
self.assertIsInstance(df, pd.DataFrame)
def test_parse_ren_report_bot(self):
c = client_factory('CAISO')
# bottom half
bot_df = c.parse_to_df(self.ren_report_tsv,
skiprows=29, nrows=24, header=0,
delimiter='\t+', engine='python')
self.assertEqual(list(bot_df.columns), ['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO'])
self.assertEqual(len(bot_df), 24)
def setUp(self):
self.c = client_factory('CAISO')
self.ren_report_tsv = read_fixture('ren_report.csv')
self.sld_fcst_xml = read_fixture('sld_forecast.xml')
self.ene_slrs_xml = read_fixture('ene_slrs.xml')
self.sld_ren_fcst_xml = read_fixture('sld_ren_forecast.xml')
self.systemconditions_html = read_fixture('systemconditions.html')
self.todays_outlook_renewables = read_fixture(
'todays_outlook_renewables.html')
