def test_get_load_forecast_dst_start(self, expected_requests): self.nbpower_client = client_factory('NBP') exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-03-12%2000.csv' mocked_csv = read_fixture('nbpower', '2017-03-12 00.csv').encode('utf8') expected_requests.get(exp_forect_url, content=mocked_csv) start_at = parse('2017-03-12T00:00:00-04:00') end_at = parse('2017-03-12T04:00:00-03:00') load_ts = self.nbpower_client.get_load(start_at=start_at, end_at=end_at) self.assertEqual( len(load_ts), 3 ) # 4 forecasts - 1 for quirky duplicate 2017-03-12T03:00-04:00 overwrite self.assertEqual(load_ts[0].get('timestamp', None), Timestamp('2017-03-12T04:00:00.000Z')) self.assertEqual(load_ts[0].get('load_MW', None), 2020) self.assertEqual(load_ts[1].get('timestamp', None), Timestamp('2017-03-12T05:00:00.000Z')) self.assertEqual(load_ts[1].get('load_MW', None), 1982) self.assertEqual(load_ts[2].get('timestamp', None), Timestamp('2017-03-12T06:00:00.000Z')) self.assertEqual( load_ts[2].get('load_MW', None), 1974) # second T03:00-03:00 load value overwrites first
def test_fetch_csv_load(self): c = client_factory("NYISO") now = pytz.utc.localize(datetime.utcnow()) today = now.astimezone(pytz.timezone(c.TZ_NAME)).date() content_list = c.fetch_csvs(today, "pal") self.assertEqual(len(content_list), 1) self.assertEqual(content_list[0].split("\r\n")[0], '"Time Stamp","Time Zone","Name","PTID","Load"')
def test_get_load_dange_range_with_latest_and_forecast( self, expected_requests): self.nbpower_client = client_factory('NBP') exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-07-16%2022.csv' mocked_csv = read_fixture('nbpower', '2017-07-16 22.csv').encode('utf8') mocked_html = read_fixture( 'nbpower', 'SystemInformation_realtime.html').encode('utf8') expected_requests.get(self.nbpower_client.LATEST_REPORT_URL, content=mocked_html) expected_requests.get(exp_forect_url, content=mocked_csv) start_at = parse('2017-07-16T00:00:00-03:00') end_at = parse('2017-07-17T01:00:00-03:00') load_ts = self.nbpower_client.get_load(start_at=start_at, end_at=end_at) self.assertEqual(len(load_ts), 4) # latest + 3 forecasts. # The first forecast is in the past, so the first timestamp should be from the latest report. self.assertEqual(load_ts[0].get('timestamp', None), Timestamp('2017-07-17T01:57:29.000Z')) self.assertEqual(load_ts[0].get('load_MW', None), 1150) self.assertEqual(load_ts[1].get('timestamp', None), Timestamp('2017-07-17T02:00:00.000Z')) self.assertEqual(load_ts[1].get('load_MW', None), 1160) self.assertEqual(load_ts[2].get('timestamp', None), Timestamp('2017-07-17T03:00:00.000Z')) self.assertEqual(load_ts[2].get('load_MW', None), 1129) self.assertEqual(load_ts[3].get('timestamp', None), Timestamp('2017-07-17T04:00:00.000Z')) self.assertEqual(load_ts[3].get('load_MW', None), 1089)
def test_get_load_forecast_standard_time_start(self, expected_requests): self.nbpower_client = client_factory('NBP') exp_forect_url = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/2017-11-05%2000.csv' mocked_csv = read_fixture('nbpower', '2017-11-05 00.csv').encode('utf8') expected_requests.get(exp_forect_url, content=mocked_csv) start_at = parse('2017-11-05T00:00:00-03:00') end_at = parse('2017-11-05T03:00:00-04:00') load_ts = self.nbpower_client.get_load(start_at=start_at, end_at=end_at) self.assertEqual(len(load_ts), 4) # 4 hours of load forecasts self.assertEqual(load_ts[0].get('timestamp', None), Timestamp('2017-11-05T03:00:00.000Z')) self.assertEqual(load_ts[0].get('load_MW', None), 1293) self.assertEqual(load_ts[1].get('timestamp', None), Timestamp('2017-11-05T04:00:00.000Z')) self.assertEqual(load_ts[1].get('load_MW', None), 1266) self.assertEqual(load_ts[2].get('timestamp', None), Timestamp('2017-11-05T05:00:00.000Z')) self.assertEqual(load_ts[2].get('load_MW', None), 1261) # CSV skips time 20171105020000AS (i.e. 2017-11-05T02:00:00-04:00) self.assertEqual(load_ts[3].get('timestamp', None), Timestamp('2017-11-05T07:00:00.000Z')) self.assertEqual(load_ts[3].get('load_MW', None), 1262)
def setUp(self): environ['EIA_KEY'] = 'test' self.c = client_factory("EIA") self.longMessage = True self.BA_CHOICES = EIAClient.EIA_BAs self.BALists = BALists()
def test_fetch_csv_genmix(self): c = client_factory("NYISO") now = pytz.utc.localize(datetime.utcnow()) today = now.astimezone(pytz.timezone(c.TZ_NAME)).date() content_list = c.fetch_csvs(today, "rtfuelmix") self.assertEqual(len(content_list), 1) self.assertEqual(content_list[0].split("\r\n")[0], "Time Stamp,Time Zone,Fuel Category,Gen MWh")
def create_client(self, ba_name): # set up client with logging c = client_factory(ba_name) handler = logging.StreamHandler() c.logger.addHandler(handler) c.logger.setLevel(logging.DEBUG) return c
def create_client(self, ba_name): # set up client with logging c = client_factory(ba_name) handler = logging.StreamHandler() c.logger.addHandler(handler) c.logger.setLevel(logging.INFO) return c
def test_get_generation_valid_date_range_during_dst_returns_expected( self, mocked_request): frozen_client = client_factory('YUKON') frozen_utcnow = datetime.utcnow().replace(tzinfo=pytz.utc) start_at = frozen_utcnow - timedelta(hours=12) end_at = frozen_utcnow expected_url = 'http://www.yukonenergy.ca/consumption/chart.php?chart=hourly' expected_response = read_fixture(frozen_client.__module__, 'hourly_2017-10-11.html') mocked_request.get(expected_url, content=expected_response.encode('utf-8')) results = frozen_client.get_generation(start_at=start_at, end_at=end_at) expected_length = 22 # 2 fuels * 11 hours (1 hour is missing in this particular response) self.assertEqual(len(results), expected_length) # Spot check values at the start and end of the results self.assertEqual(results[0]['timestamp'], Timestamp('2017-10-10T23:00:00Z')) self.assertEqual(results[0]['fuel_name'], 'hydro') self.assertAlmostEqual(results[0]['gen_MW'], 51.36) self.assertEqual(results[21]['timestamp'], Timestamp('2017-10-11T09:00:00Z')) self.assertEqual(results[21]['fuel_name'], 'thermal') self.assertAlmostEqual(results[21]['gen_MW'], 0)
def _run_test(self, ba, kwargs): expected = client_factory(ba).get_trade(**kwargs) received = tasks.get_trade(ba, **kwargs) for i in range(len(expected)): if expected[i]['timestamp'] == received[i]['timestamp']: self.assertEqual(expected[i]['net_exp_MW'], received[i]['net_exp_MW'])
def _run_test(self, ba, kwargs): expected = client_factory(ba).get_generation(**kwargs) received = tasks.get_generation(ba, **kwargs) for i in range(len(expected)): if expected[i]['timestamp'] == received[i]['timestamp']: self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW']) self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])
def test_parse_ren_report_both(self): c = client_factory('CAISO') # top half top_df = c.parse_to_df(self.ren_report_tsv, skiprows=1, nrows=24, header=0, delimiter='\t+', engine='python') self.assertEqual(list(top_df.columns), [ 'Hour', 'GEOTHERMAL', 'BIOMASS', 'BIOGAS', 'SMALL HYDRO', 'WIND TOTAL', 'SOLAR PV', 'SOLAR THERMAL' ]) self.assertEqual(len(top_df), 24) # bottom half bot_df = c.parse_to_df(self.ren_report_tsv, skiprows=3, nrows=24, header=0, delimiter='\t+', engine='python') self.assertEqual( list(bot_df.columns), ['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO']) self.assertEqual(len(bot_df), 24)
def setUp(self): self.c = client_factory('AZPS') handler = logging.StreamHandler() self.c.logger.addHandler(handler) self.c.logger.setLevel(logging.INFO) self.TZ_NAME = 'America/Phoenix' self.now = datetime.now(pytz.timezone(self.TZ_NAME)) self.today = datetime.combine(self.now.date(), time()) self.yesterday = self.today - timedelta(days=1) self.tomorrow = self.today + timedelta(days=1) self.last_month = self.today - timedelta(days=32) self.next_month = self.today + timedelta(days=32) self.sample = '"Time (MST)","Solar Aggregate (MW)","Wind Aggregate (MW)","Other Renewables Aggregate (MW)","Hydro Aggregate (MW)"\n\ "2015-07-18 00:00:05",0.028038,134.236,116.294,835.628\n\ "2015-07-18 00:00:15",0.079792,132.875,115.742,817.136\n\ "2015-07-18 00:00:25",0.101218,131.663,115.727,802.076\n\ "2015-07-18 00:00:35",0.119465,130.456,115.712,798.891\n\ "2015-07-18 00:00:45",0.180171,129.403,115.697,820.483\n\ "2015-07-18 00:00:55",0.268247,129.221,115.681,849.028\n\ "2015-07-18 00:01:05",0.29724,128.836,115.666,865.843\n\ "2015-07-18 00:01:15",0.281987,127.656,115.354,862.289\n\ ' self.sample_start = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 18, 0, 0)) self.sample_end = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 19, 0, 0))
def test_pjm_latest(self): expected = client_factory("PJM").get_generation(**self.latest_kwargs) received = tasks.get_generation("PJM", **self.latest_kwargs) for i in range(len(expected)): if expected[i]["timestamp"] == received[i]["timestamp"]: self.assertEqual(expected[i]["gen_MW"], received[i]["gen_MW"]) self.assertEqual(expected[i]["fuel_name"], received[i]["fuel_name"])
def test_pjm_latest(self): expected = client_factory('PJM').get_generation(**self.latest_kwargs) received = tasks.get_generation('PJM', **self.latest_kwargs) for i in range(len(expected)): if expected[i]['timestamp'] == received[i]['timestamp']: self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW']) self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])
def lmp_nyiso(self, start_time, end_time, product): if self.options['verbose']: 'Retrieving NYISO data' end_time = end_time - datetime.timedelta(seconds=1) nyiso = client_factory('NYISO') data = nyiso.get_lmp(data='lmp', node_id='ALL', start_at=start_time, end_at=end_time, market=product) data = pd.DataFrame(data) data.drop(['ba_name', 'freq', 'lmp_type', 'market'], axis=1, inplace=True) data.rename(columns={ 'PTID': 'iso_node_id', 'node_id': 'name', 'timestamp': 'datatime' }, inplace=True) return data
def test_parse_ren_report_bot(self): c = client_factory("CAISO") # bottom half bot_df = c.parse_to_df(self.ren_report_tsv, skiprows=29, nrows=24, header=0, delimiter="\t+", engine="python") self.assertEqual(list(bot_df.columns), ["Hour", "RENEWABLES", "NUCLEAR", "THERMAL", "IMPORTS", "HYDRO"]) self.assertEqual(len(bot_df), 24)
def test_fetch_oasis_ren_dam(self): c = client_factory("CAISO") ts = c.utcify("2014-05-08 12:00") payload = { "queryname": "SLD_REN_FCST", "market_run_id": "DAM", "startdatetime": (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format), "enddatetime": (ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format), } payload.update(c.base_payload) data = c.fetch_oasis(payload=payload) self.assertEqual(len(data), 4) self.assertEqual( str(data[0]), "<report_data>\n\ <data_item>RENEW_FCST_DA_MW</data_item>\n\ <opr_date>2014-05-08</opr_date>\n\ <interval_num>13</interval_num>\n\ <interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\ <interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\ <value>813.7</value>\n\ <trading_hub>NP15</trading_hub>\n\ <renewable_type>Solar</renewable_type>\n\ </report_data>", )
def _run_test(self, ba_name, **kwargs): # set up c = client_factory(ba_name) # get data data = c.get_generation(**kwargs) # test number self.assertGreater(len(data), 1) # test contents for dp in data: # test key names self.assertEqual(set(['gen_MW', 'ba_name', 'fuel_name', 'timestamp', 'freq', 'market']), set(dp.keys())) # test values self.assertEqual(dp['timestamp'].tzinfo, pytz.utc) self.assertIn(dp['fuel_name'], self.FUEL_CHOICES) self.assertIn(dp['ba_name'], self.BA_CHOICES) # test for numeric gen self.assertGreaterEqual(dp['gen_MW']+1, dp['gen_MW']) # test earlier than now if c.options.get('forecast', False): self.assertGreater(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) else: self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) # return return data
def test_fetch_oasis_demand_rtm(self): c = client_factory('CAISO') ts = c.utcify('2014-05-08 12:00') payload = { 'queryname': 'SLD_FCST', 'market_run_id': 'RTM', 'startdatetime': (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format), 'enddatetime': (ts + timedelta(minutes=20)).strftime(c.oasis_request_time_format), } payload.update(c.base_payload) data = c.fetch_oasis(payload=payload) self.assertEqual(len(data), 55) self.assertEqual( str(data[0]).lower(), '<report_data>\n\ <data_item>SYS_FCST_15MIN_MW</data_item>\n\ <resource_name>CA ISO-TAC</resource_name>\n\ <opr_date>2014-05-08</opr_date>\n\ <interval_num>50</interval_num>\n\ <interval_start_gmt>2014-05-08T19:15:00-00:00</interval_start_gmt>\n\ <interval_end_gmt>2014-05-08T19:30:00-00:00</interval_end_gmt>\n\ <value>26723</value>\n\ </report_data>'.lower())
def test_fetch_oasis_slrs_dam(self): c = client_factory('CAISO') ts = c.utcify('2014-05-08 12:00') payload = { 'queryname': 'ENE_SLRS', 'market_run_id': 'DAM', 'startdatetime': (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format), 'enddatetime': (ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format), } payload.update(c.base_payload) data = c.fetch_oasis(payload=payload) self.assertEqual(len(data), 17) self.assertEqual( str(data[0]).lower(), '<report_data>\n\ <data_item>ISO_TOT_EXP_MW</data_item>\n\ <resource_name>Caiso_Totals</resource_name>\n\ <opr_date>2014-05-08</opr_date>\n\ <interval_num>13</interval_num>\n\ <interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\ <interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\ <value>1044</value>\n\ </report_data>'.lower())
def test_fetch_csv_load_forecast(self): c = client_factory('NYISO') now = pytz.utc.localize(datetime.utcnow()) today = now.astimezone(pytz.timezone(c.TZ_NAME)).date() content = c.fetch_csv(today, 'isolf') self.assertEqual(content.split('\n')[0], '"Time Stamp","Capitl","Centrl","Dunwod","Genese","Hud Vl","Longil","Mhk Vl","Millwd","N.Y.C.","North","West","NYISO"')
def test_fetch_oasis_ren_dam(self): c = client_factory('CAISO') ts = c.utcify('2014-05-08 12:00') payload = { 'queryname': 'SLD_REN_FCST', 'market_run_id': 'DAM', 'startdatetime': (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format), 'enddatetime': (ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format), } payload.update(c.base_payload) data = c.fetch_oasis(payload=payload) self.assertEqual(len(data), 4) self.assertEqual( str(data[0]).lower(), '<report_data>\n\ <data_item>RENEW_FCST_DA_MW</data_item>\n\ <opr_date>2014-05-08</opr_date>\n\ <interval_num>13</interval_num>\n\ <interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\ <interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\ <value>813.7</value>\n\ <trading_hub>NP15</trading_hub>\n\ <renewable_type>Solar</renewable_type>\n\ </report_data>'.lower())
def test_utcify_index(self): c = client_factory('BPA') df = c.parse_to_df(self.wind_tsv, skiprows=6, header=0, delimiter='\t', index_col=0, parse_dates=True) utc_index = c.utcify_index(df.index) self.assertEqual(utc_index[0].to_pydatetime(), datetime(2014, 4, 15, 10+7, 10, tzinfo=pytz.utc)) self.assertEqual(len(df), len(utc_index))
def setUp(self): self.c = client_factory('AZPS') self.TZ_NAME = 'America/Phoenix' self.now = datetime.now(pytz.timezone(self.TZ_NAME)) self.today = datetime.combine(self.now.date(), time()) self.yesterday = self.today - timedelta(days=1) self.tomorrow = self.today + timedelta(days=1) self.last_month = self.today - timedelta(days=32) self.next_month = self.today + timedelta(days=32) self.sample = '"Time (MST)","Solar Aggregate (MW)","Wind Aggregate (MW)","Other Renewables Aggregate (MW)","Hydro Aggregate (MW)"\n\ "2015-07-18 00:00:05",0.028038,134.236,116.294,835.628\n\ "2015-07-18 00:00:15",0.079792,132.875,115.742,817.136\n\ "2015-07-18 00:00:25",0.101218,131.663,115.727,802.076\n\ "2015-07-18 00:00:35",0.119465,130.456,115.712,798.891\n\ "2015-07-18 00:00:45",0.180171,129.403,115.697,820.483\n\ "2015-07-18 00:00:55",0.268247,129.221,115.681,849.028\n\ "2015-07-18 00:01:05",0.29724,128.836,115.666,865.843\n\ "2015-07-18 00:01:15",0.281987,127.656,115.354,862.289\n\ ' self.sample_start = pytz.timezone(self.TZ_NAME).localize( datetime(2015, 7, 18, 0, 0)) self.sample_end = pytz.timezone(self.TZ_NAME).localize( datetime(2015, 7, 19, 0, 0))
def _run_pairwise_test(self, ba_name, **kwargs): # set up c = client_factory(ba_name) # get data data = c.get_trade(**kwargs) # test number self.assertGreaterEqual(len(data), 1) # test contents for dp in data: # test key names for key in ['source_ba_name', 'dest_ba_name', 'timestamp', 'freq', 'market']: self.assertIn(key, dp.keys()) self.assertEqual(len(dp.keys()), 6) # test values self.assertEqual(dp['timestamp'].tzinfo, pytz.utc) self.assertIn(dp['source_ba_name'], self.BA_CHOICES) # test for numeric value self.assertGreaterEqual(dp['export_MW']+1, dp['export_MW']) # test correct temporal relationship to now if c.options['forecast']: self.assertGreaterEqual(dp['timestamp'], datetime.now(pytz.utc)) else: self.assertLess(dp['timestamp'], datetime.now(pytz.utc)) # return return data
def test_get_ancillary_services(self): c = client_factory('CAISO') ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc) start = ts - timedelta(days=2) as_prc = c.get_ancillary_services('AS_CAISO_EXP', start_at=start, end_at=ts, market_run_id='DAM') self.assertEqual(len(as_prc), 48) self.assertGreaterEqual(min([i['timestamp'] for i in as_prc]), start - timedelta(minutes=5)) self.assertLessEqual(max([i['timestamp'] for i in as_prc]), ts + timedelta(minutes=5)) means = { 'SR': 1.685417, 'RU': 3.074583, 'RMU': 5.729167e-02, 'RMD': 3.620833e-01, 'RD': 3.901667, 'NR': 9.000000e-02, } for anc_type in means: dp = [i[anc_type] for i in as_prc] self.assertAlmostEqual(numpy.mean(dp), means[anc_type], places=6)
def test_names(self): for name in self.expected_names: if 'env_vars' in name: for var in name['env_vars']: environ[var] = name['env_vars'][var] c = client_factory(name['name']) self.assertIsNotNone(c)
def test_ercot_latest(self): # before min 32, will not have wind data if datetime.now().minute >= 32: # basic test data = self._run_test('ERCOT', latest=True) # test all timestamps are equal timestamps = [d['timestamp'] for d in data] self.assertEqual(len(set(timestamps)), 1) # test flags for dp in data: self.assertEqual(dp['market'], self.MARKET_CHOICES.hourly) self.assertEqual(dp['freq'], self.FREQUENCY_CHOICES.hourly) # test fuel names fuels = set([d['fuel_name'] for d in data]) expected_fuels = ['wind', 'nonwind'] for expfuel in expected_fuels: self.assertIn(expfuel, fuels) else: c = client_factory('ERCOT') data = c.get_generation(latest=True) self.assertEqual(len(data), 0)
def test_request_renewable_report(self): c = client_factory('CAISO') response = c.request( 'http://content.caiso.com/green/renewrpt/20140312_DailyRenewablesWatch.txt' ) self.assertIn('Hourly Breakdown of Renewable Resources (MW)', response.text)
def setUp(self): self.edata_inst_load = open( FIXTURES_DIR + '/pjm/InstantaneousLoad.html').read().encode('utf8') self.edata_forecast_load = open( FIXTURES_DIR + '/pjm/ForecastedLoadHistory.html').read().encode('utf8') self.c = client_factory('PJM')
def test_names(self): for name in self.expected_names: if 'env_vars' in name: for var in name['env_vars']: environ[var] = name['env_vars'][var]; c = client_factory(name['name']) self.assertIsNotNone(c)
def _run_test(self, ba, node_list, kwargs): expected = client_factory(ba).get_lmp(node_list, **kwargs) received = tasks.get_lmp(ba, node_list, **kwargs) for i in range(len(expected)): if expected[i]['timestamp'] == received[i]['timestamp']: self.assertEqual(expected[i]['lmp'], received[i]['lmp']) self.assertEqual(expected[i]['lmp_type'], received[i]['lmp_type'])
def _run_net_test(self, ba_name, **kwargs): # set up c = client_factory(ba_name) # get data data = c.get_trade(**kwargs) # test number self.assertGreaterEqual(len(data), 1) # test contents for dp in data: # test key names for key in ['ba_name', 'timestamp', 'freq', 'market']: self.assertIn(key, dp.keys()) self.assertEqual(len(dp.keys()), 5) # test values self.assertEqual(dp['timestamp'].tzinfo, pytz.utc) self.assertIn(dp['ba_name'], self.BA_CHOICES) # test for numeric value self.assertGreaterEqual(dp['net_exp_MW'] + 1, dp['net_exp_MW']) # test correct temporal relationship to now if c.options['forecast']: self.assertGreaterEqual(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) else: self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) # return return data
def test_get_AS_dataframe_latest(self): c = client_factory('CAISO') as_prc = c.get_AS_dataframe('AS_CAISO_EXP') # Could be 1 or 2 prices in last 61 minutes self.assertLessEqual(len(as_prc), 12) self.assertGreaterEqual(len(as_prc), 6)
def _run_test(self, ba_name, expect_data=True, **kwargs): # set up c = client_factory(ba_name) # get data data = c.get_load(**kwargs) # test number if expect_data: self.assertGreaterEqual(len(data), 1) else: self.assertEqual(data, []) # test contents for dp in data: # test key names self.assertEqual(set(['load_MW', 'ba_name', 'timestamp', 'freq', 'market']), set(dp.keys())) # test values self.assertEqual(dp['timestamp'].tzinfo, pytz.utc) self.assertIn(dp['ba_name'], self.BA_CHOICES) # test for numeric gen self.assertGreaterEqual(dp['load_MW']+1, dp['load_MW']) # test correct temporal relationship to now if c.options['forecast']: self.assertGreaterEqual(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) else: self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow())) # return return data
def test_get_AS_dataframe_empty(self): c = client_factory('CAISO') st = pytz.utc.localize(datetime.now() + timedelta(days=2)) et = st + timedelta(days=1) as_prc = c.get_AS_dataframe('AS_CAISO_EXP', start_at=st, end_at=et, market_run_id='DAM', anc_type='RU') self.assertTrue(as_prc.empty)
def test_get_AS_dataframe(self): c = client_factory('CAISO') ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc) start = ts - timedelta(days=2) as_prc = c.get_AS_dataframe(node_id='AS_CAISO_EXP', start_at=start, end_at=ts, market_run_id='DAM') self.assertEqual(len(as_prc), 288) self.assertAlmostEqual(as_prc['MW'].mean(), 1.528506944444443) grouped = as_prc.groupby('XML_DATA_ITEM') self.assertEqual(len(grouped), 6) means = { 'SP_CLR_PRC': 1.685417, 'RU_CLR_PRC': 3.074583, 'RMU_CLR_PRC': 5.729167e-02, 'RMD_CLR_PRC': 3.620833e-01, 'RD_CLR_PRC': 3.901667, 'NS_CLR_PRC': 9.000000e-02, } for group in means: self.assertAlmostEqual(grouped.get_group(group)['MW'].mean(), means[group], places=6) self.assertEqual(len(grouped.get_group(group)), 48)
def test_parse_systemconditions(self): """Test for a newly discovered edge case: an extra, empty `docdata` cell.""" c = client_factory('CAISO') soup = BeautifulSoup(self.systemconditions_html, 'lxml') c.handle_options() parsed_data = c.todays_outlook_time(soup) expected = datetime(2017, 1, 5, 21, 50, tzinfo=pytz.utc) self.assertEqual(parsed_data, expected)
def test_dt_index(self): c = client_factory('CAISO') df = c.parse_to_df(self.ren_report_tsv, skiprows=1, nrows=24, header=0, delimiter='\t+', engine='python') indexed = c.set_dt_index(df, date(2014, 3, 12), df['Hour']) self.assertEqual(type(indexed.index), pd.tseries.index.DatetimeIndex) self.assertEqual(indexed.index[0].hour, 7)
def test_get_AS_empty(self): """No AS data available 2 days in future""" c = client_factory('CAISO') st = pytz.utc.localize(datetime.now() + timedelta(days=2)) et = st + timedelta(days=1) as_prc = c.get_ancillary_services(node_id='AS_CAISO_EXP', start_at=st, end_at=et, market_run_id='DAM', anc_type='RU') self.assertEqual(as_prc, {})
def _run_test(self, ba, kwargs): expected = client_factory(ba).get_trade(**kwargs) received = tasks.get_trade(ba, **kwargs) self.assertEqual(len(expected), len(received)) for i in range(len(expected)): if expected[i]['timestamp'] == received[i]['timestamp']: self.assertEqual(expected[i]['net_exp_MW'], received[i]['net_exp_MW'])
def test_request_report(self): # get data c = client_factory('ERCOT') result = c._request_report('gen_hrly') # should be a list containing 1 dict self.assertEqual(len(result), 1) self.assertIn('SE_MW', result[0].keys())
def setUp(self): self.c = client_factory('CAISO') self.ren_report_tsv = read_fixture(self.c.__module__, 'ren_report.csv') self.sld_fcst_xml = read_fixture(self.c.__module__, 'sld_forecast.xml') self.ene_slrs_xml = read_fixture(self.c.__module__, 'ene_slrs.xml') self.sld_ren_fcst_xml = read_fixture(self.c.__module__, 'sld_ren_forecast.xml') self.systemconditions_html = read_fixture(self.c.__module__, 'systemconditions.html') self.todays_outlook_renewables = read_fixture(self.c.__module__, 'todays_outlook_renewables.html')
def test_bad_data_lmp_only(self, mock_request): mock_request.return_value = requests.get('https://httpbin.org/') c = client_factory('CAISO') ts = pytz.utc.localize(datetime(2015, 3, 1, 12)) start = ts - timedelta(hours=2) df = c.get_lmp_as_dataframe('CAISO_AS', latest=False, start_at=start, end_at=ts, lmp_only=False) self.assertIsInstance(df, pd.DataFrame)
def test_parse_ren_report_bot(self): c = client_factory('CAISO') # bottom half bot_df = c.parse_to_df(self.ren_report_tsv, skiprows=29, nrows=24, header=0, delimiter='\t+', engine='python') self.assertEqual(list(bot_df.columns), ['Hour', 'RENEWABLES', 'NUCLEAR', 'THERMAL', 'IMPORTS', 'HYDRO']) self.assertEqual(len(bot_df), 24)
def setUp(self): self.c = client_factory('CAISO') self.ren_report_tsv = read_fixture('ren_report.csv') self.sld_fcst_xml = read_fixture('sld_forecast.xml') self.ene_slrs_xml = read_fixture('ene_slrs.xml') self.sld_ren_fcst_xml = read_fixture('sld_ren_forecast.xml') self.systemconditions_html = read_fixture('systemconditions.html') self.todays_outlook_renewables = read_fixture( 'todays_outlook_renewables.html')