def latest_fuel_mix(self):
# set up request
url = self.base_url + '/ria/FuelMix.aspx?CSV=True'
# carry out request
response = self.request(url)
if not response:
return pd.DataFrame()
# test for valid content
if 'The page cannot be displayed' in response.text:
LOGGER.error('MISO: Error in source data for generation')
return pd.DataFrame()
# preliminary parsing
df = pd.read_csv(BytesIO(response.content), header=0, index_col=0, parse_dates=True)
# set index
df.index = self.utcify_index(df.index)
df.index.set_names(['timestamp'], inplace=True)
# set names and labels
df['fuel_name'] = df.apply(lambda x: self.fuels[x['CATEGORY']], axis=1)
df['gen_MW'] = df['ACT']
# return
return df[['fuel_name', 'gen_MW']]
def handle_options(self, **kwargs):
"""
Process and store keyword argument options.
"""
super(EIAClient, self).handle_options(**kwargs)
if not hasattr(self, 'BA'):
LOGGER.error('Balancing authority not set.')
raise ValueError('Balancing authority not set.')
if 'market' not in self.options:
if self.options['forecast']:
self.options['market'] = self.MARKET_CHOICES.dam
elif self.options['sliceable'] and self.options['data'] == 'gen':
self.options['market'] = self.MARKET_CHOICES.dam
else:
self.options['market'] = self.MARKET_CHOICES.hourly
if 'freq' not in self.options:
if self.options['forecast']:
self.options['freq'] = self.FREQUENCY_CHOICES.hourly
elif self.options['sliceable'] and self.options['data'] == 'gen':
self.options['freq'] = self.FREQUENCY_CHOICES.hourly
else:
self.options['freq'] = self.FREQUENCY_CHOICES.hourly
if 'yesterday' not in self.options:
self.options['yesterday'] = False
def get_trade(self, latest=False,
start_at=False, end_at=False, **kwargs):
# set args
self.handle_options(data='trade', latest=latest,
start_at=start_at, end_at=end_at, **kwargs)
# set up storage
parsed_data = []
# collect data
for this_date in self.dates():
# fetch
try:
df, mode = self.fetch_df(this_date)
except (HTTPError, ValueError):
LOGGER.warn('No data available in NVEnergy at %s' % this_date)
continue
# store
try:
parsed_data += self.parse_trade(df, this_date, mode)
except KeyError:
LOGGER.warn('Unparseable data available in NVEnergy at %s: %s' % (this_date, df))
continue
# return
return self.time_subset(parsed_data)
def _dst_active_hours_for_transition_day(self, local_dt_index):
"""
When attempting to localize a timezone-naive list of dates, the daylight savings status may be ambigous. This
method is meant as a fallback when the ambiguous='infer' datetime handling in pandas fails. It assumes
that the datetime index is a daylight saving transition day.
:param pandas.DatetimeIndex local_dt_index: A list of timezone-naive DatetimeIndex values.
:return: A list of bool values indicating whether daylight savings time is active for the list provided.
This returned list of boolean value is useful for passing to pandas 'ambiguous' kwarg.
:rtype: list
"""
dst_active_list = []
hour_idx = local_dt_index.hour
if len(hour_idx) > 3:
starting_timestamp = local_dt_index[0]
starting_month = starting_timestamp.month
starting_hour = starting_timestamp.hour
if starting_month == 3 and starting_hour == 0:
dst_active_list = [h > 1 for h in hour_idx]
elif starting_month == 11 and starting_hour == 0:
dst_active_list = [h < 2 for h in hour_idx]
elif 3 < starting_month < 11:
dst_active_list = [True for h in hour_idx]
elif starting_month < 3 or starting_month > 11:
dst_active_list = [False for h in hour_idx]
else:
LOGGER.warn("Uanble to infer fallback DST status for ambiguous DatetimeIndex values.")
return dst_active_list
def get_lmp(self, node_id='INTERNALHUB', latest=True, start_at=False, end_at=False, **kwargs):
# set args
self.handle_options(data='lmp', latest=latest,
start_at=start_at, end_at=end_at, node_id=node_id, **kwargs)
# get location id
try:
locationid = self.locations[node_id.upper()]
except KeyError:
raise ValueError('No LMP data available for location %s' % node_id)
# set up storage
raw_data = []
# collect raw data
for endpoint in self.request_endpoints(locationid):
# carry out request
data = self.fetch_data(endpoint, self.auth)
# pull out data
try:
raw_data += self.parse_json_lmp_data(data)
except ValueError as e:
LOGGER.warn(e)
continue
# parse and slice
df = self._parse_json(raw_data)
df = self.slice_times(df)
# return
return df.to_dict(orient='record')
def unzip(self, content):
"""
Unzip encoded data.
Returns the unzipped content as an array of strings, each representing one file's content
or returns None if an error was encountered.
***Previous behavior: Only returned the content from the first file***
"""
# create zip file
try:
filecontent = BytesIO(content)
except TypeError:
filecontent = StringIO(content)
try:
# have zipfile
z = zipfile.ZipFile(filecontent)
except zipfile.BadZipfile:
LOGGER.error('%s: unzip failure for content:\n%s' % (self.NAME, content))
return None
# have unzipped content
unzipped = [z.read(thisfile) for thisfile in z.namelist()]
z.close()
# return
return unzipped
def request(self, *args, **kwargs):
response = super(PJMClient, self).request(*args, **kwargs)
if response and response.status_code == 400:
LOGGER.warn('PJM request returned Bad Request %s' % response)
return None
return response
def get_load(self, latest=False, start_at=False, end_at=False,
forecast=False, **kwargs):
# set args
self.handle_options(data='load', latest=latest, forecast=forecast,
start_at=start_at, end_at=end_at, **kwargs)
# set up storage
raw_data = []
# collect raw data
for endpoint in self.request_endpoints():
# carry out request
data = self.fetch_data(endpoint, self.auth)
# pull out data
try:
raw_data += self.parse_json_load_data(data)
except ValueError as e:
LOGGER.warn(e)
continue
# parse data
try:
df = self._parse_json(raw_data)
except ValueError:
return []
df = self.slice_times(df)
# return
return self.serialize_faster(df, drop_index=True)
def fetch_forecast(self, date):
# construct url
datestr = date.strftime('%Y%m%d')
url = self.base_url + '/Library/Repository/Market%20Reports/' + datestr + '_da_ex.xls'
# make request with self.request for easier debugging, mocking
response = self.request(url)
if not response:
return pd.DataFrame()
if response.status_code == 404:
LOGGER.debug('No MISO forecast data available at %s' % datestr)
return pd.DataFrame()
xls = pd.read_excel(BytesIO(response.content))
# clean header
header_df = xls.iloc[:5]
df = xls.iloc[5:]
df.columns = ['hour_str'] + list(header_df.iloc[-1][1:])
# set index
idx = []
for hour_str in df['hour_str']:
# format like 'Hour 01' to 'Hour 24'
ihour = int(hour_str[5:]) - 1
local_ts = datetime(date.year, date.month, date.day, ihour)
idx.append(self.utcify(local_ts))
df.index = idx
df.index.set_names(['timestamp'], inplace=True)
# return
return df
def fetch_csvs(self, date, label):
# construct url
datestr = date.strftime('%Y%m%d')
if self.options['data'] == 'lmp':
url = '%s/%s/%s%s_zone.csv' % (self.base_url, label, datestr, label)
else:
url = '%s/%s/%s%s.csv' % (self.base_url, label, datestr, label)
# make request
response = self.request(url)
# if 200, return
if response and response.status_code == 200:
return [response.text]
# if failure, try zipped monthly data
datestr = date.strftime('%Y%m01')
if self.options['data'] == 'lmp':
url = '%s/%s/%s%s_zone_csv.zip' % (self.base_url, label, datestr, label)
else:
url = '%s/%s/%s%s_csv.zip' % (self.base_url, label, datestr, label)
# make request and unzip
response_zipped = self.request(url)
if response_zipped:
unzipped = self.unzip(response_zipped.content)
else:
return []
# return
if unzipped:
LOGGER.info('Failed to find daily %s data for %s but found monthly data, using that' % (self.options['data'], date))
return unzipped
else:
return []
def fetch_forecast(self, date):
# construct url
datestr = date.strftime("%Y%m%d")
url = self.base_url + "/Library/Repository/Market%20Reports/" + datestr + "_da_ex.xls"
# make request
try:
xls = pd.read_excel(url)
except HTTPError:
LOGGER.debug("No MISO forecast data available at %s" % datestr)
return pd.DataFrame()
# clean header
header_df = xls.iloc[:5]
df = xls.iloc[5:]
df.columns = ["hour_str"] + list(header_df.iloc[-1][1:])
# set index
idx = []
for hour_str in df["hour_str"]:
# format like 'Hour 01' to 'Hour 24'
ihour = int(hour_str[5:]) - 1
local_ts = datetime(date.year, date.month, date.day, ihour)
idx.append(self.utcify(local_ts))
df.index = idx
df.index.set_names(["timestamp"], inplace=True)
# return
return df
def utcify_index(self, local_index, tz_name=None):
"""
Convert a DateTimeIndex to UTC.
:param DateTimeIndex local_index: The local DateTimeIndex to be converted.
:param string tz_name: If local_ts is naive, it is assumed to be in timezone tz.
If tz is not provided, the client's default timezone is used.
:return: DatetimeIndex in UTC.
:rtype: DatetimeIndex
"""
# set up tz
if tz_name is None:
tz_name = self.TZ_NAME
# localize
try:
aware_local_index = local_index.tz_localize(tz_name)
except AmbiguousTimeError as e:
LOGGER.debug(e)
aware_local_index = local_index.tz_localize(tz_name, ambiguous='infer')
# except Exception as e:
# LOGGER.debug(e) # already aware
# print e
# aware_local_index = local_index
# convert to utc
aware_utc_index = aware_local_index.tz_convert('UTC')
# return
return aware_utc_index
def latest_fuel_mix(self):
# set up request
url = self.base_url + "/ria/FuelMix.aspx?CSV=True"
# carry out request
response = self.request(url)
if not response:
return pd.DataFrame()
# test for valid content
if "The page cannot be displayed" in response.text:
LOGGER.error("MISO: Error in source data for generation")
return pd.DataFrame()
# preliminary parsing
df = pd.read_csv(StringIO(response.text), header=0, index_col=0, parse_dates=True)
# set index
df.index = self.utcify_index(df.index)
df.index.set_names(["timestamp"], inplace=True)
# set names and labels
df["fuel_name"] = df.apply(lambda x: self.fuels[x["CATEGORY"]], axis=1)
df["gen_MW"] = df["ACT"]
# return
return df[["fuel_name", "gen_MW"]]
def fetch_todays_outlook_renewables(self):
# get renewables data
response = self.request(self.base_url_outlook+'renewables.html')
try:
return BeautifulSoup(response.content)
except AttributeError:
LOGGER.warn('No response for CAISO today outlook renewables')
return None
def get_load(self, latest=False, start_at=None, end_at=None, forecast=False, **kwargs):
# set args
self.handle_options(data='load', latest=latest,
start_at=start_at, end_at=end_at, forecast=forecast,
**kwargs)
if self.options['forecast']:
# fetch from eData
df = self.fetch_edata_series('ForecastedLoadHistory', {'name': 'PJM RTO Total'})
sliced = self.slice_times(df)
sliced.columns = ['load_MW']
# format
extras = {
'freq': self.FREQUENCY_CHOICES.hourly,
'market': self.MARKET_CHOICES.dam,
'ba_name': self.NAME,
}
data = self.serialize_faster(sliced, extras=extras)
# return
return data
elif self.options['end_at'] and self.options['end_at'] < datetime.now(pytz.utc) - timedelta(hours=1):
df = self.fetch_historical_load(self.options['start_at'].year)
sliced = self.slice_times(df)
# format
extras = {
'freq': self.FREQUENCY_CHOICES.hourly,
'market': self.MARKET_CHOICES.dam,
'ba_name': self.NAME,
}
data = self.serialize_faster(sliced, extras=extras)
# return
return data
else:
# handle real-time
load_ts, load_val = self.fetch_edata_point('InstantaneousLoad', 'PJM RTO Total', 'MW')
# fall back to OASIS
if not (load_ts and load_val):
load_ts, load_val = self.fetch_oasis_data()
if not (load_ts and load_val):
LOGGER.warn('No PJM latest load data')
return []
# format and return
return [{
'timestamp': load_ts,
'freq': self.FREQUENCY_CHOICES.fivemin,
'market': self.MARKET_CHOICES.fivemin,
'load_MW': load_val,
'ba_name': self.NAME,
}]
def time_from_soup(self, soup):
"""
Returns a UTC timestamp if one is found in the soup,
or None if an error was encountered.
"""
ts_elt = soup.find(class_='ts')
if not ts_elt:
LOGGER.error('PJM: Timestamp not found in soup:\n%s' % soup)
return None
return self.utcify(ts_elt.string)
def fetch_oasis(self, payload={}, return_all_files=False):
"""
Returns a list of report data elements, or an empty list if an error was encountered.
If return_all_files=False, returns only the content from the first file in the .zip -
this is the default behavior and was used in earlier versions of this function.
If return_all_files=True, will return an array representing the content from each file.
This is useful for processing LMP data or other fields where multiple price components are returned in a zip.
"""
# set up storage
raw_data = []
if return_all_files is True:
default_return_val = []
else:
default_return_val = ''
# try get
response = self.request(self.base_url_oasis, params=payload)
if not response:
return default_return_val
# read data from zip
# This will be an array of content if successful, and None if unsuccessful
content = self.unzip(response.content)
if not content:
return default_return_val
# check xml content for errors
soup = BeautifulSoup(content[0], 'lxml')
error = soup.find('m:error')
if error:
code = error.find('m:err_code')
desc = error.find('m:err_desc')
msg = 'XML error for CAISO OASIS with payload %s: %s %s' % (payload, code, desc)
LOGGER.error(msg)
return default_return_val
# return xml or csv data
if payload.get('resultformat', False) == 6:
# If we requested CSV files
if return_all_files:
return content
else:
return content[0]
else:
# Return XML content
if return_all_files:
raw_data = [BeautifulSoup(thisfile).find_all('report_data') for thisfile in content]
return raw_data
else:
raw_data = soup.find_all('report_data')
return raw_data
def utcify_index(self, local_index, tz_name=None, tz_col=None):
"""
Convert a DateTimeIndex to UTC.
:param DateTimeIndex local_index: The local DateTimeIndex to be converted.
:param string tz_name: If local_ts is naive, it is assumed to be in timezone tz.
If tz is not provided, the client's default timezone is used.
:return: DatetimeIndex in UTC.
:rtype: DatetimeIndex
"""
# set up tz
if tz_name is None:
tz_name = self.TZ_NAME
# use tz col if given
if tz_col is not None:
# it seems like we shouldn't have to iterate, but all the smart ways aren't working
aware_utc_list = []
for i in range(len(local_index)):
try:
aware_local_ts = pytz.timezone(tz_col[i]).localize(
local_index[i])
except pytz.UnknownTimeZoneError:
# fall back to local ts
aware_local_ts = pytz.timezone(tz_name).localize(
local_index[i])
# utcify
aware_utc_ts = self.utcify(aware_local_ts)
aware_utc_list.append(aware_utc_ts)
# indexify
aware_utc_index = pd.DatetimeIndex(aware_utc_list)
else:
# localize
try:
aware_local_index = local_index.tz_localize(tz_name)
except AmbiguousTimeError as e:
LOGGER.debug(e)
aware_local_index = local_index.tz_localize(tz_name,
ambiguous='infer')
except TypeError as e:
# already aware
LOGGER.debug(e)
aware_local_index = local_index
# convert to utc
aware_utc_index = aware_local_index.tz_convert('UTC')
# return
return aware_utc_index
def fetch_oasis(self, payload={}, return_all_files=False):
"""
Returns a list of report data elements, or an empty list if an error was encountered.
If return_all_files=False, returns only the content from the first file in the .zip -
this is the default behavior and was used in earlier versions of this function.
If return_all_files=True, will return an array representing the content from each file.
This is useful for processing LMP data or other fields where multiple price components are returned in a zip.
"""
# set up storage
raw_data = []
if return_all_files is True:
default_return_val = []
else:
default_return_val = ''
# try get
response = self.request(self.base_url_oasis, params=payload)
if not response:
return default_return_val
# read data from zip
# This will be an array of content if successful, and None if unsuccessful
content = self.unzip(response.content)
if not content:
return default_return_val
# check xml content for errors
soup = BeautifulSoup(content[0], 'xml')
error = soup.find(['error', 'ERROR'])
if error:
code = error.find(['err_code', 'ERR_CODE'])
desc = error.find(['err_desc', 'ERR_DESC'])
msg = 'XML error for CAISO OASIS with payload %s: %s %s' % (payload, code, desc)
LOGGER.error(msg)
return default_return_val
# return xml or csv data
if payload.get('resultformat', False) == 6:
# If we requested CSV files
if return_all_files:
return content
else:
return content[0]
else:
# Return XML content
if return_all_files:
raw_data = [BeautifulSoup(thisfile, 'xml').find_all(['REPORT_DATA', 'report_data']) for thisfile in content]
return raw_data
else:
raw_data = soup.find_all(['REPORT_DATA', 'report_data'])
return raw_data
def _assert_entries_1hr_apart(self, result_ts):
prev_entry = None
for entry in result_ts:
if prev_entry:
seconds_delta = (entry['timestamp'] - prev_entry['timestamp']).total_seconds()
if seconds_delta > 3600:
LOGGER.error('prev_entry timestamp: ' + str(
prev_entry['timestamp'].astimezone(pytz.timezone(self.nbpower_client.TZ_NAME))
))
LOGGER.error('entry timestamp: ' + str(
entry['timestamp'].astimezone(pytz.timezone(self.nbpower_client.TZ_NAME))
))
self.assertEqual(3600, seconds_delta)
prev_entry = entry
def get_load(self, latest=False, yesterday=False, start_at=False, end_at=False, **kwargs):
super(AESOClient, self).handle_options(latest=latest, yesterday=yesterday, start_at=start_at, end_at=end_at,
**kwargs)
if latest:
return self._get_latest_report(request_type=ParserFormat.load)
elif self.options.get('start_at', None) and self.options.get('end_at', None):
earliest_load_dt = self.mtn_tz.localize(datetime(year=2000, month=1, day=1, hour=0, minute=0, second=0))
latest_load_dt = self.local_now().replace(hour=23, minute=59, second=59, microsecond=999999)
start_at = max(self.options['start_at'], earliest_load_dt).astimezone(self.mtn_tz)
end_at = min(self.options['end_at'], latest_load_dt).astimezone(self.mtn_tz)
return self._get_load_for_date_range(start_at=start_at, end_at=end_at)
else:
LOGGER.warn('No valid options were supplied.')
def get_lmp(self, node_id, latest=True, start_at=False, end_at=False, **kwargs):
# set args
self.handle_options(data='lmp', latest=latest,
start_at=start_at, end_at=end_at, **kwargs)
# get location id
try:
locationid = self.locations[node_id.upper()]
except KeyError:
raise ValueError('No LMP data available for location %s' % node_id)
# set up storage
raw_data = []
parsed_data = []
# collect raw data
for endpoint in self.request_endpoints(locationid):
# carry out request
data = self.fetch_data(endpoint, self.auth)
# pull out data
try:
raw_data += self.parse_json_lmp_data(data)
except ValueError as e:
LOGGER.warn(e)
continue
# parse data
for raw_dp in raw_data:
# set up storage
parsed_dp = {}
# add values
parsed_dp['timestamp'] = self.utcify(raw_dp['BeginDate'])
parsed_dp['lmp'] = raw_dp['LmpTotal']
parsed_dp['ba_name'] = self.NAME
parsed_dp['market'] = self.options['market']
parsed_dp['freq'] = self.options['frequency']
parsed_dp['node_id'] = node_id
parsed_dp['lmp_type'] = 'energy'
# add to full storage
to_store = True
if self.options['sliceable']:
if self.options['start_at'] > parsed_dp['timestamp'] or self.options['end_at'] < parsed_dp['timestamp']:
to_store = False
if to_store:
parsed_data.append(parsed_dp)
return parsed_data
def get_trade(self,
latest=False,
yesterday=False,
start_at=None,
end_at=None,
**kwargs):
trade_ts = list([])
self.handle_options(latest=latest,
yesterday=yesterday,
start_at=start_at,
end_at=end_at,
**kwargs)
inter_sched_flow_handler = IntertieScheduleFlowReportHandler(
ieso_client=self)
adequacy_handler = AdequacyReportHandler(ieso_client=self)
if self.options.get('latest', False):
self._get_latest_report_trimmed(
result_ts=trade_ts,
report_handler=inter_sched_flow_handler,
parser_format=ParserFormat.trade)
elif self.options.get('start_at', None) and self.options.get(
'end_at', None):
if self.options.get('historical', False):
range_start = max(
self.options['start_at'],
inter_sched_flow_handler.earliest_available_datetime())
range_end = min(
self.options['end_at'],
inter_sched_flow_handler.latest_available_datetime())
self._get_report_range(result_ts=trade_ts,
report_handler=inter_sched_flow_handler,
parser_format=ParserFormat.trade,
range_start=range_start,
range_end=range_end)
if self.options.get('forecast', False):
range_start = max(
self.options['start_at'],
inter_sched_flow_handler.latest_available_datetime(),
adequacy_handler.earliest_available_datetime())
range_end = min(self.options['end_at'],
adequacy_handler.latest_available_datetime())
self._get_report_range(result_ts=trade_ts,
report_handler=adequacy_handler,
parser_format=ParserFormat.trade,
range_start=range_start,
range_end=range_end)
else:
LOGGER.warn('No valid options were supplied.')
return trade_ts
def parse_oasis_renewable(self, raw_data):
"""Parse raw data output of fetch_oasis for renewables."""
# set up storage
preparsed_data = {}
parsed_data = []
# extract values from xml
for raw_soup_dp in raw_data:
# set up storage for timestamp
ts = self.utcify(
raw_soup_dp.find(['INTERVAL_START_GMT',
'interval_start_gmt']).string)
if ts not in preparsed_data:
preparsed_data[ts] = {'wind': 0, 'solar': 0}
# store generation value
try:
fuel_name = raw_soup_dp.find(
['RENEWABLE_TYPE', 'renewable_type']).string.lower()
gen_MW = float(raw_soup_dp.find(['VALUE', 'value']).string)
preparsed_data[ts][fuel_name] += gen_MW
except TypeError:
LOGGER.error('Error in schema for CAISO OASIS result %s' %
raw_soup_dp.prettify())
continue
# collect values into dps
freq = self.options.get('freq', self.FREQUENCY_CHOICES.hourly)
market = self.options.get('market', self.MARKET_CHOICES.hourly)
for ts, preparsed_dp in preparsed_data.items():
# set up base
base_parsed_dp = {
'timestamp': ts,
'freq': freq,
'market': market,
'gen_MW': 0,
'ba_name': self.NAME
}
# collect data
for fuel_name in ['wind', 'solar']:
parsed_dp = copy.deepcopy(base_parsed_dp)
parsed_dp['fuel_name'] = fuel_name
parsed_dp['gen_MW'] += preparsed_dp[fuel_name]
parsed_data.append(parsed_dp)
# return
return parsed_data
def handle_ba_limitations(self):
"""Handle BA limitations"""
today = pytz.utc.localize(datetime.utcnow()).astimezone(
pytz.timezone(self.TZ_NAME))
two_days_ago = today - timedelta(days=2)
load_not_supported_bas = [
'DEAA', 'EEI', 'GRIF', 'GRMA', 'GWA', 'HGMA', 'SEPA', 'WWA', 'YAD'
]
delay_bas = [
'AEC', 'DOPD', 'GVL', 'HST', 'NSB', 'PGE', 'SCL', 'TAL', 'TIDC',
'TPWR'
]
canada_mexico = [
'IESO', 'BCTC', 'MHEB', 'AESO', 'HQT', 'NBSO', 'CFE', 'SPC'
]
if self.BA in delay_bas:
if self.options['end_at'] and self.options['end_at'] > two_days_ago:
LOGGER.error('No data for %s due to 2 day delay' % self.BA)
raise ValueError('No data: 2 day delay for this BA.')
elif self.options['yesterday']:
LOGGER.error('No data for %s due to 2 day delay' % self.BA)
raise ValueError('No data: 2 day delay for this BA.')
elif self.options['forecast']:
raise ValueError('No data: 2 day delay for this BA.')
if self.BA in load_not_supported_bas:
if self.options['data'] == 'load':
LOGGER.error('Load data not supported for %s' % self.BA)
raise ValueError('Load data not supported for this BA.')
if self.BA in canada_mexico:
LOGGER.error('Data not supported for %s' % self.BA)
raise ValueError(
'Data not currently supported for Canada and Mexico')
def handle_ba_limitations(self):
"""Handle BA limitations"""
today = pytz.utc.localize(datetime.utcnow()).astimezone(pytz.timezone(self.TZ_NAME))
two_days_ago = today - timedelta(days=2)
load_not_supported_bas = ['DEAA', 'EEI', 'GRIF', 'GRMA', 'GWA',
'HGMA', 'SEPA', 'WWA', 'YAD']
delay_bas = ['AEC', 'DOPD', 'GVL', 'HST', 'NSB', 'PGE', 'SCL',
'TAL', 'TIDC', 'TPWR']
canada_mexico = ['IESO', 'BCTC', 'MHEB', 'AESO', 'HQT', 'NBSO',
'CFE', 'SPC']
if self.BA in delay_bas:
if self.options['end_at'] and self.options['end_at'] > two_days_ago:
LOGGER.error('No data for %s due to 2 day delay' % self.BA)
raise ValueError('No data: 2 day delay for this BA.')
elif self.options['yesterday']:
LOGGER.error('No data for %s due to 2 day delay' % self.BA)
raise ValueError('No data: 2 day delay for this BA.')
elif self.options['forecast']:
raise ValueError('No data: 2 day delay for this BA.')
if self.BA in load_not_supported_bas:
if self.options['data'] == 'load':
LOGGER.error('Load data not supported for %s' % self.BA)
raise ValueError('Load data not supported for this BA.')
if self.BA in canada_mexico:
LOGGER.error('Data not supported for %s' % self.BA)
raise ValueError('Data not currently supported for Canada and Mexico')
def utcify_index(self, local_index, tz_name=None, tz_col=None):
"""
Convert a DateTimeIndex to UTC.
:param DateTimeIndex local_index: The local DateTimeIndex to be converted.
:param string tz_name: If local_ts is naive, it is assumed to be in timezone tz.
If tz is not provided, the client's default timezone is used.
:return: DatetimeIndex in UTC.
:rtype: DatetimeIndex
"""
# set up tz
if tz_name is None:
tz_name = self.TZ_NAME
# use tz col if given
if tz_col is not None:
# it seems like we shouldn't have to iterate, but all the smart ways aren't working
aware_utc_list = []
for i in range(len(local_index)):
try:
aware_local_ts = pytz.timezone(tz_col[i]).localize(local_index[i])
except pytz.UnknownTimeZoneError:
# fall back to local ts
aware_local_ts = pytz.timezone(tz_name).localize(local_index[i])
# utcify
aware_utc_ts = self.utcify(aware_local_ts)
aware_utc_list.append(aware_utc_ts)
# indexify
aware_utc_index = pd.DatetimeIndex(aware_utc_list)
else:
# localize
try:
aware_local_index = local_index.tz_localize(tz_name)
except AmbiguousTimeError as e:
LOGGER.debug(e)
aware_local_index = local_index.tz_localize(tz_name, ambiguous='infer')
except TypeError as e:
# already aware
LOGGER.debug(e)
aware_local_index = local_index
# convert to utc
aware_utc_index = aware_local_index.tz_convert('UTC')
# return
return aware_utc_index
def _assert_entires_5min_apart(self, result_ts):
prev_entry = None
for entry in result_ts:
if prev_entry:
seconds_delta = (entry['timestamp'] -
prev_entry['timestamp']).total_seconds()
if seconds_delta > 300:
LOGGER.error('prev_entry timestamp: ' +
str(prev_entry['timestamp'].astimezone(
pytz.timezone(self.ieso_client.TZ_NAME))))
LOGGER.error('entry timestamp: ' +
str(entry['timestamp'].astimezone(
pytz.timezone(self.ieso_client.TZ_NAME))))
self.assertEqual(300, seconds_delta)
prev_entry = entry
def _get_load_forecast_report(self):
"""
:return: List of dicts, each with keys ``[ba_name, timestamp, freq, market, load_MW]``.
Timestamps are in UTC.
:rtype: list
"""
load_ts = list([])
forecast_url_base = 'http://tso.nbpower.com/reports%20%26%20assessments/load%20forecast/hourly/'
forecast_filename_fmt = '%Y-%m-%d %H.csv'
earliest_forecast = copy(self.atlantic_now).replace(minute=0,
second=0,
microsecond=0)
latest_forecast = earliest_forecast + timedelta(hours=3)
if self.local_start_at <= latest_forecast:
forecast_filename = earliest_forecast.strftime(
forecast_filename_fmt)
load_forecast_url = forecast_url_base + quote(forecast_filename)
response = self.request(load_forecast_url)
response_body = BytesIO(response.content)
response_df = read_csv(response_body,
names=['timestamp', 'load'],
usecols=[0, 1],
dtype={'load': float},
parse_dates=[0],
date_parser=self.parse_forecast_timestamps)
for idx, row in response_df.iterrows():
if self.atlantic_now <= row.timestamp and self.local_start_at <= row.timestamp <= self.local_end_at:
row_pd_timestamp = Timestamp(
row.timestamp.astimezone(pytz.utc))
# In the event of a duplicate timestamp (e.g. daylight savings transition hours), use latest value.
if len(load_ts) > 0 and load_ts[-1][
'timestamp'] == row_pd_timestamp:
del load_ts[-1:]
load_ts.append({
'ba_name': self.NAME,
'timestamp': row_pd_timestamp,
'freq': self.FREQUENCY_CHOICES.hourly,
'market': self.MARKET_CHOICES.dam,
'load_MW': row.load
})
else:
LOGGER.warn('The latest load forecast available is ' +
str(latest_forecast) +
'. The requested start_at must be before this time.')
return load_ts
def get_latest_fuel_mix(self):
# set up request
url = self.base_url + '/ria/FuelMix.aspx?CSV=True'
# carry out request
response = self.request(url)
if not response:
return None
# test for valid content
if 'The page cannot be displayed' in response.text:
LOGGER.error('MISO: Error in source data for generation')
return None
# return good
return response.content
def get_load(self, latest=False, yesterday=False, start_at=False, end_at=False, **kwargs):
self.handle_options(latest=latest, yesterday=yesterday, start_at=start_at, end_at=end_at, data='load')
loads = []
if latest:
self._load_latest(loads)
elif self._is_valid_date_range():
self._hourly_range(loads)
else:
if self.options.get('forecast', False):
LOGGER.warn(self.NAME + ': Load forecasts are not supported.')
else:
msg = '%s: Requested date range %s to %s is outside range of available data from %s to %s.' % \
(self.NAME, self.options.get('start_at', None), self.options.get('end_at', None),
self.options.get('earliest_data_at', None), self.options.get('latest_data_at', None))
LOGGER.warn(msg)
return loads
def parse_ace_data(self, content):
if not content:
return pd.DataFrame()
# preliminary parsing
df = pd.DataFrame(content, columns=['instantEST', 'value'])
df['instantEST'] = pd.to_datetime(df['instantEST'])
df.set_index('instantEST', inplace=True)
# set index
try:
df.index = self.utcify_index(df.index)
except AttributeError:
LOGGER.error('MISO: Error in source data for ACE %s' % content)
return pd.DataFrame()
df.index.set_names(['timestamp'], inplace=True)
return df
def get_latest_fuel_mix(self):
# set up request
url = self.base_url + '/ria/FuelMix.aspx?CSV=True'
# carry out request
response = self.request(url)
if not response:
return None
# test for valid content
if 'The page cannot be displayed' in response.text:
LOGGER.error('MISO: Error in source data for generation')
return None
# return good
return response.content
def get_latest_fuel_mix(self):
# set up request
url = self.base_url + '?messageType=getfuelmix&returnType=csv'
# carry out request
response = self.request(url)
if not response:
return None
# test for valid content
if 'The page cannot be displayed' in response.text:
LOGGER.error('MISO: Error in source data for generation')
return None
# return good
return response.content
def _format_start_end(self, data):
formatted_sliced = []
if 'gen' not in self.options['data']:
formatted_sliced = [i for i in data if i['timestamp'] >= self.options['start_at'] and i['timestamp'] <= self.options['end_at']]
else:
try:
yesterday = (self.local_now() - timedelta(days=2)).replace(hour=0, minute=0,
second=0, microsecond=0)
tomorrow = (self.local_now() + timedelta(days=1)).replace(hour=23, minute=0,
second=0, microsecond=0)
assert ((self.options['start_at'] >= yesterday) and (self.options['end_at'] <= tomorrow))
formatted_sliced = [i for i in data if i['timestamp'] >= self.options['start_at'] and i['timestamp'] <= self.options['end_at']]
except:
LOGGER.error('Generation data error for %s' % self.BA)
raise ValueError('Generation data is available for the \
previous and current day.', self.options)
return formatted_sliced
def get_trade(self, latest=False, yesterday=False, start_at=False, end_at=False, **kwargs):
self.handle_options(latest=latest, yesterday=yesterday, start_at=start_at, end_at=end_at, data='trade')
# http://yukonenergy.ca/energy-in-yukon/electricity-101/electricity-library/whats-an-isolated-grid-and-what-does-that-mean-for-me
LOGGER.warn('Yukon Energy is an isolated grid. Trade will always be zero.')
trades = []
hourly_rounded_dt = self.options.get('start_at').replace(minute=0, second=0, microsecond=0)
while hourly_rounded_dt <= self.options.get('end_at'):
if self.options['start_at'] <= hourly_rounded_dt <= self.options['end_at']:
trades.append({
'ba_name': self.NAME,
'timestamp': Timestamp(hourly_rounded_dt),
'freq': self.FREQUENCY_CHOICES.hourly,
'market': self.MARKET_CHOICES.hourly,
'net_exp_MW': 0
})
hourly_rounded_dt = hourly_rounded_dt + timedelta(hours=1)
return trades
def fetch_entsoe(self, url, payload, count=0):
if not getattr(self, 'session', None):
self.auth()
r = self.request(url, params=payload)
# TODO error checking
if len(r.text) == 0:
if count > 3: # try 3 times to get response
LOGGER.warn('Request failed, no response found after %i attempts' % count)
return False
# throttled
sleep(5)
return self.fetch_entsoe(url, payload, count + 1)
if 'UNKNOWN_EXCEPTION' in r.text:
LOGGER.warn('UNKNOWN EXCEPTION')
return False
return r.text
def val_from_soup(self, soup, key):
"""
Returns a float value if one is found in the soup for the provided key,
or None if an error was encountered.
"""
for elt in soup.find_all('td'):
try:
if elt.find('a').string == key:
# numbers may have commas in the thousands
val_str = elt.next_sibling.string.replace(',', '')
return float(val_str)
except AttributeError: # no 'a' child
continue
# no value found
LOGGER.error('PJM: Value for %s not found in soup:\n%s' % (key, soup))
return None
def fetch_entsoe(self, url, payload, count=0):
if not getattr(self, 'session', None):
self.auth()
r = self.request(url, params=payload)
# TODO error checking
if len(r.text) == 0:
if count > 3: # try 3 times to get response
LOGGER.warn('Request failed, no response found after %i attempts' % count)
return False
# throttled
sleep(5)
return self.fetch_entsoe(url, payload, count + 1)
if 'UNKNOWN_EXCEPTION' in r.text:
LOGGER.warn('UNKNOWN EXCEPTION')
return False
return r.text
def get_load(self, latest=False, yesterday=False, start_at=False,
end_at=False, forecast=False, **kwargs):
"""
Scrape and parse load data.
"""
self.handle_options(data='load', latest=latest, start_at=start_at,
end_at=end_at, **kwargs)
self.handle_ba_limitations()
self.format_url()
result = self.request(self.url)
if result is not None:
result_json = json.loads(result.text)
result_formatted = self.format_result(result_json)
return result_formatted
else:
LOGGER.error('No results for %s' % self.BA)
return []
def get_latest_ace(self):
# set up request
url = self.base_url + '?messageType=getACE&returnType=json'
# carry out request
response = self.request(url)
if not response:
return None
# test for valid content
if 'The page cannot be displayed' in response.text:
LOGGER.error('MISO: Error in source data for ACE')
return None
ace = json.loads(response.content)
# return good
data = self.parse_ace_data(ace['ACE'])
return self.serialize_faster(data)
def fetch_csvs(self, date, label):
# construct url
datestr = date.strftime('%Y%m%d')
if self.options['data'] == 'zone_lmp':
url = '%s/%s/%s%s_zone.csv' % (self.base_url, label, datestr,
label)
elif self.options['data'] == 'lmp':
url = '%s/%s/%s%s_gen.csv' % (self.base_url, label, datestr, label)
else:
url = '%s/%s/%s%s.csv' % (self.base_url, label, datestr, label)
# make request
response = self.request(url)
# if 200, return
if response and response.status_code == 200:
return [response.text]
# if failure, try zipped monthly data
datestr = date.strftime('%Y%m01')
if self.options['data'] == 'zone_lmp':
url = '%s/%s/%s%s_zone_csv.zip' % (self.base_url, label, datestr,
label)
elif self.options['data'] == 'lmp':
url = '%s/%s/%s%s_gen_csv.zip' % (self.base_url, label, datestr,
label)
else:
url = '%s/%s/%s%s_csv.zip' % (self.base_url, label, datestr, label)
# make request and unzip
response_zipped = self.request(url)
if response_zipped:
unzipped = self.unzip(response_zipped.content)
else:
return []
# return
if unzipped:
LOGGER.info(
'Failed to find daily %s data for %s but found monthly data, using that'
% (self.options['data'], date))
return unzipped
else:
return []
def get_generation(self, latest=False, yesterday=False,
start_at=False, end_at=False, **kwargs):
"""
Scrape and parse generation fuel mix data.
Note: Generation may be quite low for HST and NSB BAs.
"""
self.handle_options(data='gen', latest=latest, yesterday=yesterday,
start_at=start_at, end_at=end_at, **kwargs)
self.handle_ba_limitations()
self.format_url()
result = self.request(self.url)
if result is not None:
result_json = json.loads(result.text)
result_formatted = self.format_result(result_json)
return result_formatted
else:
LOGGER.error('No results for %s' % self.BA)
return []
def format_url(self):
"""Set EIA API URL based on options"""
if self.options['data'] == 'gen':
if self.options['forecast']:
LOGGER.error('Forecast not supported for generation.')
raise ValueError('Forecast not supported for generation.')
else:
self.set_url('series', '-ALL.NG.H')
elif self.options['data'] == 'load':
if self.options['forecast']:
self.set_url('series', '-ALL.DF.H')
else:
self.set_url('series', '-ALL.D.H')
elif self.options['data'] == 'trade':
if self.options['forecast']:
LOGGER.error('Forecast not supported for generation.')
raise ValueError('Forecast not supported for trade.')
elif self.options['end_at']:
if self.options['end_at'] > pytz.utc.localize(
datetime.utcnow()):
LOGGER.error('Forecast not supported for generation.')
raise ValueError('Forecast not supported for trade.')
else:
self.set_url('series', '-ALL.TI.H')
else:
self.set_url('series', '-ALL.TI.H')
def parse_forecast(self, df):
sliced = self.slice_times(df)
if self.options['data'] == 'gen':
try:
sliced['gen_MW'] = 1000.0 * sliced['Supply Cleared (GWh) - Physical']
sliced['fuel_name'] = 'other'
return sliced[['gen_MW', 'fuel_name']]
except KeyError:
LOGGER.warn('MISO genmix error: missing key %s in %s' % ('Supply Cleared (GWh) - Physical', sliced.columns))
return pd.DataFrame()
elif self.options['data'] == 'load':
try:
sliced['load_MW'] = 1000.0 * (sliced['Demand Cleared (GWh) - Physical - Fixed'] +
sliced['Demand Cleared (GWh) - Physical - Price Sen.'])
return sliced['load_MW']
except KeyError:
LOGGER.warn('MISO load error: missing key %s in %s' % ('Demand Cleared (GWh) - Physical - Fixed', sliced.columns))
return pd.DataFrame()
elif self.options['data'] == 'trade':
try:
sliced['net_exp_MW'] = -1000.0 * sliced['Net Scheduled Imports (GWh)']
return sliced['net_exp_MW']
except KeyError:
LOGGER.warn('MISO trade error: missing key %s in %s' % ('Net Scheduled Imports (GWh)', sliced.columns))
return pd.DataFrame()
else:
raise ValueError('Can only parse MISO forecast gen, load, or trade data, not %s'
% self.options['data'])
def get_generation(self, latest=False, yesterday=False, start_at=None, end_at=None, **kwargs):
generation_ts = list([])
self.handle_options(latest=latest, yesterday=yesterday, start_at=start_at, end_at=end_at, **kwargs)
gen_out_cap_handler = GeneratorOutputCapabilityReportHandler(ieso_client=self)
gen_out_by_fuel_handler = GeneratorOutputByFuelHourlyReportHandler(ieso_client=self)
adequacy_handler = AdequacyReportHandler(ieso_client=self)
if self.options.get('latest', False):
self._get_latest_report_trimmed(result_ts=generation_ts, report_handler=gen_out_cap_handler,
parser_format=ParserFormat.generation)
elif self.options.get('start_at', None) and self.options.get('end_at', None):
# For long time ranges more than hour ending 1, seven days in the past, it is more efficient to request the
# Generator Output by Fuel Type Hourly Report rather than repeated calls to the Generator Output and
# Capability Report.
# TODO Minor optimization, but this actually check if the start/end range is greater than 7 days.
if self.options['start_at'] < self.local_start_of_day.replace(hour=1) - timedelta(days=7):
self.timeout_seconds = 90 # These reports can get rather large ~7MB for a full year.
range_start = max(self.options['start_at'], gen_out_by_fuel_handler.earliest_available_datetime())
range_end = min(self.options['end_at'], gen_out_by_fuel_handler.latest_available_datetime())
self._get_report_range(result_ts=generation_ts, report_handler=gen_out_by_fuel_handler,
parser_format=ParserFormat.generation, range_start=range_start,
range_end=range_end)
elif self.options.get('historical', False):
range_start = max(self.options['start_at'], gen_out_cap_handler.earliest_available_datetime())
range_end = min(self.options['end_at'], gen_out_cap_handler.latest_available_datetime())
self._get_report_range(result_ts=generation_ts, report_handler=gen_out_cap_handler,
parser_format=ParserFormat.generation, range_start=range_start,
range_end=range_end)
if self.options.get('forecast', False):
range_start = max(self.options['start_at'], self.local_now)
range_end = min(self.options['end_at'], adequacy_handler.latest_available_datetime())
self._get_report_range(result_ts=generation_ts, report_handler=adequacy_handler,
parser_format=ParserFormat.generation, range_start=range_start,
range_end=range_end)
else:
LOGGER.warn('No valid options were supplied.')
return generation_ts
def parse_latest_fuel_mix(self, content):
# handle bad input
if not content:
return pd.DataFrame()
# preliminary parsing
df = pd.read_csv(BytesIO(content), header=0, index_col=0, skiprows=2, parse_dates=True)
# set index
try:
df.index = self.utcify_index(df.index)
except AttributeError:
LOGGER.error('MISO: Error in source data for generation %s' % content)
return pd.DataFrame()
df.index.set_names(['timestamp'], inplace=True)
# set names and labels
df['fuel_name'] = df.apply(lambda x: self.fuels[x['CATEGORY']], axis=1)
df['gen_MW'] = df['ACT']
# return
return df[['fuel_name', 'gen_MW']]
def format_result(self, data):
"""Output EIA API results in pyiso format"""
try:
assert ('series' in data)
except:
LOGGER.error('Unable to format result for %s' % data['request'])
raise ValueError('Query error for %s:' % data['request'])
market = self._set_market()
data_type = self._set_data_type()
data_formatted = []
if self.options['latest']:
data_formatted = self._format_latest(data, data_type, market)
elif self.options['yesterday']:
data_formatted = self._format_yesterday(data, data_type, market)
else:
data_formatted = self._format_general(data, data_type, market)
if self.options['start_at'] and self.options['end_at']:
data_formatted = self._format_start_end(data_formatted)
if self.options['data'] == 'gen':
data_formatted = self.add_gen_data(data_formatted)
return data_formatted
def get_lmp(self,
node_id='INTERNALHUB',
latest=True,
start_at=False,
end_at=False,
**kwargs):
# set args
self.handle_options(data='lmp',
latest=latest,
start_at=start_at,
end_at=end_at,
node_id=node_id,
**kwargs)
# get location id
try:
locationid = self.locations[node_id.upper()]
except KeyError:
raise ValueError('No LMP data available for location %s' % node_id)
# set up storage
raw_data = []
# collect raw data
for endpoint in self.request_endpoints(locationid):
# carry out request
data = self.fetch_data(endpoint, self.auth)
# pull out data
try:
raw_data += self.parse_json_lmp_data(data)
except ValueError as e:
LOGGER.warn(e)
continue
# parse and slice
df = self._parse_json(raw_data)
df = self.slice_times(df)
# return
return df.to_dict(orient='record')
def time_as_of(self, content):
"""
Returns a UTC timestamp if one is found in the html content,
or None if an error was encountered.
"""
# soup it up
soup = BeautifulSoup(content, 'lxml')
# like 12.11.2015 17:15
ts_elt = soup.find(id='ctl00_ContentPlaceHolder1_DateAndTime')
if not ts_elt:
LOGGER.error('PJM: Timestamp not found in soup:\n%s' % soup)
return None
ts_str = ts_elt.string
# EDT or EST
tz_elt = ts_elt.next_sibling
tz_str = tz_elt.string.strip()
is_dst = tz_str == 'EDT'
# utcify and return
return self.utcify(ts_str, is_dst=is_dst)
def get_load(self,
latest=False,
start_at=False,
end_at=False,
forecast=False,
**kwargs):
# set args
self.handle_options(data='load',
latest=latest,
forecast=forecast,
start_at=start_at,
end_at=end_at,
**kwargs)
# set up storage
raw_data = []
# collect raw data
for endpoint in self.request_endpoints():
# carry out request
data = self.fetch_data(endpoint, self.auth)
# pull out data
try:
raw_data += self.parse_json_load_data(data)
except ValueError as e:
LOGGER.warn(e)
continue
# parse data
try:
df = self._parse_json(raw_data)
except ValueError:
return []
df = self.slice_times(df)
# return
return self.serialize_faster(df, drop_index=True)
def get_load(self, latest=False, yesterday=False, start_at=None, end_at=None, **kwargs):
load_ts = list([])
self.handle_options(latest=latest, yesterday=yesterday, start_at=start_at, end_at=end_at, **kwargs)
rt_const_totals_handler = RealTimeConstrainedTotalsReportHandler(ieso_client=self)
predisp_const_totals_handler = PredispatchConstrainedTotalsReportHandler(ieso_client=self)
if self.options.get('latest', False):
self._get_latest_report_trimmed(result_ts=load_ts, report_handler=rt_const_totals_handler,
parser_format=ParserFormat.load)
elif self.options.get('start_at', None) and self.options.get('end_at', None):
if self.options.get('historical', False):
range_start = max(self.options['start_at'], rt_const_totals_handler.earliest_available_datetime())
range_end = min(self.options['end_at'], rt_const_totals_handler.latest_available_datetime())
self._get_report_range(result_ts=load_ts, report_handler=rt_const_totals_handler,
parser_format=ParserFormat.load, range_start=range_start, range_end=range_end)
if self.options.get('forecast', False):
range_start = max(self.options['start_at'], rt_const_totals_handler.latest_available_datetime(),
predisp_const_totals_handler.earliest_available_datetime())
range_end = min(self.options['end_at'], predisp_const_totals_handler.latest_available_datetime())
self._get_report_range(result_ts=load_ts, report_handler=predisp_const_totals_handler,
parser_format=ParserFormat.load, range_start=range_start, range_end=range_end)
else:
LOGGER.warn('No valid options were supplied.')
return load_ts
def get_lmp(self, node_id='HB_HUBAVG', **kwargs):
self.handle_options(data='lmp', node_id=node_id, **kwargs)
if self.options['market'] == self.MARKET_CHOICES.fivemin:
report_name = 'rt5m_lmp'
elif self.options['market'] == self.MARKET_CHOICES.dam:
report_name = 'dam_hrly_lmp'
elif self.options['market'] == self.MARKET_CHOICES.hourly:
raise NotImplementedError(
'ERCOT does not produce realtime hourly prices?')
self.now = datetime.now(pytz.utc)
if 'start_at' in self.options:
# get start and end days in local time
tz = pytz.timezone(self.TZ_NAME)
start = tz.normalize(self.options['start_at'])
end = tz.normalize(self.options['end_at'])
pieces = []
if self.options['market'] == self.MARKET_CHOICES.fivemin:
# warning, this could take a long time
fivemin_periods = int(
(end - start).total_seconds() / (60 * 5)) + 1
p_list = [
end - timedelta(minutes=5 * x)
for x in range(fivemin_periods)
]
for period in p_list:
try:
report = self._request_report(report_name, date=period)
pieces.append(report)
except ValueError:
pass
else:
start = datetime(start.year,
start.month,
start.day,
tzinfo=start.tzinfo)
days_list = [
end - timedelta(days=x)
for x in range((end - start).days + 1)
]
for day in days_list:
try:
report = self._request_report(report_name, day)
pieces.append(report)
except ValueError:
pass
# combine pieces, if any
if len(pieces) > 0:
report = pd.concat(pieces)
else:
LOGGER.warn('No ERCOT LMP found for %s' % self.options)
return []
else:
report = self._request_report(report_name, self.now)
if report is None:
report = self._request_report(report_name,
self.now - timedelta(days=1))
df = self.format_lmp(report)
# strip uneeded times
df = self.slice_times(df)
# strip out unwanted nodes
if node_id:
if not isinstance(node_id, list):
node_id = [node_id]
reg = re.compile('|'.join(node_id))
df = df.ix[df['node_id'].str.contains(reg)]
return df.to_dict(orient='records')
def no_forecast_warn(self):
if not self.options['latest'] and self.options[
'start_at'] >= pytz.utc.localize(datetime.utcnow()):
LOGGER.warn(
"SVERI does not have forecast data. There will be no data for the chosen time frame."
)
def _generation_historical(self):
# set up storage
parsed_data = []
# collect data
request_date = self.options['start_at'].astimezone(self.ca_tz).date()
local_end_at = self.options['end_at'].astimezone(self.ca_tz).date()
while request_date <= local_end_at:
# set up request
url_file = request_date.strftime('%Y%m%d_DailyRenewablesWatch.txt')
url = self.base_url_gen + url_file
# carry out request
response = self.request(url)
if not response:
request_date += timedelta(days=1)
continue
dst_error_text = 'The supplied DateTime represents an invalid time. For example, when the clock is ' \
'adjusted forward, any time in the period that is skipped is invalid.'
header_idx = 1
for part in [1, 2]: # process both halves of page (i.e. two parts)
num_data_rows = 24
# The day transitioning to daylight saving time adds extra erroneous lines of text.
if part == 1 and dst_error_text in response.text:
num_data_rows = 29
df = self.parse_to_df(response.text,
nrows=num_data_rows,
header=header_idx,
delimiter='\t+')
# The day transitioning to daylight saving time has errors in part two of the file that need removal.
if part == 2:
df = df[df.THERMAL.map(str) != '#VALUE!']
# combine date with hours to index
try:
indexed = self.set_dt_index(df, request_date, df['Hour'])
except Exception as e:
LOGGER.error(e)
continue
# original header is fuel names
indexed.rename(columns=self.fuels, inplace=True)
# remove non-fuel cols
fuel_cols = list(
set(self.fuels.values()) & set(indexed.columns))
subsetted = indexed[fuel_cols]
# pivot
pivoted = self.unpivot(subsetted)
pivoted.rename(columns={
'level_1': 'fuel_name',
0: 'gen_MW'
},
inplace=True)
# slice times
sliced = self.slice_times(pivoted)
# store
parsed_data += self.serialize(
sliced,
header=['timestamp', 'fuel_name', 'gen_MW'],
extras={
'ba_name': self.NAME,
'market': self.MARKET_CHOICES.hourly,
'freq': self.FREQUENCY_CHOICES.hourly
})
# If processing the first part, set the header index for second part.
if part == 1:
header_idx = num_data_rows + 3
# finish day
request_date += timedelta(days=1)
# return
return parsed_data
def request(self,
url,
mode='get',
retry_sec=5,
retries_remaining=5,
**kwargs):
"""
Get or post to a URL with the provided kwargs.
Returns the response, or None if an error was encountered.
If the mode is not 'get' or 'post', raises ValueError.
"""
# check args
allowed_modes = ['get', 'post']
if mode not in allowed_modes:
raise ValueError('Invalid request mode %s' % mode)
# check for session
try:
session = getattr(self, 'session')
except AttributeError:
self.session = requests.Session()
session = self.session
# carry out request
try:
response = getattr(session, mode)(url,
verify=False,
timeout=self.timeout_seconds,
**kwargs)
# except requests.exceptions.ChunkedEncodingError as e:
# # JSON incomplete or not found
# msg = '%s: chunked encoding error for %s, %s:\n%s' % (self.NAME, url, kwargs, e)
# LOGGER.error(msg)
# return None
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
# eg max retries exceeded
msg = '%s: connection error for %s, %s:\n%s' % (self.NAME, url,
kwargs, e)
LOGGER.error(msg)
return None
# except requests.exceptions.RequestException:
# msg = '%s: request exception for %s, %s:\n%s' % (self.NAME, url, kwargs, e)
# LOGGER.error(msg)
# return None
if response.status_code == 200:
# success
LOGGER.debug('%s: request success for %s, %s with cache hit %s' %
(self.NAME, url, kwargs,
getattr(response, 'from_cache', None)))
elif response.status_code == 429:
if retries_remaining > 0:
# retry on throttle
LOGGER.warn(
'%s: retrying in %d seconds (%d retries remaining), throttled for %s, %s'
% (self.NAME, retry_sec, retries_remaining, url, kwargs))
sleep(retry_sec)
retries_remaining -= 1
return self.request(url,
mode=mode,
retry_sec=retry_sec * 2,
retries_remaining=retries_remaining,
**kwargs)
else:
# exhausted retries
LOGGER.warn('%s: exhausted retries for %s, %s' %
(self.NAME, url, kwargs))
return None
else:
# non-throttle error
LOGGER.error('%s: request failure with code %s for %s, %s' %
(self.NAME, response.status_code, url, kwargs))
return response
