def _fetch_series(zone_key,
series_id,
session=None,
target_datetime=None,
logger=None):
"""Fetches and converts a data series."""
key = os.environ['EIA_KEY']
assert key and key != 'eia_key', key
s = session or requests.Session()
series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = series.last(24)
# UTC timestamp with no offset returned.
if not raw_data.get('series'):
# Series doesn't exist. Probably requesting a fuel from a region that
# doesn't have any capacity for that fuel type.
return []
return [{
'zoneKey': zone_key,
'datetime': parser.parse(datapoint[0]),
'value': datapoint[1],
'source': 'eia.gov',
} for datapoint in raw_data['series'][0]['data']]
def _fetch_series(zone_key, series_id, session=None, target_datetime=None,
logger=None):
"""Fetches and converts a data series."""
s = session or requests.Session()
# local import to avoid the exception that happens if EIAPY token is not set
# even if this module is unused
from eiapy import Series
series = Series(series_id=series_id, session=s)
if target_datetime:
utc = tz.gettz('UTC')
#eia currently only accepts utc timestamps in the form YYYYMMDDTHHZ
end = target_datetime.astimezone(utc).strftime('%Y%m%dT%HZ')
start = (target_datetime.astimezone(utc) - datetime.timedelta(days=1)).strftime('%Y%m%dT%HZ')
raw_data = series.get_data(start=start, end=end)
else:
# Get the last 24 hours available.
raw_data = series.last(24)
# UTC timestamp with no offset returned.
if not raw_data.get('series'):
# Series doesn't exist. Probably requesting a fuel from a region that
# doesn't have any capacity for that fuel type.
return []
return [{
'zoneKey': zone_key,
'datetime': parser.parse(datapoint[0]),
'value': datapoint[1],
'source': 'eia.gov',
} for datapoint in raw_data['series'][0]['data']]
def fetchEIA(series_ID, update=True):
try:
sr = Series(series_ID)
if update:
data = sr.last(20)['series'][0]['data']
else:
data = sr.get_data(all_data=True)['series'][0]['data']
df = pd.DataFrame.from_records(data, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'])
df.dropna(inplace=True)
return df
except:
raise fetchEIA_Error('Fetching EIA failed')
def fetch_exchange(zone_key1,
zone_key2,
session=None,
target_datetime=None,
logger=None):
"""Requests the last known power exchange (in MW) between two zones
Arguments:
zone_key1 -- the first country code
zone_key2 -- the second country code; order of the two codes in params doesn't matter
session (optional) -- request session passed in order to re-use an existing session
target_datetime (optional) -- string in form YYYYMMDDTHHZ
Return:
A list of dictionaries in the form:
{
'sortedZoneKeys': 'DK->NO',
'datetime': '2017-01-01T00:00:00Z',
'netFlow': 0.0,
'source': 'mysource.com'
}
where net flow is from DK into NO
"""
sortedcodes = '->'.join(sorted([zone_key1, zone_key2]))
series_id = EXCHANGES[sortedcodes]
s = session or requests.Session()
exchange_series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = exchange_series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = exchange_series.last(24)['series'][0]['data']
data = []
for datapoint in raw_data:
if sortedcodes == 'MX->US-CA':
datapoint[1] = -1 * datapoint[1]
exchange = {
'sortedZoneKeys': sortedcodes,
'datetime': parser.parse(datapoint[0]),
'netFlow': datapoint[1],
'source': 'mysource.com'
}
data.append(exchange)
return data
def _fetch_series(zone_key,
series_id,
session=None,
target_datetime=None,
logger=None):
"""Fetches and converts a data series."""
s = session or requests.Session()
# local import to avoid the exception that happens if EIAPY token is not set
# even if this module is unused
from eiapy import Series
series = Series(series_id=series_id, session=s)
if target_datetime:
utc = tz.gettz("UTC")
# eia currently only accepts utc timestamps in the form YYYYMMDDTHHZ
end = target_datetime.astimezone(utc).strftime("%Y%m%dT%HZ")
start = (target_datetime.astimezone(utc) -
datetime.timedelta(days=1)).strftime("%Y%m%dT%HZ")
raw_data = series.get_data(start=start, end=end)
else:
# Get the last 24 hours available.
raw_data = series.last(24)
eia_error_message = raw_data.get("data", {}).get("error")
if eia_error_message:
logger.error(
f"EIA error, for series_id [{series_id}]: {eia_error_message}")
return []
# UTC timestamp with no offset returned.
if not raw_data.get("series"):
# Series doesn't exist. Probably requesting a fuel from a region that
# doesn't have any capacity for that fuel type.
return []
return [{
"zoneKey": zone_key,
"datetime": parser.parse(datapoint[0]),
"value": datapoint[1],
"source": "eia.gov",
} for datapoint in raw_data["series"][0]["data"]]
def fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):
"""Requests the last known power exchange (in MW) between two zones
Arguments:
zone_key1 -- the first country code
zone_key2 -- the second country code; order of the two codes in params doesn't matter
session (optional) -- request session passed in order to re-use an existing session
target_datetime (optional) -- string in form YYYYMMDDTHHZ
Return:
A list of dictionaries in the form:
{
'sortedZoneKeys': 'DK->NO',
'datetime': '2017-01-01T00:00:00Z',
'netFlow': 0.0,
'source': 'mysource.com'
}
where net flow is from DK into NO
"""
sortedcodes = '->'.join(sorted([zone_key1, zone_key2]))
series_id = EXCHANGES[sortedcodes]
s = session or requests.Session()
exchange_series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = exchange_series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = exchange_series.last(24)['series'][0]['data']
data = []
for datapoint in raw_data:
if sortedcodes == 'MX-BC->US-CA':
datapoint[1] = -1*datapoint[1]
exchange = {'sortedZoneKeys': sortedcodes,
'datetime': parser.parse(datapoint[0]),
'netFlow': datapoint[1],
'source': 'mysource.com'}
data.append(exchange)
return data
def _fetch_production_or_consumption(zone_key, series_id, session=None,
target_datetime=None, logger=None):
"""Fetches production or consumption forecast, determined by series_id."""
s = session or requests.Session()
series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = series.last(24)
# UTC timestamp with no offset returned.
return [{
'zoneKey': zone_key,
'datetime': parser.parse(datapoint[0]),
'value': datapoint[1],
'source': 'eia.org',
} for datapoint in raw_data['series'][0]['data']]
def fetch_consumption_forecast(zone_key, session=None, target_datetime=None,
logger=None):
series_id = DAY_AHEAD[zone_key]
s = session or requests.Session()
forecast_series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = forecast_series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = forecast_series.last(24)['series'][0]['data']
# UTC timestamp with no offset returned.
return [{
'zoneKey': zone_key,
'datetime': parser.parse(datapoint[0]),
'value': datapoint[1],
'source': 'eia.org',
} for datapoint in raw_data]
def fetch_consumption_forecast(zone_key, session=None, target_datetime=None,
logger=None):
series_id = DAY_AHEAD[zone_key]
s = session or requests.Session()
forecast_series = Series(series_id=series_id, session=s)
if target_datetime:
raw_data = forecast_series.last_from(24, end=target_datetime)
else:
# Get the last 24 hours available.
raw_data = forecast_series.last(24)['series'][0]['data']
# UTC timestamp with no offset returned.
return [{
'zoneKey': zone_key,
'datetime': parser.parse(datapoint[0]),
'value': datapoint[1],
'source': 'eia.org',
} for datapoint in raw_data]