def fetch_production(
zone_key=DEFAULT_ZONE_KEY,
session=None,
target_datetime=None,
logger=logging.getLogger(__name__)) -> list:
"""Request the production mix (in MW) of a given zone."""
date_string = arrow.get(target_datetime).to(TIMEZONE).format('DD/MM/YYYY')
return [
validation.validate(
{
'datetime': arrow.get(breakdown['DT'],
tzinfo=TIMEZONE).datetime,
'production': {
'coal': breakdown['Coal'],
'gas': breakdown['Gas'],
'hydro': breakdown['Hydro'],
'oil': breakdown['Oil'],
'solar': breakdown['Solar'],
'unknown': breakdown['CoGen'],
},
'source': DOMAIN,
'zoneKey': zone_key,
},
logger,
floor=PRODUCTION_THRESHOLD,
remove_negative=True) for breakdown in get_api_data(
session or requests.Session(), PRODUCTION_URL, {
'Fromdate': date_string,
'Todate': date_string,
})
]
def fetch_production(zone_key=DEFAULT_ZONE_KEY,
session=None,
target_datetime=None,
logger=logging.getLogger(__name__)):
"""Fetch a list of hourly production data, in MW, for the day of the
requested date-time.
"""
date_time = arrow.get(target_datetime).to(TIMEZONE).floor('hour')
results = [{
'datetime': date_time.replace(hour=hour).datetime,
'production': {
'biomass': row.get('BIOGAS'),
'coal': row.get('TURBINA DE VAPOR'),
'gas': row.get('TURBINA DE GAS'),
'hydro': row.get('HIDROELÉCTRICA'),
'oil': row.get('MOTOR RECIPROCANTE'),
'solar': row.get('FOTOVOLTAICA'),
'wind': row.get('EÓLICO'),
'geothermal': row.get('GEOTÉRMICA'),
},
'source': URL.netloc,
'zoneKey': zone_key,
} for hour, row in enumerate(
index_api_data_by_hour(request_api_data(session, date_time)))]
# If the current day is selected, the API will return zero-filled future
# data until the end of the day. Truncate the list to avoid returning any
# future data.
if not target_datetime:
results = results[:date_time.hour + 1]
return [
validation.validate(result,
logger,
floor=PRODUCTION_THRESHOLD,
remove_negative=True) for result in results
]
def fetch_production(
zone_key=DEFAULT_ZONE_KEY,
session=None,
target_datetime=None,
logger=logging.getLogger(__name__)) -> dict:
"""Request the last known production mix (in MW) of a given country."""
if target_datetime:
raise NotImplementedError(
'This parser is not yet able to parse past dates')
session = session or requests.Session()
response = session.get(f'{URL_STRING}/CSDReportServlet',
params={'contentType': 'csv'})
generation = {
row[0]: {
'MC': float(row[1]), # maximum capability
'TNG': float(row[2]), # total net generation
}
for row in csv.reader(response.text.split('\r\n\r\n')[3].splitlines())
}
return validation.validate(
{
'capacity': {
'gas': generation['GAS']['MC'],
'hydro': generation['HYDRO']['MC'],
'battery storage': generation['ENERGY STORAGE']['MC'],
'solar': generation['SOLAR']['MC'],
'wind': generation['WIND']['MC'],
'biomass': generation['OTHER']['MC'],
'unknown': generation['DUAL FUEL']['MC'],
'coal': generation['COAL']['MC'],
},
'datetime': get_csd_report_timestamp(response.text),
'production': {
'gas': generation['GAS']['TNG'],
'hydro': generation['HYDRO']['TNG'],
'solar': generation['SOLAR']['TNG'],
'wind': generation['WIND']['TNG'],
'biomass': generation['OTHER']['TNG'],
'unknown': generation['DUAL FUEL']['TNG'],
'coal': generation['COAL']['TNG'],
},
'source': URL.netloc,
'storage': {
'battery': generation['ENERGY STORAGE']['TNG'],
},
'zoneKey': zone_key,
},
logger,
floor=MINIMUM_PRODUCTION_THRESHOLD,
remove_negative=True)
def fetch_production(
zone_key="NG",
session=None,
target_datetime=None,
logger=logging.getLogger(__name__),
) -> dict:
"""Requests the last known production mix (in MW) of a given zone."""
timestamp = (arrow.get(target_datetime).to("Africa/Lagos").replace(
minute=0, second=0, microsecond=0))
# GET the landing page (HTML) and scrape some form data from it.
session = session or requests.Session()
response = session.get(API_URL_STRING)
soup = bs4.BeautifulSoup(response.text, "html.parser")
data = {tag["name"]: tag["value"] for tag in soup.find_all("input")}
data["ctl00$MainContent$txtReadingDate"] = timestamp.format("DD/MM/YYYY")
data["ctl00$MainContent$ddlTime"] = timestamp.format("HH:mm")
# Send a POST request for the desired grid data using parameters from the
# landing page form. The grid data is presented as an HTML table; we ignore
# its header and footer rows.
response = session.post(API_URL_STRING, data=data)
rows = bs4.BeautifulSoup(response.text, "html.parser").find_all("tr")[1:-1]
production_mix = {technology: 0.0 for technology in NORMALISE.values()}
for row in rows:
_, source, power, _ = (tag.text for tag in row.find_all("td"))
try:
technology = NORMALISE[PATTERN.search(source).group(1).casefold()]
except (AttributeError, KeyError) as error:
logger.warning(f"Unexpected source '{source.strip()}' encountered")
continue
production_mix[technology] += float(power)
# Return the production mix.
return validation.validate(
{
"zoneKey": zone_key,
"datetime": timestamp.datetime,
"production": production_mix,
"source": API_URL.netloc,
},
logger,
floor=10.0,
remove_negative=True,
)
def fetch_production(
zone_key=DEFAULT_ZONE_KEY,
session=None,
target_datetime=None,
logger=logging.getLogger(__name__),
) -> list:
"""Request the production mix (in MW) of a given zone."""
date_string = arrow.get(target_datetime).to(TIMEZONE).format("DD/MM/YYYY")
return [
validation.validate(
{
"datetime": arrow.get(breakdown["DT"],
tzinfo=TIMEZONE).datetime,
"production": {
"coal": breakdown["Coal"],
"gas": breakdown["Gas"],
"hydro": breakdown["Hydro"],
"oil": breakdown["Oil"],
"solar": breakdown["Solar"],
"unknown": breakdown["CoGen"],
},
"source": DOMAIN,
"zoneKey": zone_key,
},
logger,
floor=PRODUCTION_THRESHOLD,
remove_negative=True,
) for breakdown in get_api_data(
session or requests.Session(),
PRODUCTION_URL,
{
"Fromdate": date_string,
"Todate": date_string,
},
)
]
def fetch_production(
zone_key='GE',
session=None,
target_datetime=None,
logger=logging.getLogger(__name__)) -> dict:
"""Request the last known production mix (in MW) of a given country."""
session = session or requests.session()
if target_datetime is None: # Get the current production mix.
# TODO: remove `verify=False` ASAP.
production_mix = session.get(f'{URL_STRING}/map', verify=False) \
.json()['typeSum']
return validation.validate(
{
'datetime': arrow.now(TIMEZONE).floor('minute').datetime,
'production': {
'gas': production_mix['thermalData'],
'hydro': production_mix['hydroData'],
'solar': production_mix['solarData'],
'wind': production_mix['windPowerData'],
},
'source': URL.netloc,
'zoneKey': 'GE',
},
logger,
remove_negative=True,
floor=MINIMUM_PRODUCTION_THRESHOLD)
else:
# Get the production mix for every hour on the day of interest.
timestamp_from, timestamp_to = arrow.get(target_datetime, TIMEZONE) \
.replace(hour=0) \
.floor('hour') \
.span('day')
response = session.get(
f'{URL_STRING}/diagramDownload',
params={
'fromDate': timestamp_from.format('YYYY-MM-DDTHH:mm:ss'),
'lang': 'EN',
'toDate': timestamp_to.format('YYYY-MM-DDTHH:mm:ss'),
'type': 'FACT',
},
verify=False) # TODO: remove `verify=False` ASAP.
table = pandas.read_excel(response.content, header=2, index_col=1) \
.iloc[2:6, 2:] \
.dropna(axis='columns', how='all')
table.index = 'gas', 'hydro', 'wind', 'solar'
table.columns = pandas.date_range(start=timestamp_from.datetime,
freq='1H',
periods=table.shape[1])
# Collect the data into a list of dictionaries, then validate and
# return it.
production_mixes = ({
'datetime': arrow.get(timestamp, TIMEZONE).datetime,
'production': {
'gas': production_mix['gas'],
'hydro': production_mix['hydro'],
'wind': production_mix['wind'],
'solar': production_mix['solar'],
},
'source': URL.netloc,
'zoneKey': zone_key,
} for timestamp, production_mix in table.items())
return [
validation.validate(production_mix,
logger,
remove_negative=True,
floor=MINIMUM_PRODUCTION_THRESHOLD)
for production_mix in production_mixes
]
def fetch_production(
zone_key="GE",
session=None,
target_datetime=None,
logger=logging.getLogger(__name__),
) -> dict:
"""Request the last known production mix (in MW) of a given country."""
session = session or requests.session()
if target_datetime is None: # Get the current production mix.
# TODO: remove `verify=False` ASAP.
production_mix = session.get(f"{URL_STRING}/map",
verify=False).json()["typeSum"]
return validation.validate(
{
"datetime": arrow.now(TIMEZONE).floor("minute").datetime,
"production": {
"gas": production_mix["thermalData"],
"hydro": production_mix["hydroData"],
"solar": production_mix["solarData"],
"wind": production_mix["windPowerData"],
},
"source": URL.netloc,
"zoneKey": "GE",
},
logger,
remove_negative=True,
floor=MINIMUM_PRODUCTION_THRESHOLD,
)
else:
# Get the production mix for every hour on the day of interest.
timestamp_from, timestamp_to = (arrow.get(
target_datetime,
TIMEZONE).replace(hour=0).floor("hour").span("day"))
response = session.get(
f"{URL_STRING}/diagramDownload",
params={
"fromDate": timestamp_from.format("YYYY-MM-DDTHH:mm:ss"),
"lang": "EN",
"toDate": timestamp_to.format("YYYY-MM-DDTHH:mm:ss"),
"type": "FACT",
},
verify=False,
) # TODO: remove `verify=False` ASAP.
table = (pandas.read_excel(response.content, header=2,
index_col=1).iloc[2:6,
2:].dropna(axis="columns",
how="all"))
table.index = "gas", "hydro", "wind", "solar"
table.columns = pandas.date_range(start=timestamp_from.datetime,
freq="1H",
periods=table.shape[1])
# Collect the data into a list of dictionaries, then validate and
# return it.
production_mixes = ({
"datetime": arrow.get(timestamp, TIMEZONE).datetime,
"production": {
"gas": production_mix["gas"],
"hydro": production_mix["hydro"],
"wind": production_mix["wind"],
"solar": production_mix["solar"],
},
"source": URL.netloc,
"zoneKey": zone_key,
} for timestamp, production_mix in table.items())
return [
validation.validate(
production_mix,
logger,
remove_negative=True,
floor=MINIMUM_PRODUCTION_THRESHOLD,
) for production_mix in production_mixes
]