def fetch_production( zone_key=DEFAULT_ZONE_KEY, session=None, target_datetime=None, logger=logging.getLogger(__name__)) -> list: """Request the production mix (in MW) of a given zone.""" date_string = arrow.get(target_datetime).to(TIMEZONE).format('DD/MM/YYYY') return [ validation.validate( { 'datetime': arrow.get(breakdown['DT'], tzinfo=TIMEZONE).datetime, 'production': { 'coal': breakdown['Coal'], 'gas': breakdown['Gas'], 'hydro': breakdown['Hydro'], 'oil': breakdown['Oil'], 'solar': breakdown['Solar'], 'unknown': breakdown['CoGen'], }, 'source': DOMAIN, 'zoneKey': zone_key, }, logger, floor=PRODUCTION_THRESHOLD, remove_negative=True) for breakdown in get_api_data( session or requests.Session(), PRODUCTION_URL, { 'Fromdate': date_string, 'Todate': date_string, }) ]
def fetch_production(zone_key=DEFAULT_ZONE_KEY, session=None, target_datetime=None, logger=logging.getLogger(__name__)): """Fetch a list of hourly production data, in MW, for the day of the requested date-time. """ date_time = arrow.get(target_datetime).to(TIMEZONE).floor('hour') results = [{ 'datetime': date_time.replace(hour=hour).datetime, 'production': { 'biomass': row.get('BIOGAS'), 'coal': row.get('TURBINA DE VAPOR'), 'gas': row.get('TURBINA DE GAS'), 'hydro': row.get('HIDROELÉCTRICA'), 'oil': row.get('MOTOR RECIPROCANTE'), 'solar': row.get('FOTOVOLTAICA'), 'wind': row.get('EÓLICO'), 'geothermal': row.get('GEOTÉRMICA'), }, 'source': URL.netloc, 'zoneKey': zone_key, } for hour, row in enumerate( index_api_data_by_hour(request_api_data(session, date_time)))] # If the current day is selected, the API will return zero-filled future # data until the end of the day. Truncate the list to avoid returning any # future data. if not target_datetime: results = results[:date_time.hour + 1] return [ validation.validate(result, logger, floor=PRODUCTION_THRESHOLD, remove_negative=True) for result in results ]
def fetch_production( zone_key=DEFAULT_ZONE_KEY, session=None, target_datetime=None, logger=logging.getLogger(__name__)) -> dict: """Request the last known production mix (in MW) of a given country.""" if target_datetime: raise NotImplementedError( 'This parser is not yet able to parse past dates') session = session or requests.Session() response = session.get(f'{URL_STRING}/CSDReportServlet', params={'contentType': 'csv'}) generation = { row[0]: { 'MC': float(row[1]), # maximum capability 'TNG': float(row[2]), # total net generation } for row in csv.reader(response.text.split('\r\n\r\n')[3].splitlines()) } return validation.validate( { 'capacity': { 'gas': generation['GAS']['MC'], 'hydro': generation['HYDRO']['MC'], 'battery storage': generation['ENERGY STORAGE']['MC'], 'solar': generation['SOLAR']['MC'], 'wind': generation['WIND']['MC'], 'biomass': generation['OTHER']['MC'], 'unknown': generation['DUAL FUEL']['MC'], 'coal': generation['COAL']['MC'], }, 'datetime': get_csd_report_timestamp(response.text), 'production': { 'gas': generation['GAS']['TNG'], 'hydro': generation['HYDRO']['TNG'], 'solar': generation['SOLAR']['TNG'], 'wind': generation['WIND']['TNG'], 'biomass': generation['OTHER']['TNG'], 'unknown': generation['DUAL FUEL']['TNG'], 'coal': generation['COAL']['TNG'], }, 'source': URL.netloc, 'storage': { 'battery': generation['ENERGY STORAGE']['TNG'], }, 'zoneKey': zone_key, }, logger, floor=MINIMUM_PRODUCTION_THRESHOLD, remove_negative=True)
def fetch_production( zone_key="NG", session=None, target_datetime=None, logger=logging.getLogger(__name__), ) -> dict: """Requests the last known production mix (in MW) of a given zone.""" timestamp = (arrow.get(target_datetime).to("Africa/Lagos").replace( minute=0, second=0, microsecond=0)) # GET the landing page (HTML) and scrape some form data from it. session = session or requests.Session() response = session.get(API_URL_STRING) soup = bs4.BeautifulSoup(response.text, "html.parser") data = {tag["name"]: tag["value"] for tag in soup.find_all("input")} data["ctl00$MainContent$txtReadingDate"] = timestamp.format("DD/MM/YYYY") data["ctl00$MainContent$ddlTime"] = timestamp.format("HH:mm") # Send a POST request for the desired grid data using parameters from the # landing page form. The grid data is presented as an HTML table; we ignore # its header and footer rows. response = session.post(API_URL_STRING, data=data) rows = bs4.BeautifulSoup(response.text, "html.parser").find_all("tr")[1:-1] production_mix = {technology: 0.0 for technology in NORMALISE.values()} for row in rows: _, source, power, _ = (tag.text for tag in row.find_all("td")) try: technology = NORMALISE[PATTERN.search(source).group(1).casefold()] except (AttributeError, KeyError) as error: logger.warning(f"Unexpected source '{source.strip()}' encountered") continue production_mix[technology] += float(power) # Return the production mix. return validation.validate( { "zoneKey": zone_key, "datetime": timestamp.datetime, "production": production_mix, "source": API_URL.netloc, }, logger, floor=10.0, remove_negative=True, )
def fetch_production( zone_key=DEFAULT_ZONE_KEY, session=None, target_datetime=None, logger=logging.getLogger(__name__), ) -> list: """Request the production mix (in MW) of a given zone.""" date_string = arrow.get(target_datetime).to(TIMEZONE).format("DD/MM/YYYY") return [ validation.validate( { "datetime": arrow.get(breakdown["DT"], tzinfo=TIMEZONE).datetime, "production": { "coal": breakdown["Coal"], "gas": breakdown["Gas"], "hydro": breakdown["Hydro"], "oil": breakdown["Oil"], "solar": breakdown["Solar"], "unknown": breakdown["CoGen"], }, "source": DOMAIN, "zoneKey": zone_key, }, logger, floor=PRODUCTION_THRESHOLD, remove_negative=True, ) for breakdown in get_api_data( session or requests.Session(), PRODUCTION_URL, { "Fromdate": date_string, "Todate": date_string, }, ) ]
def fetch_production( zone_key='GE', session=None, target_datetime=None, logger=logging.getLogger(__name__)) -> dict: """Request the last known production mix (in MW) of a given country.""" session = session or requests.session() if target_datetime is None: # Get the current production mix. # TODO: remove `verify=False` ASAP. production_mix = session.get(f'{URL_STRING}/map', verify=False) \ .json()['typeSum'] return validation.validate( { 'datetime': arrow.now(TIMEZONE).floor('minute').datetime, 'production': { 'gas': production_mix['thermalData'], 'hydro': production_mix['hydroData'], 'solar': production_mix['solarData'], 'wind': production_mix['windPowerData'], }, 'source': URL.netloc, 'zoneKey': 'GE', }, logger, remove_negative=True, floor=MINIMUM_PRODUCTION_THRESHOLD) else: # Get the production mix for every hour on the day of interest. timestamp_from, timestamp_to = arrow.get(target_datetime, TIMEZONE) \ .replace(hour=0) \ .floor('hour') \ .span('day') response = session.get( f'{URL_STRING}/diagramDownload', params={ 'fromDate': timestamp_from.format('YYYY-MM-DDTHH:mm:ss'), 'lang': 'EN', 'toDate': timestamp_to.format('YYYY-MM-DDTHH:mm:ss'), 'type': 'FACT', }, verify=False) # TODO: remove `verify=False` ASAP. table = pandas.read_excel(response.content, header=2, index_col=1) \ .iloc[2:6, 2:] \ .dropna(axis='columns', how='all') table.index = 'gas', 'hydro', 'wind', 'solar' table.columns = pandas.date_range(start=timestamp_from.datetime, freq='1H', periods=table.shape[1]) # Collect the data into a list of dictionaries, then validate and # return it. production_mixes = ({ 'datetime': arrow.get(timestamp, TIMEZONE).datetime, 'production': { 'gas': production_mix['gas'], 'hydro': production_mix['hydro'], 'wind': production_mix['wind'], 'solar': production_mix['solar'], }, 'source': URL.netloc, 'zoneKey': zone_key, } for timestamp, production_mix in table.items()) return [ validation.validate(production_mix, logger, remove_negative=True, floor=MINIMUM_PRODUCTION_THRESHOLD) for production_mix in production_mixes ]
def fetch_production( zone_key="GE", session=None, target_datetime=None, logger=logging.getLogger(__name__), ) -> dict: """Request the last known production mix (in MW) of a given country.""" session = session or requests.session() if target_datetime is None: # Get the current production mix. # TODO: remove `verify=False` ASAP. production_mix = session.get(f"{URL_STRING}/map", verify=False).json()["typeSum"] return validation.validate( { "datetime": arrow.now(TIMEZONE).floor("minute").datetime, "production": { "gas": production_mix["thermalData"], "hydro": production_mix["hydroData"], "solar": production_mix["solarData"], "wind": production_mix["windPowerData"], }, "source": URL.netloc, "zoneKey": "GE", }, logger, remove_negative=True, floor=MINIMUM_PRODUCTION_THRESHOLD, ) else: # Get the production mix for every hour on the day of interest. timestamp_from, timestamp_to = (arrow.get( target_datetime, TIMEZONE).replace(hour=0).floor("hour").span("day")) response = session.get( f"{URL_STRING}/diagramDownload", params={ "fromDate": timestamp_from.format("YYYY-MM-DDTHH:mm:ss"), "lang": "EN", "toDate": timestamp_to.format("YYYY-MM-DDTHH:mm:ss"), "type": "FACT", }, verify=False, ) # TODO: remove `verify=False` ASAP. table = (pandas.read_excel(response.content, header=2, index_col=1).iloc[2:6, 2:].dropna(axis="columns", how="all")) table.index = "gas", "hydro", "wind", "solar" table.columns = pandas.date_range(start=timestamp_from.datetime, freq="1H", periods=table.shape[1]) # Collect the data into a list of dictionaries, then validate and # return it. production_mixes = ({ "datetime": arrow.get(timestamp, TIMEZONE).datetime, "production": { "gas": production_mix["gas"], "hydro": production_mix["hydro"], "wind": production_mix["wind"], "solar": production_mix["solar"], }, "source": URL.netloc, "zoneKey": zone_key, } for timestamp, production_mix in table.items()) return [ validation.validate( production_mix, logger, remove_negative=True, floor=MINIMUM_PRODUCTION_THRESHOLD, ) for production_mix in production_mixes ]