def read_text_by_regex(regex, text): date_match = search(regex, text) if not date_match: raise ParserException('IN-PB', 'Not date_match') date_text = date_match.group(0) if not date_text: raise ParserException('IN-PB', 'Not date_text') return date_text
def fetch_exchange(country_code1='ES', country_code2='ES-IB', session=None): ses = session or Session() responses = BalearicIslands(ses).get_all() if not responses: raise ParserException("ES-IB", "No response") else: data = [] for response in responses: sorted_country_codes = sorted([country_code1, country_code2]) net_flow = response.link['pe_ma'] response_data = { 'sortedCountryCodes': '->'.join(sorted_country_codes), 'datetime': get(response.timestamp).datetime, 'netFlow': net_flow if country_code1 == sorted_country_codes[0] else -1 * net_flow, 'source': 'demanda.ree.es', } data.append(response_data) return data
def get_response(country_code, url, session=None): """Get response""" ses = session or Session() response = ses.get(url) if response.status_code != 200: raise ParserException( country_code, 'Response code: {0}'.format(response.status_code)) return response
def assert_country_code(country_code, expected, parser_name=None): """Assert country code""" if not country_code or country_code != expected: if not parser_name: parser_name = country_code raise ParserException( parser_name, 'Country_code expected {0}, is {1}'.format(expected, country_code), country_code)
def fetch_consumption(country_code='ES-CN', session=None): ses = session or Session() elhierro = ElHierro(ses).get() if not elhierro: raise ParserException("ES-CN", "ElHierro not response") granacanaria = GranCanaria(ses).get() if not granacanaria: raise ParserException("ES-CN", "GranCanaria not response") gomera = Gomera(ses).get() if not gomera: raise ParserException("ES-CN", "Gomera not response") lanzarotefuerteventura = LanzaroteFuerteventura(ses).get() if not lanzarotefuerteventura: raise ParserException("ES-CN", "LanzaroteFuerteventura not response") palma = LaPalma(ses).get() if not palma: raise ParserException("ES-CN", "LaPalma not response") tenerife = Tenerife(ses).get() if not tenerife: raise ParserException("ES-CN", "Tenerife not response") ## Compare timestamps ## Raise ParserException if timestamps aren't equals if elhierro.timestamp != granacanaria.timestamp \ and elhierro.timestamp != gomera.timestamp \ and elhierro.timestamp != lanzarotefuerteventura.timestamp \ and elhierro.timestamp != palma.timestamp \ and elhierro.timestamp != tenerife.timestamp: raise ParserException("ES-CN", "Response timestamps aren't equals") demand = round( elhierro.demand + granacanaria.demand + gomera.demand + lanzarotefuerteventura.demand + palma.demand + tenerife.demand, 2) data = { 'countryCode': country_code, 'datetime': get(elhierro.timestamp).datetime, 'consumption': demand, 'source': 'demanda.ree.es' } return data
def fetch_consumption(country_code='ES-IB', session=None): ses = session or Session() responses = BalearicIslands(ses).get_all() if not responses: raise ParserException("ES-IB", "No response") else: data = [] for response in responses: response_data = { 'countryCode': country_code, 'datetime': get(response.timestamp).datetime, 'consumption': response.demand, 'source': 'demanda.ree.es' } data.append(response_data) return data
def fetch_production(country_code='ES-IB', session=None): ses = session or Session() responses = BalearicIslands(ses).get_all() if not responses: raise ParserException("ES-IB", "No response") else: data = [] for response in responses: response_data = { 'countryCode': country_code, 'datetime': get(response.timestamp).datetime, 'production': { 'coal': response.carbon, 'gas': round(response.gas + response.combined, 2), 'solar': response.solar, 'oil': round(response.vapor + response.diesel, 2), 'wind': response.wind, 'hydro': response.hydraulic, 'biomass': 0.0, 'nuclear': 0.0, 'geothermal': 0.0, 'unknown': response.unknown() }, 'storage': { 'hydro': 0.0, 'battery': 0.0 }, 'source': 'demanda.ree.es', } data.append(response_data) return data
def fetch_island_data(country_code, session): if country_code == 'ES-CN-FVLZ': lanzarote_fuerteventura_data = LanzaroteFuerteventura(session).get_all() if not lanzarote_fuerteventura_data: raise ParserException(country_code, "LanzaroteFuerteventura not response") else: return lanzarote_fuerteventura_data elif country_code == 'ES-CN-GC': gran_canaria_data = GranCanaria(session).get_all() if not gran_canaria_data: raise ParserException(country_code, "GranCanaria not response") else: return gran_canaria_data elif country_code == 'ES-CN-IG': gomera_data = Gomera(session).get_all() if not gomera_data: raise ParserException(country_code, "Gomera not response") else: return gomera_data elif country_code == 'ES-CN-LP': la_palma_data = LaPalma(session).get_all() if not la_palma_data: raise ParserException(country_code, "LaPalma not response") else: return la_palma_data elif country_code == 'ES-CN-TE': tenerife_data = Tenerife(session).get_all() if not tenerife_data: raise ParserException(country_code, "Tenerife not response") else: return tenerife_data elif country_code == 'ES-CN-HI': el_hierro_data = ElHierro(session).get_all() if not el_hierro_data: raise ParserException(country_code, "ElHierro not response") else: return el_hierro_data else: raise ParserException(country_code, 'Can\'t read this country code {0}'.format(country_code))
def fetch_islands_data(country_code, session): data = {} el_hierro_data = ElHierro(session).get_all() if not el_hierro_data: raise ParserException(country_code, "ElHierro not response") else: data.update({'el_hierro': el_hierro_data}) gran_canaria_data = GranCanaria(session).get_all() if not gran_canaria_data: raise ParserException(country_code, "GranCanaria not response") else: data.update({'gran_canaria': gran_canaria_data}) gomera_data = Gomera(session).get_all() if not gomera_data: raise ParserException(country_code, "Gomera not response") else: data.update({'gomera': gomera_data}) lanzarote_fuerteventura_data = LanzaroteFuerteventura(session).get_all() if not lanzarote_fuerteventura_data: raise ParserException(country_code, "LanzaroteFuerteventura not response") else: data.update({'lanzarote_fuerteventura': lanzarote_fuerteventura_data}) la_palma_data = LaPalma(session).get_all() if not la_palma_data: raise ParserException(country_code, "LaPalma not response") else: data.update({'la_palma': la_palma_data}) tenerife_data = Tenerife(session).get_all() if not tenerife_data: raise ParserException(country_code, "Tenerife not response") else: data.update({'tenerife': tenerife_data}) return data
def fetch_production(zone_key='LK', session=None, target_datetime=None, logger: logging.Logger = logging.getLogger(__name__)): """Requests the previous day's production mix (in MW) for Sri Lanka, per quarter-hour """ if target_datetime is not None and target_datetime < arrow.utcnow().shift(days=-2): raise NotImplementedError('The datasource currently only has data for yesterday') # so 0-24 hours ago just after local midnight to approx. 24-48 hours ago just before midnight r = session or requests.session() response = r.get(GENERATION_BREAKDOWN_URL) assert response.status_code == 200, 'Exception when fetching production for ' \ '{}: error when calling url={}'.format( zone_key, GENERATION_BREAKDOWN_URL) source_data = json.loads(response.json()) # Response is double encoded; a JSON array encoded as a JSON string logger.debug(f"Raw generation breakdown: {source_data}", extra={"key": zone_key}) output = [] for quarter_hourly_source_data in source_data: output_for_timestamp = { 'zoneKey': zone_key, 'datetime': arrow.get(quarter_hourly_source_data['DateTime'], 'YYYY-MM-DD HH:mm:ss', tzinfo=TIMEZONE_NAME).datetime, 'production': { 'biomass': 0.0, 'coal': 0.0, 'gas': 0.0, 'hydro': 0.0, 'nuclear': 0.0, 'oil': 0.0, 'solar': 0.0, 'wind': 0.0, 'geothermal': 0.0, 'unknown': 0.0 }, 'source': SOURCE_NAME } for generation_type, outputInMW in quarter_hourly_source_data.items(): if(generation_type == "DateTime"): continue if generation_type == "Coal": output_for_timestamp['production']['coal'] += outputInMW elif generation_type == "Major Hydro" or generation_type == 'SPP Minihydro': output_for_timestamp['production']['hydro'] += outputInMW elif generation_type == "SPP Biomass": output_for_timestamp['production']['biomass'] += outputInMW elif generation_type == "Solar": output_for_timestamp['production']['solar'] += outputInMW elif generation_type == "Thermal-Oil": output_for_timestamp['production']['oil'] += outputInMW elif generation_type == "Wind": output_for_timestamp['production']['wind'] += outputInMW else: raise ParserException(zone_key, "Unknown production type: " + generation_type) output.append(output_for_timestamp) return output
def fetch_exchange(country_code1='ES', country_code2='MA', session=None, token=None): # Get ESIOS token token = environ.get('ESIOS_TOKEN', token) if not token: raise ParserException("ESIOS", "Require access token") ses = session or requests.Session() # Request headers headers = { 'Content-Type': 'application/json', 'Accept': 'application/json; application/vnd.esios-api-v2+json', 'Authorization': 'Token token="{0}"'.format(token) } # Request query url utc = arrow.utcnow() start_date = utc.shift(hours=-24).floor('hour').isoformat() end_date = utc.ceil('hour').isoformat() dates = {'start_date': start_date, 'end_date': end_date} query = urlencode(dates) url = 'https://api.esios.ree.es/indicators/10209?{0}'.format(query) response = ses.get(url, headers=headers) if response.status_code != 200 or not response.text: raise ParserException( 'ESIOS', 'Response code: {0}'.format(response.status_code)) json = response.json() values = json['indicator']['values'] if not values: raise ParserException('ESIOS', 'No values received') else: data = [] sorted_country_codes = sorted([country_code1, country_code2]) for value in values: # Get last value in datasource datetime = arrow.get(value['datetime_utc']).datetime # Datasource negative value is exporting, positive value is importing net_flow = -value['value'] value_data = { 'sortedCountryCodes': '->'.join(sorted_country_codes), 'datetime': datetime, 'netFlow': net_flow if country_code1 == sorted_country_codes[0] else -1 * net_flow, 'source': 'api.esios.ree.es', } data.append(value_data) return data
def test_instance_with_country_code(self): exception = ParserException('ESIOS', "Parser exception", "ES") self.assertIsInstance(exception, ParserException) self.assertEqual(str(exception), 'ESIOS Parser (ES): Parser exception')
def fetch_production(country_code='IN-KA', session=None): """Fetch Karnataka production""" countrycode.assert_country_code(country_code, 'IN-KA') html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateGen.aspx', session) india_date_time = IN.read_datetime_from_span_id(html, 'lbldate', 'M/D/YYYY h:mm:ss A') # RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station rtps_value = IN.read_value_from_span_id(html, 'lblrtptot') # BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station btps_value = IN.read_value_from_span_id(html, 'lblbtptot') # YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station ytps_value = IN.read_value_from_span_id(html, 'ytptot') # UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant upcl_value = IN.read_value_from_span_id(html, 'lblupctot') # JINDAl Production: https://en.wikipedia.org/wiki/JSW_Vijayanagar_Power_Station jindal_value = IN.read_value_from_span_id(html, 'lbljintot') # Coal Production coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value # Sharavati Production: Sharavati Hydroelectric sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot') # Nagjhari Production: Kalinadi-Nagjhari Hydroelectric nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot') # Varahi Production: https://en.wikipedia.org/wiki/Varahi_River#Varahi_Hydro-electric_Project varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot') # Kodsalli Production: Kalinadi Kodasalli Hydroelectric kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot') # Kadra Production: https://en.wikipedia.org/wiki/Kadra_Dam kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot') # GERUSOPPA production: Gerusoppa Dam gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot') # JOG production: https://en.wikipedia.org/wiki/Jog_Falls jog_value = IN.read_value_from_span_id(html, 'lbljogtot') # LPH Production: Linganamakki Dam lph_value = IN.read_value_from_span_id(html, 'lbllphtot') # Supa production: https://en.wikipedia.org/wiki/Supa_Dam supa_value = IN.read_value_from_span_id(html, 'lblsupatot') # SHIMSHA: https://en.wikipedia.org/wiki/Shimsha#Power_generation shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot') # SHIVASAMUDRA: https://en.wikipedia.org/wiki/Shivanasamudra_Falls#Power_generation shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot') # MANIDAM: Mani Dam Hydroelectric manidam_value = IN.read_value_from_span_id(html, 'lblmanitot') # MUNRABAD: Munirabad Hydroelectric munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot') # BHADRA: https://en.wikipedia.org/wiki/Bhadra_Dam bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot') # GHATAPRABHA: Ghataprabha Hydroelectric ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot') # ALMATTI: https://en.wikipedia.org/wiki/Almatti_Dam almatti_value = IN.read_value_from_span_id(html, 'lblalmttot') # CGS (Central Generating Stations) Production # TODO: Search CGS production type cgs_value = IN.read_value_from_span_id(html, 'lblcgs') # NCEP (Non-Conventional Energy Production) ncep_html = web.get_response_soup(country_code, 'http://kptclsldc.com/StateNCEP.aspx', session) ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1', 'DD/MM/YYYY HH:mm:ss') # Check ncep date is similar than state gen date if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600: raise ParserException('IN-KA', 'NCEP or State datetime is not valid') biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb') # TODO: Cogeneration value production type? cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc') mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm') wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw') solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts') # Hydro production hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \ + kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \ + shimsha_value + shivasamudra_value + manidam_value + munrabad_value \ + bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value # Unknown production unknown_value = cgs_value + cogen_value data = { 'countryCode': country_code, 'datetime': india_date_time.datetime, 'production': { 'biomass': biomass_value, 'coal': coal_value, 'gas': 0.0, 'hydro': hydro_value, 'nuclear': 0.0, 'oil': 0.0, 'solar': solar_value, 'wind': wind_value, 'geothermal': 0.0, 'unknown': unknown_value }, 'storage': { 'hydro': 0.0 }, 'source': 'kptclsldc.com', } return data
def fetch_production(country_code='ES-CN', session=None): ses = session or Session() elhierro = ElHierro(ses).get() if not elhierro: raise ParserException("ES-CN", "ElHierro not response") granacanaria = GranCanaria(ses).get() if not granacanaria: raise ParserException("ES-CN", "GranCanaria not response") gomera = Gomera(ses).get() if not gomera: raise ParserException("ES-CN", "Gomera not response") lanzarotefuerteventura = LanzaroteFuerteventura(ses).get() if not lanzarotefuerteventura: raise ParserException("ES-CN", "LanzaroteFuerteventura not response") palma = LaPalma(ses).get() if not palma: raise ParserException("ES-CN", "LaPalma not response") tenerife = Tenerife(ses).get() if not tenerife: raise ParserException("ES-CN", "Tenerife not response") ## Compare timestamps ## Raise ParserException if timestamps aren't equals if elhierro.timestamp != granacanaria.timestamp \ and elhierro.timestamp != gomera.timestamp \ and elhierro.timestamp != lanzarotefuerteventura.timestamp \ and elhierro.timestamp != palma.timestamp \ and elhierro.timestamp != tenerife.timestamp: raise ParserException("ES-CN", "Response timestamps aren't equals") ## Gas production gas_elhierro = elhierro.gas + elhierro.combined gas_granacanaria = granacanaria.gas + granacanaria.combined gas_gomera = gomera.gas + gomera.combined gas_lanzarotefuerteventura = lanzarotefuerteventura.gas + lanzarotefuerteventura.combined gas_palma = palma.gas + palma.combined gas_tenerife = tenerife.gas + tenerife.combined gas_total = gas_elhierro + gas_granacanaria + gas_gomera + gas_lanzarotefuerteventura + gas_palma + gas_tenerife ## Solar production solar_total = elhierro.solar + granacanaria.solar + gomera.solar + lanzarotefuerteventura.solar + palma.solar + tenerife.solar ## Oil production oil_elhierro = elhierro.vapor + elhierro.diesel oil_granacanaria = granacanaria.gas + granacanaria.combined oil_gomera = gomera.gas + gomera.combined oil_lanzarotefuerteventura = lanzarotefuerteventura.gas + lanzarotefuerteventura.combined oil_palma = palma.gas + palma.combined oil_tenerife = tenerife.gas + tenerife.combined oil_total = oil_elhierro + oil_granacanaria + oil_gomera + oil_lanzarotefuerteventura + oil_palma + oil_tenerife ## Wind production wind_total = elhierro.wind + granacanaria.wind + gomera.wind + lanzarotefuerteventura.wind + palma.wind + tenerife.wind ## Hydro production (EL Hierrro is exluded) hydro_total = granacanaria.hydraulic + gomera.hydraulic + lanzarotefuerteventura.hydraulic + palma.hydraulic + tenerife.hydraulic ## Hydro storage hydro_storage = -elhierro.hydraulic data = { 'countryCode': country_code, 'datetime': get(elhierro.timestamp).datetime, 'production': { 'coal': 0.0, 'gas': round(gas_total, 2), 'solar': round(solar_total, 2), 'oil': round(oil_total, 2), 'wind': round(wind_total, 2), 'hydro': round(hydro_total, 2), 'biomass': 0.0, 'nuclear': 0.0, 'geothermal': 0.0, 'unknown': 0.0 }, 'storage': { 'hydro': round(hydro_storage, 2) }, 'source': 'demanda.ree.es', } return data
def test_instance(self): exception = ParserException("ESIOS", "Parser exception") self.assertIsInstance(exception, ParserException) self.assertEqual(str(exception), "ESIOS Parser: Parser exception")
def test_instance(self): exception = ParserException('ESIOS', "Parser exception") self.assertIsInstance(exception, ParserException) self.assertEquals(str(exception), 'ESIOS Parser: Parser exception')
def get_response_text(country_code, url, session=None): """Get text response""" response = get_response(country_code, url, session) if not response.text: raise ParserException(country_code, 'Response empty') return response.text
def test_instance_with_zone_key(self): exception = ParserException("ESIOS", "Parser exception", "ES") self.assertIsInstance(exception, ParserException) self.assertEqual(str(exception), "ESIOS Parser (ES): Parser exception")
def fetch_production( zone_key="LK", session=None, target_datetime=None, logger: logging.Logger = logging.getLogger(__name__), ): """Requests the previous day's production mix (in MW) for Sri Lanka, per quarter-hour""" if target_datetime is not None and target_datetime < arrow.utcnow().shift(days=-2): raise NotImplementedError( "The datasource currently only has data for yesterday" ) # so 0-24 hours ago just after local midnight to approx. 24-48 hours ago just before midnight r = session or requests.session() response = r.get(GENERATION_BREAKDOWN_URL) assert response.status_code == 200, ( "Exception when fetching production for " "{}: error when calling url={}".format(zone_key, GENERATION_BREAKDOWN_URL) ) source_data = json.loads( response.json() ) # Response is double encoded; a JSON array encoded as a JSON string logger.debug(f"Raw generation breakdown: {source_data}", extra={"key": zone_key}) output = [] for quarter_hourly_source_data in source_data: output_for_timestamp = { "zoneKey": zone_key, "datetime": arrow.get( quarter_hourly_source_data["DateTime"], "YYYY-MM-DD HH:mm:ss", tzinfo=TIMEZONE_NAME, ).datetime, "production": { "biomass": 0.0, "coal": 0.0, "gas": 0.0, "hydro": 0.0, "nuclear": 0.0, "oil": 0.0, "solar": 0.0, "wind": 0.0, "geothermal": 0.0, "unknown": 0.0, }, "source": SOURCE_NAME, } for generation_type, outputInMW in quarter_hourly_source_data.items(): if generation_type == "DateTime": continue if generation_type == "Coal": output_for_timestamp["production"]["coal"] += outputInMW elif generation_type == "Major Hydro" or generation_type == "SPP Minihydro": output_for_timestamp["production"]["hydro"] += outputInMW elif generation_type == "SPP Biomass": output_for_timestamp["production"]["biomass"] += outputInMW elif generation_type == "Solar": output_for_timestamp["production"]["solar"] += outputInMW elif generation_type == "Thermal-Oil": output_for_timestamp["production"]["oil"] += outputInMW elif generation_type == "Wind": output_for_timestamp["production"]["wind"] += outputInMW else: raise ParserException( zone_key, "Unknown production type: " + generation_type ) output.append(output_for_timestamp) return output