def read_text_by_regex(regex, text):
date_match = search(regex, text)
if not date_match:
raise ParserException('IN-PB', 'Not date_match')
date_text = date_match.group(0)
if not date_text:
raise ParserException('IN-PB', 'Not date_text')
return date_text
def fetch_exchange(country_code1='ES', country_code2='ES-IB', session=None):
ses = session or Session()
responses = BalearicIslands(ses).get_all()
if not responses:
raise ParserException("ES-IB", "No response")
else:
data = []
for response in responses:
sorted_country_codes = sorted([country_code1, country_code2])
net_flow = response.link['pe_ma']
response_data = {
'sortedCountryCodes':
'->'.join(sorted_country_codes),
'datetime':
get(response.timestamp).datetime,
'netFlow':
net_flow if country_code1 == sorted_country_codes[0] else -1 *
net_flow,
'source':
'demanda.ree.es',
}
data.append(response_data)
return data
def get_response(country_code, url, session=None):
"""Get response"""
ses = session or Session()
response = ses.get(url)
if response.status_code != 200:
raise ParserException(
country_code, 'Response code: {0}'.format(response.status_code))
return response
def assert_country_code(country_code, expected, parser_name=None):
"""Assert country code"""
if not country_code or country_code != expected:
if not parser_name:
parser_name = country_code
raise ParserException(
parser_name,
'Country_code expected {0}, is {1}'.format(expected, country_code),
country_code)
def fetch_consumption(country_code='ES-CN', session=None):
ses = session or Session()
elhierro = ElHierro(ses).get()
if not elhierro:
raise ParserException("ES-CN", "ElHierro not response")
granacanaria = GranCanaria(ses).get()
if not granacanaria:
raise ParserException("ES-CN", "GranCanaria not response")
gomera = Gomera(ses).get()
if not gomera:
raise ParserException("ES-CN", "Gomera not response")
lanzarotefuerteventura = LanzaroteFuerteventura(ses).get()
if not lanzarotefuerteventura:
raise ParserException("ES-CN", "LanzaroteFuerteventura not response")
palma = LaPalma(ses).get()
if not palma:
raise ParserException("ES-CN", "LaPalma not response")
tenerife = Tenerife(ses).get()
if not tenerife:
raise ParserException("ES-CN", "Tenerife not response")
## Compare timestamps
## Raise ParserException if timestamps aren't equals
if elhierro.timestamp != granacanaria.timestamp \
and elhierro.timestamp != gomera.timestamp \
and elhierro.timestamp != lanzarotefuerteventura.timestamp \
and elhierro.timestamp != palma.timestamp \
and elhierro.timestamp != tenerife.timestamp:
raise ParserException("ES-CN", "Response timestamps aren't equals")
demand = round(
elhierro.demand + granacanaria.demand + gomera.demand +
lanzarotefuerteventura.demand + palma.demand + tenerife.demand, 2)
data = {
'countryCode': country_code,
'datetime': get(elhierro.timestamp).datetime,
'consumption': demand,
'source': 'demanda.ree.es'
}
return data
def fetch_consumption(country_code='ES-IB', session=None):
ses = session or Session()
responses = BalearicIslands(ses).get_all()
if not responses:
raise ParserException("ES-IB", "No response")
else:
data = []
for response in responses:
response_data = {
'countryCode': country_code,
'datetime': get(response.timestamp).datetime,
'consumption': response.demand,
'source': 'demanda.ree.es'
}
data.append(response_data)
return data
def fetch_production(country_code='ES-IB', session=None):
ses = session or Session()
responses = BalearicIslands(ses).get_all()
if not responses:
raise ParserException("ES-IB", "No response")
else:
data = []
for response in responses:
response_data = {
'countryCode': country_code,
'datetime': get(response.timestamp).datetime,
'production': {
'coal': response.carbon,
'gas': round(response.gas + response.combined, 2),
'solar': response.solar,
'oil': round(response.vapor + response.diesel, 2),
'wind': response.wind,
'hydro': response.hydraulic,
'biomass': 0.0,
'nuclear': 0.0,
'geothermal': 0.0,
'unknown': response.unknown()
},
'storage': {
'hydro': 0.0,
'battery': 0.0
},
'source': 'demanda.ree.es',
}
data.append(response_data)
return data
def fetch_island_data(country_code, session):
if country_code == 'ES-CN-FVLZ':
lanzarote_fuerteventura_data = LanzaroteFuerteventura(session).get_all()
if not lanzarote_fuerteventura_data:
raise ParserException(country_code, "LanzaroteFuerteventura not response")
else:
return lanzarote_fuerteventura_data
elif country_code == 'ES-CN-GC':
gran_canaria_data = GranCanaria(session).get_all()
if not gran_canaria_data:
raise ParserException(country_code, "GranCanaria not response")
else:
return gran_canaria_data
elif country_code == 'ES-CN-IG':
gomera_data = Gomera(session).get_all()
if not gomera_data:
raise ParserException(country_code, "Gomera not response")
else:
return gomera_data
elif country_code == 'ES-CN-LP':
la_palma_data = LaPalma(session).get_all()
if not la_palma_data:
raise ParserException(country_code, "LaPalma not response")
else:
return la_palma_data
elif country_code == 'ES-CN-TE':
tenerife_data = Tenerife(session).get_all()
if not tenerife_data:
raise ParserException(country_code, "Tenerife not response")
else:
return tenerife_data
elif country_code == 'ES-CN-HI':
el_hierro_data = ElHierro(session).get_all()
if not el_hierro_data:
raise ParserException(country_code, "ElHierro not response")
else:
return el_hierro_data
else:
raise ParserException(country_code, 'Can\'t read this country code {0}'.format(country_code))
def fetch_islands_data(country_code, session):
data = {}
el_hierro_data = ElHierro(session).get_all()
if not el_hierro_data:
raise ParserException(country_code, "ElHierro not response")
else:
data.update({'el_hierro': el_hierro_data})
gran_canaria_data = GranCanaria(session).get_all()
if not gran_canaria_data:
raise ParserException(country_code, "GranCanaria not response")
else:
data.update({'gran_canaria': gran_canaria_data})
gomera_data = Gomera(session).get_all()
if not gomera_data:
raise ParserException(country_code, "Gomera not response")
else:
data.update({'gomera': gomera_data})
lanzarote_fuerteventura_data = LanzaroteFuerteventura(session).get_all()
if not lanzarote_fuerteventura_data:
raise ParserException(country_code,
"LanzaroteFuerteventura not response")
else:
data.update({'lanzarote_fuerteventura': lanzarote_fuerteventura_data})
la_palma_data = LaPalma(session).get_all()
if not la_palma_data:
raise ParserException(country_code, "LaPalma not response")
else:
data.update({'la_palma': la_palma_data})
tenerife_data = Tenerife(session).get_all()
if not tenerife_data:
raise ParserException(country_code, "Tenerife not response")
else:
data.update({'tenerife': tenerife_data})
return data
def fetch_production(zone_key='LK', session=None, target_datetime=None, logger: logging.Logger = logging.getLogger(__name__)):
"""Requests the previous day's production mix (in MW) for Sri Lanka, per quarter-hour
"""
if target_datetime is not None and target_datetime < arrow.utcnow().shift(days=-2):
raise NotImplementedError('The datasource currently only has data for yesterday') # so 0-24 hours ago just after local midnight to approx. 24-48 hours ago just before midnight
r = session or requests.session()
response = r.get(GENERATION_BREAKDOWN_URL)
assert response.status_code == 200, 'Exception when fetching production for ' \
'{}: error when calling url={}'.format(
zone_key, GENERATION_BREAKDOWN_URL)
source_data = json.loads(response.json()) # Response is double encoded; a JSON array encoded as a JSON string
logger.debug(f"Raw generation breakdown: {source_data}", extra={"key": zone_key})
output = []
for quarter_hourly_source_data in source_data:
output_for_timestamp = {
'zoneKey': zone_key,
'datetime': arrow.get(quarter_hourly_source_data['DateTime'], 'YYYY-MM-DD HH:mm:ss', tzinfo=TIMEZONE_NAME).datetime,
'production': {
'biomass': 0.0,
'coal': 0.0,
'gas': 0.0,
'hydro': 0.0,
'nuclear': 0.0,
'oil': 0.0,
'solar': 0.0,
'wind': 0.0,
'geothermal': 0.0,
'unknown': 0.0
},
'source': SOURCE_NAME
}
for generation_type, outputInMW in quarter_hourly_source_data.items():
if(generation_type == "DateTime"):
continue
if generation_type == "Coal":
output_for_timestamp['production']['coal'] += outputInMW
elif generation_type == "Major Hydro" or generation_type == 'SPP Minihydro':
output_for_timestamp['production']['hydro'] += outputInMW
elif generation_type == "SPP Biomass":
output_for_timestamp['production']['biomass'] += outputInMW
elif generation_type == "Solar":
output_for_timestamp['production']['solar'] += outputInMW
elif generation_type == "Thermal-Oil":
output_for_timestamp['production']['oil'] += outputInMW
elif generation_type == "Wind":
output_for_timestamp['production']['wind'] += outputInMW
else:
raise ParserException(zone_key, "Unknown production type: " + generation_type)
output.append(output_for_timestamp)
return output
def fetch_exchange(country_code1='ES',
country_code2='MA',
session=None,
token=None):
# Get ESIOS token
token = environ.get('ESIOS_TOKEN', token)
if not token:
raise ParserException("ESIOS", "Require access token")
ses = session or requests.Session()
# Request headers
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json; application/vnd.esios-api-v2+json',
'Authorization': 'Token token="{0}"'.format(token)
}
# Request query url
utc = arrow.utcnow()
start_date = utc.shift(hours=-24).floor('hour').isoformat()
end_date = utc.ceil('hour').isoformat()
dates = {'start_date': start_date, 'end_date': end_date}
query = urlencode(dates)
url = 'https://api.esios.ree.es/indicators/10209?{0}'.format(query)
response = ses.get(url, headers=headers)
if response.status_code != 200 or not response.text:
raise ParserException(
'ESIOS', 'Response code: {0}'.format(response.status_code))
json = response.json()
values = json['indicator']['values']
if not values:
raise ParserException('ESIOS', 'No values received')
else:
data = []
sorted_country_codes = sorted([country_code1, country_code2])
for value in values:
# Get last value in datasource
datetime = arrow.get(value['datetime_utc']).datetime
# Datasource negative value is exporting, positive value is importing
net_flow = -value['value']
value_data = {
'sortedCountryCodes':
'->'.join(sorted_country_codes),
'datetime':
datetime,
'netFlow':
net_flow if country_code1 == sorted_country_codes[0] else -1 *
net_flow,
'source':
'api.esios.ree.es',
}
data.append(value_data)
return data
def test_instance_with_country_code(self):
exception = ParserException('ESIOS', "Parser exception", "ES")
self.assertIsInstance(exception, ParserException)
self.assertEqual(str(exception), 'ESIOS Parser (ES): Parser exception')
def fetch_production(country_code='IN-KA', session=None):
"""Fetch Karnataka production"""
countrycode.assert_country_code(country_code, 'IN-KA')
html = web.get_response_soup(country_code,
'http://kptclsldc.com/StateGen.aspx', session)
india_date_time = IN.read_datetime_from_span_id(html, 'lbldate',
'M/D/YYYY h:mm:ss A')
# RTPS Production: https://en.wikipedia.org/wiki/Raichur_Thermal_Power_Station
rtps_value = IN.read_value_from_span_id(html, 'lblrtptot')
# BTPS Production: https://en.wikipedia.org/wiki/Bellary_Thermal_Power_station
btps_value = IN.read_value_from_span_id(html, 'lblbtptot')
# YTPS Production: https://en.wikipedia.org/wiki/Yermarus_Thermal_Power_Station
ytps_value = IN.read_value_from_span_id(html, 'ytptot')
# UPCL Production: https://en.wikipedia.org/wiki/Udupi_Power_Plant
upcl_value = IN.read_value_from_span_id(html, 'lblupctot')
# JINDAl Production: https://en.wikipedia.org/wiki/JSW_Vijayanagar_Power_Station
jindal_value = IN.read_value_from_span_id(html, 'lbljintot')
# Coal Production
coal_value = rtps_value + btps_value + ytps_value + upcl_value + jindal_value
# Sharavati Production: Sharavati Hydroelectric
sharavati_value = IN.read_value_from_span_id(html, 'lblshvytot')
# Nagjhari Production: Kalinadi-Nagjhari Hydroelectric
nagjhari_value = IN.read_value_from_span_id(html, 'lblngjtot')
# Varahi Production: https://en.wikipedia.org/wiki/Varahi_River#Varahi_Hydro-electric_Project
varahi_value = IN.read_value_from_span_id(html, 'lblvrhtot')
# Kodsalli Production: Kalinadi Kodasalli Hydroelectric
kodsalli_value = IN.read_value_from_span_id(html, 'lblkdsltot')
# Kadra Production: https://en.wikipedia.org/wiki/Kadra_Dam
kadra_value = IN.read_value_from_span_id(html, 'lblkdrtot')
# GERUSOPPA production: Gerusoppa Dam
gerusoppa_value = IN.read_value_from_span_id(html, 'lblgrsptot')
# JOG production: https://en.wikipedia.org/wiki/Jog_Falls
jog_value = IN.read_value_from_span_id(html, 'lbljogtot')
# LPH Production: Linganamakki Dam
lph_value = IN.read_value_from_span_id(html, 'lbllphtot')
# Supa production: https://en.wikipedia.org/wiki/Supa_Dam
supa_value = IN.read_value_from_span_id(html, 'lblsupatot')
# SHIMSHA: https://en.wikipedia.org/wiki/Shimsha#Power_generation
shimsha_value = IN.read_value_from_span_id(html, 'lblshimtot')
# SHIVASAMUDRA: https://en.wikipedia.org/wiki/Shivanasamudra_Falls#Power_generation
shivasamudra_value = IN.read_value_from_span_id(html, 'lblshivtot')
# MANIDAM: Mani Dam Hydroelectric
manidam_value = IN.read_value_from_span_id(html, 'lblmanitot')
# MUNRABAD: Munirabad Hydroelectric
munrabad_value = IN.read_value_from_span_id(html, 'lblmbdtot')
# BHADRA: https://en.wikipedia.org/wiki/Bhadra_Dam
bhadra_value = IN.read_value_from_span_id(html, 'lblbdratot')
# GHATAPRABHA: Ghataprabha Hydroelectric
ghataprabha_value = IN.read_value_from_span_id(html, 'lblgtprtot')
# ALMATTI: https://en.wikipedia.org/wiki/Almatti_Dam
almatti_value = IN.read_value_from_span_id(html, 'lblalmttot')
# CGS (Central Generating Stations) Production
# TODO: Search CGS production type
cgs_value = IN.read_value_from_span_id(html, 'lblcgs')
# NCEP (Non-Conventional Energy Production)
ncep_html = web.get_response_soup(country_code,
'http://kptclsldc.com/StateNCEP.aspx',
session)
ncep_date_time = IN.read_datetime_from_span_id(ncep_html, 'Label1',
'DD/MM/YYYY HH:mm:ss')
# Check ncep date is similar than state gen date
if abs(india_date_time.timestamp - ncep_date_time.timestamp) > 600:
raise ParserException('IN-KA', 'NCEP or State datetime is not valid')
biomass_value = IN.read_value_from_span_id(ncep_html, 'lbl_tb')
# TODO: Cogeneration value production type?
cogen_value = IN.read_value_from_span_id(ncep_html, 'lbl_tc')
mini_hydro_value = IN.read_value_from_span_id(ncep_html, 'lbl_tm')
wind_value = IN.read_value_from_span_id(ncep_html, 'lbl_tw')
solar_value = IN.read_value_from_span_id(ncep_html, 'lbl_ts')
# Hydro production
hydro_value = sharavati_value + nagjhari_value + varahi_value + kodsalli_value \
+ kadra_value + gerusoppa_value + jog_value + lph_value + supa_value \
+ shimsha_value + shivasamudra_value + manidam_value + munrabad_value \
+ bhadra_value + ghataprabha_value + almatti_value + mini_hydro_value
# Unknown production
unknown_value = cgs_value + cogen_value
data = {
'countryCode': country_code,
'datetime': india_date_time.datetime,
'production': {
'biomass': biomass_value,
'coal': coal_value,
'gas': 0.0,
'hydro': hydro_value,
'nuclear': 0.0,
'oil': 0.0,
'solar': solar_value,
'wind': wind_value,
'geothermal': 0.0,
'unknown': unknown_value
},
'storage': {
'hydro': 0.0
},
'source': 'kptclsldc.com',
}
return data
def fetch_production(country_code='ES-CN', session=None):
ses = session or Session()
elhierro = ElHierro(ses).get()
if not elhierro:
raise ParserException("ES-CN", "ElHierro not response")
granacanaria = GranCanaria(ses).get()
if not granacanaria:
raise ParserException("ES-CN", "GranCanaria not response")
gomera = Gomera(ses).get()
if not gomera:
raise ParserException("ES-CN", "Gomera not response")
lanzarotefuerteventura = LanzaroteFuerteventura(ses).get()
if not lanzarotefuerteventura:
raise ParserException("ES-CN", "LanzaroteFuerteventura not response")
palma = LaPalma(ses).get()
if not palma:
raise ParserException("ES-CN", "LaPalma not response")
tenerife = Tenerife(ses).get()
if not tenerife:
raise ParserException("ES-CN", "Tenerife not response")
## Compare timestamps
## Raise ParserException if timestamps aren't equals
if elhierro.timestamp != granacanaria.timestamp \
and elhierro.timestamp != gomera.timestamp \
and elhierro.timestamp != lanzarotefuerteventura.timestamp \
and elhierro.timestamp != palma.timestamp \
and elhierro.timestamp != tenerife.timestamp:
raise ParserException("ES-CN", "Response timestamps aren't equals")
## Gas production
gas_elhierro = elhierro.gas + elhierro.combined
gas_granacanaria = granacanaria.gas + granacanaria.combined
gas_gomera = gomera.gas + gomera.combined
gas_lanzarotefuerteventura = lanzarotefuerteventura.gas + lanzarotefuerteventura.combined
gas_palma = palma.gas + palma.combined
gas_tenerife = tenerife.gas + tenerife.combined
gas_total = gas_elhierro + gas_granacanaria + gas_gomera + gas_lanzarotefuerteventura + gas_palma + gas_tenerife
## Solar production
solar_total = elhierro.solar + granacanaria.solar + gomera.solar + lanzarotefuerteventura.solar + palma.solar + tenerife.solar
## Oil production
oil_elhierro = elhierro.vapor + elhierro.diesel
oil_granacanaria = granacanaria.gas + granacanaria.combined
oil_gomera = gomera.gas + gomera.combined
oil_lanzarotefuerteventura = lanzarotefuerteventura.gas + lanzarotefuerteventura.combined
oil_palma = palma.gas + palma.combined
oil_tenerife = tenerife.gas + tenerife.combined
oil_total = oil_elhierro + oil_granacanaria + oil_gomera + oil_lanzarotefuerteventura + oil_palma + oil_tenerife
## Wind production
wind_total = elhierro.wind + granacanaria.wind + gomera.wind + lanzarotefuerteventura.wind + palma.wind + tenerife.wind
## Hydro production (EL Hierrro is exluded)
hydro_total = granacanaria.hydraulic + gomera.hydraulic + lanzarotefuerteventura.hydraulic + palma.hydraulic + tenerife.hydraulic
## Hydro storage
hydro_storage = -elhierro.hydraulic
data = {
'countryCode': country_code,
'datetime': get(elhierro.timestamp).datetime,
'production': {
'coal': 0.0,
'gas': round(gas_total, 2),
'solar': round(solar_total, 2),
'oil': round(oil_total, 2),
'wind': round(wind_total, 2),
'hydro': round(hydro_total, 2),
'biomass': 0.0,
'nuclear': 0.0,
'geothermal': 0.0,
'unknown': 0.0
},
'storage': {
'hydro': round(hydro_storage, 2)
},
'source': 'demanda.ree.es',
}
return data
def test_instance(self):
exception = ParserException("ESIOS", "Parser exception")
self.assertIsInstance(exception, ParserException)
self.assertEqual(str(exception), "ESIOS Parser: Parser exception")
def test_instance(self):
exception = ParserException('ESIOS', "Parser exception")
self.assertIsInstance(exception, ParserException)
self.assertEquals(str(exception), 'ESIOS Parser: Parser exception')
def get_response_text(country_code, url, session=None):
"""Get text response"""
response = get_response(country_code, url, session)
if not response.text:
raise ParserException(country_code, 'Response empty')
return response.text
def test_instance_with_zone_key(self):
exception = ParserException("ESIOS", "Parser exception", "ES")
self.assertIsInstance(exception, ParserException)
self.assertEqual(str(exception), "ESIOS Parser (ES): Parser exception")
def fetch_production(
zone_key="LK",
session=None,
target_datetime=None,
logger: logging.Logger = logging.getLogger(__name__),
):
"""Requests the previous day's production mix (in MW) for Sri Lanka, per quarter-hour"""
if target_datetime is not None and target_datetime < arrow.utcnow().shift(days=-2):
raise NotImplementedError(
"The datasource currently only has data for yesterday"
) # so 0-24 hours ago just after local midnight to approx. 24-48 hours ago just before midnight
r = session or requests.session()
response = r.get(GENERATION_BREAKDOWN_URL)
assert response.status_code == 200, (
"Exception when fetching production for "
"{}: error when calling url={}".format(zone_key, GENERATION_BREAKDOWN_URL)
)
source_data = json.loads(
response.json()
) # Response is double encoded; a JSON array encoded as a JSON string
logger.debug(f"Raw generation breakdown: {source_data}", extra={"key": zone_key})
output = []
for quarter_hourly_source_data in source_data:
output_for_timestamp = {
"zoneKey": zone_key,
"datetime": arrow.get(
quarter_hourly_source_data["DateTime"],
"YYYY-MM-DD HH:mm:ss",
tzinfo=TIMEZONE_NAME,
).datetime,
"production": {
"biomass": 0.0,
"coal": 0.0,
"gas": 0.0,
"hydro": 0.0,
"nuclear": 0.0,
"oil": 0.0,
"solar": 0.0,
"wind": 0.0,
"geothermal": 0.0,
"unknown": 0.0,
},
"source": SOURCE_NAME,
}
for generation_type, outputInMW in quarter_hourly_source_data.items():
if generation_type == "DateTime":
continue
if generation_type == "Coal":
output_for_timestamp["production"]["coal"] += outputInMW
elif generation_type == "Major Hydro" or generation_type == "SPP Minihydro":
output_for_timestamp["production"]["hydro"] += outputInMW
elif generation_type == "SPP Biomass":
output_for_timestamp["production"]["biomass"] += outputInMW
elif generation_type == "Solar":
output_for_timestamp["production"]["solar"] += outputInMW
elif generation_type == "Thermal-Oil":
output_for_timestamp["production"]["oil"] += outputInMW
elif generation_type == "Wind":
output_for_timestamp["production"]["wind"] += outputInMW
else:
raise ParserException(
zone_key, "Unknown production type: " + generation_type
)
output.append(output_for_timestamp)
return output
