def get_entsoe_data(start, end, expected_length):
# Defining entsoe key
client = EntsoePandasClient("94aa148a-330b-4eee-ba0c-8a5eb0b17825")
country_code = 'DE' # Germany
# methods that return Pandas Series
# 'documentType': 'A75',
# 'processType': 'A16',
# https://github.com/EnergieID/entsoe-py/blob/5d176699472744c1acef546410826da6549112cf/entsoe/entsoe.py#L270
entsoe_data = client.query_generation(country_code,
start=start,
end=end,
psr_type=None)
# converting to GMT
entsoe_data.index = entsoe_data.index.tz_convert('Etc/GMT')
# Fixing missing and null values for last few datapoints (today)
last_dp = entsoe_data.loc[[entsoe_data.index[-1]]]
for i in range(expected_length - len(entsoe_data)):
entsoe_data = entsoe_data.append(last_dp, ignore_index=False)
entsoe_data = entsoe_data.ffill()
# changing type from non-type to float. because later we want to calculate the average among all cities
entsoe_data = entsoe_data.apply(pd.to_numeric)
return entsoe_data
def get_energy_data(country_code):
load_dotenv()
token = os.environ['TOKEN']
client = EntsoePandasClient(api_key=token)
end = pd.Timestamp.now(tz=TIMEZONE_MAPPINGS[country_code])
start = end - pd.DateOffset(months=1)
df = client.query_generation(country_code,
start=start,
end=end,
psr_type=None)
#Resampling the dataframe with an hourly frequency,
#because some of the countries provide time series
#with higher frequencies (15T), but we don't need that.
df = df.resample('H').mean()
return df
def get_generation(date_string, country_code) -> dict:
from entsoe import EntsoePandasClient
client = EntsoePandasClient(api_key=os.getenv("ENTSOE_API_KEY"))
start = pd.Timestamp(date_string, tz='Europe/Brussels')
end = start + pd.Timedelta("1 days")
filename = os.path.join("files", country_code, f"{date_string}.pkl")
if os.path.isfile(filename):
with open(filename, "rb") as file:
data_dict = pickle.load(file)
else:
directory = os.path.dirname(filename)
os.makedirs(directory, exist_ok=True)
data_series = client.query_generation(country_code, start=start,end=end, psr_type=None, nett=True).sum()
data_dict = dict(data_series)
if date_string < datetime.date.today().strftime("%Y%m%d"):
with open(filename, "wb") as file:
pickle.dump(data_dict, file)
# df is a series
return data_dict
def calc_generation_per_country(country_code,
start,
psr_type=None,
periods=365):
client = EntsoePandasClient(api_key=MyToken)
ts_country = pd.DataFrame()
for day in pd.date_range(start, periods=periods):
print(day)
end = start + pd.Timedelta(days=1)
ts_country = pd.concat([
ts_country,
client.query_generation(country_code, day,
day + pd.Timedelta(days=1), psr_type)
])
ts_country.to_csv('generation' + str(psr_type) + '_' + str(country_code) +
'.csv')
return ts_country
import sqlite3
import pandas as pd
from datetime import date, timedelta
import time
end_date = date.today() + timedelta(days=1)
current_date = end_date.strftime("%Y%m%d")
client = EntsoePandasClient(api_key='444fc771-5d0f-499f-9328-90c05c459219')
start = pd.Timestamp('20201109', tz='Europe/Brussels')
end = pd.Timestamp(current_date, tz='Europe/Brussels')
country_code = 'DE'
generation1 = client.query_generation(country_code,
start=start,
end=end,
psr_type=None)
generation = generation1.iloc[:,
generation1.columns.get_level_values(1) ==
'Actual Aggregated']
generation.columns = generation.columns.droplevel(level=1)
print(generation)
generation['Date'] = generation.index
generation.insert(0, 'id', range(0, len(generation)))
generation['Total_Non_Renewables'] = generation[
'Fossil Brown coal/Lignite'] + generation['Fossil Gas'] + generation[
'Fossil Hard coal'] + generation['Fossil Oil'] + generation[
'Nuclear'] + generation['Other']
# Belgum solar data
# In[29]:
client = EntsoePandasClient(api_key='#######################')
start = pd.Timestamp('20171201', tz='Europe/Brussels')
end = pd.Timestamp('20180701', tz='Europe/Brussels')
country_code = 'BE' # Belgium
# total energy generation
belgium=client.query_generation(country_code, start=start,end=end, psr_type=None)
# In[30]:
belgium.tail(10)
# In[31]:
trace0=go.Scatter(x=belgium.index,y=belgium['Wind Onshore'],name='Onshore Wind')
trace1=go.Scatter(x=belgium.index,y=belgium['Solar'],name='Solar')
start2, end2 = pd.Timestamp('20200101', tz='Europe/Brussels'), pd.Timestamp(
'20200501', tz='Europe/Brussels')
for country_code in [
'DE-LU', 'BE', 'DK', 'AT', 'BG', 'CH', 'CZ', 'EE', 'ES', 'FI', 'FR',
'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'NL', 'NO', 'PL', 'PT', 'RO', 'SE'
]:
try:
#download load data
pd.concat([
client.query_load(country_code, start=start, end=end),
client.query_load(country_code, start=start2, end=end2)
]).to_csv('./data/load_' + country_code + '.csv')
#download generation data
pd.concat([
client.query_generation(country_code,
start=start,
end=end,
psr_type=None),
client.query_generation(country_code,
start=start2,
end=end2,
psr_type=None)
]).to_csv('./data/gen' + country_code + '.csv')
except:
print('an error occured, country code: ', country_code)
## Entsoe Transparancy platform login:
key = "285a3e21-c574-453b-9690-e327c7bd73fd" ## don't publish
client = EntsoePandasClient(api_key=key)
## Time window:
start_time = pd.Timestamp('20160101', tz='Europe/Brussels')
end_time = pd.Timestamp('20161231', tz='Europe/Brussels')
country_code = "CH" # Switzerland
neighbour_countries = ["AT", "DE", "FR", "IT"]
# it is resampled to an hour with mean because the values coming out of the database are acutally given in MW and not in MWh
domestic_production = client.query_generation(country_code, start=start_time, end=end_time, psr_type=None).resample("H").mean()
neighbour_production = []
for country in neighbour_countries:
neighbour_production.append(client.query_generation(country, start=start_time, end=end_time, psr_type=None).resample("H").mean())
emission_production_matrix_filepath = r"C:\Users\walkerl\Documents\code\proof_of_concept\data\emissions_technology_matrix.xlsx"
emission_production_matrix = pd.read_excel(emission_production_matrix_filepath, index_col=0)
#import emissions per technology (gCO2eq/kWh)
domestic_emissions_df = pd.DataFrame(index=domestic_production.index, columns=domestic_production.keys())
# This selects the data source for the emission factors.
data_source = "EMPA HUESS"