def plot_capacity_per_country(tech: str,
countries: List[str],
lonrange=None,
latrange=None) -> None:
"""
Plot a choropleth map of the existing capacity of a technology in a series of countries.
Parameters
----------
tech: str
One of wind_offshore, wind_onshore, pv_utility, pv_residential
countries: List[str]
List of ISO codes
"""
ds = get_legacy_capacity_in_countries(tech, countries)
df = pd.DataFrame({"ISO": ds.index, "Capacity": ds.values})
df["ISO_3"] = convert_country_codes(df["ISO"].values, 'alpha_2', 'alpha_3')
fig = go.Figure(data=go.Choropleth(
locations=df['ISO_3'], # Spatial coordinates
z=df['Capacity'], # Data to be color-coded
text=[f"{cap} GW" for cap in df["Capacity"].values],
colorscale='Reds',
colorbar_title="Capacity (GW)"))
fig.update_layout(geo=dict(lonaxis=dict(range=lonrange, ),
lataxis=dict(range=latrange, ),
scope='europe'))
fig.show()
def get_co2_emission_level_for_country(country_code: str, year: int) -> float:
"""
Return CO2 emissions (in kT) from the electricity sector for a given country at a given year.
Parameters
----------
country_code: str
ISO codes of country
year: int
Year between 1990 and 2018
Returns
-------
float
kT of CO2 emitted
"""
assert 1990 <= year <= 2018, "Error: Data is only available for the period 1990-2018"
emission_src_dir = f"{data_path}indicators/emissions/source/"
iea_available_countries = [
c.strip(".csv") for c in listdir(f"{emission_src_dir}iea/")
if c.endswith(".csv")
]
assert country_code in iea_available_countries, f"Error: Data is not available for country {country_code}"
# First try to access co2 intensity from EEA database
eea_emission_fn = f"{emission_src_dir}eea/co2-emission-intensity-5.csv"
eea_emission_df = pd.read_csv(eea_emission_fn,
index_col=0,
usecols=[0, 1, 4])
eea_emission_df.columns = ["Country", "co2 (g/kWh)"]
country_name = convert_country_codes([country_code], 'alpha_2', 'name',
True)[0]
if country_name in set(eea_emission_df["Country"].values) and \
year in eea_emission_df[eea_emission_df["Country"] == country_name].index:
co2_intensity = eea_emission_df[eea_emission_df["Country"] ==
country_name].loc[year, "co2 (g/kWh)"]
# Multiply by production to obtain total co2 emissions (in kT)
iea_production_fn = f"{data_path}generation/misc/source/iea/total/{country_code}.csv"
iea_production_df = pd.read_csv(iea_production_fn, index_col=0)
return co2_intensity * iea_production_df.loc[
year, "Electricity Production (GWh)"] * 1e6 / 1e9
else:
# If data for the country is not accessible from EEA, use data from IEA
iea_emission_fn = f"{emission_src_dir}iea/{country_code}.csv"
iea_emission_df = pd.read_csv(iea_emission_fn, index_col=0).dropna()
co2_emissions = 0.
if year in iea_emission_df.index:
co2_emissions = iea_emission_df.loc[
year, "CO2 from electricity and heat producers (MT)"]
else:
warnings.warn(
f"No available value for {country_code} for year {year}, setting emissions to 0."
)
return co2_emissions * 1e3
def compute_storage_capacities(sto_capacity_ds: pd.Series) -> pd.Series:
"""
Computing STO energy capacities (TWh) per unit region.
Parameters
----------
sto_capacity_ds: pd.Series
DataFrame containing STO installed capacities per unit region (e.g., "countries", "NUTS3")
Returns
-------
hydro_storage_energy_cap_ds: pd.Series
DataFrame containing STO energy storage ratings.
"""
source_dir = f"{data_path}generation/hydro/source/"
# Initially reading modelled data from Hartel et. al (2017)
hydro_storage_capacities_fn = f"{source_dir}Hartel_2017_EU_hydro_storage_capacities.xlsx"
hydro_storage_energy_cap_ds = pd.read_excel(hydro_storage_capacities_fn, skiprows=1,
usecols=['ISO2', 'Eq. Storage'], index_col='ISO2', squeeze=True) * 1e3
# Get storage capacities for countries which are not in the Hartel study
iso2_codes = sorted(replace_iso2_codes(list(set([region_code[:2] for region_code in sto_capacity_ds.index]))))
hydro_storage_energy_cap_ds = hydro_storage_energy_cap_ds.reindex(iso2_codes)
for iso2_code in iso2_codes:
# If c is not covered in the Hartel study...
# if iso2_code not in hydro_storage_energy_cap_ds.index:
if np.isnan(hydro_storage_energy_cap_ds[iso2_code]):
country_name = revert_old_country_names(convert_country_codes([iso2_code], 'alpha_2', 'name', True)[0])
try:
# ...look-up for ENTSO-E reservoir data...
hydro_storage_capacities_entsoe_fn = f"{source_dir}ENTSOE/Water Reservoirs and Hydro Storage Plants" \
f"_201412290000-201912300000_{iso2_code}.csv"
hydro_storage_energy_cap = pd.read_csv(hydro_storage_capacities_entsoe_fn, index_col=0)
max_storage = np.nanmax(np.nan_to_num(hydro_storage_energy_cap.values.flatten()))
if max_storage > 0.:
hydro_storage_energy_cap_ds.loc[iso2_code] = max_storage * 1e-3
else:
# ...if ENTSO-E data is missing (NaNs replaced by 0s), approximate storage via GRanD v1.3
hydro_storage_energy_cap_ds.loc[iso2_code] = get_country_storage_from_grand(country_name)
except FileNotFoundError:
# ...if ENTSO-E file is missing altogether, approximate storage via GRanD v1.3
hydro_storage_energy_cap_ds.loc[iso2_code] = get_country_storage_from_grand(country_name)
# If topology unit is "countries", return series directly
if len(sto_capacity_ds.index[0]) == 2:
return hydro_storage_energy_cap_ds.round(3)
else:
# If some NUTS-based topology in place, storage distribution among regions is done via GRanD v1.3
storage_distribution_by_nuts = get_nuts_storage_distribution_from_grand(sto_capacity_ds.index)
for nuts in storage_distribution_by_nuts.index:
storage_distribution_by_nuts.loc[nuts] *= hydro_storage_energy_cap_ds.loc[replace_iso2_codes([nuts[:2]])[0]]
hydro_storage_energy_cap_ds = storage_distribution_by_nuts.copy()
return hydro_storage_energy_cap_ds.round(3)
def plot_capacity(tech: str, countries: List[str],
lon_range: List[float] = None, lat_range: List[float] = None):
"""
Plot on a map potential capacity (in GW) per country for a specific technology.
Parameters
----------
tech: str
Technology name.
countries: List[str]
List of ISO codes of countries.
lon_range: List[float] (default: None)
Longitudinal range over which to display the map. Automatically set if not specified.
lat_range: List[float] (default: None)
Latitudinal range over which to display the map. Automatically set if not specified.
"""
tech_config_dict = get_config_dict([tech], ["onshore", "power_density", "filters"])[tech]
cap_pot_ds = get_capacity_potential_per_country(countries, tech_config_dict["onshore"],
tech_config_dict["filters"], tech_config_dict["power_density"])
cap_pot_df = cap_pot_ds.to_frame()
cap_pot_df["ISO_3"] = convert_country_codes(cap_pot_df.index.values, 'alpha_2', 'alpha_3')
fig = go.Figure(data=go.Choropleth(
locations=cap_pot_df['ISO_3'], # Spatial coordinates
z=cap_pot_df[0], # Data to be color-coded
text=[f"{cap} GW" for cap in cap_pot_df[0].values],
colorscale='Reds',
colorbar_title=f"Capacity (GW)"
))
fig.update_layout(
geo=dict(
lonaxis=dict(
range=lon_range,
),
lataxis=dict(
range=lat_range,
),
scope='europe'),
title=f"Capacity potential for {tech}"
)
fig.show()
def plot_capacity_per_country(tech: str,
countries: List[str],
lon_range: List[float] = None,
lat_range: List[float] = None) -> None:
"""
Plot a choropleth map of the existing capacity of a technology in a series of countries.
Parameters
----------
tech: str
One of ror, sto and phs
countries: List[str]
List of ISO codes
lon_range: List[float] (default: None)
Longitudinal range over which to display the map. Automatically set if not specified.
lat_range: List[float] (default: None)
Latitudinal range over which to display the map. Automatically set if not specified.
"""
series = get_hydro_capacities("countries", tech)
if not isinstance(series, tuple):
series = (series, )
for ds in series:
df = pd.DataFrame(index=countries, columns=["ISO_3", "Capacity"])
df.loc[ds.index, "Capacity"] = ds.values
df["ISO_3"] = convert_country_codes(df.index.values, 'alpha_2',
'alpha_3')
unit = ds.name.split(" ")[1]
fig = go.Figure(data=go.Choropleth(
locations=df['ISO_3'], # Spatial coordinates
z=df['Capacity'], # Data to be color-coded
text=[f"{cap} GW" for cap in df["Capacity"].values],
colorscale='Reds',
colorbar_title=f"Capacity {unit}"))
fig.update_layout(geo=dict(lonaxis=dict(range=lon_range, ),
lataxis=dict(range=lat_range, ),
scope='europe'))
fig.show()
def plot_capacity_per_country(tech: str,
countries: List[str],
lon_range: List[float] = None,
lat_range: List[float] = None) -> None:
"""
Plot a choropleth map of the existing capacity of a technology in a series of countries.
Parameters
----------
tech: str
countries: List[str]
List of ISO codes
lon_range: List[float] (default: None)
Longitudinal range over which to display the map. Automatically set if not specified.
lat_range: List[float] (default: None)
Latitudinal range over which to display the map. Automatically set if not specified.
"""
df = get_powerplants(tech, countries)[["ISO2", "Capacity"]]
df = df.groupby(["ISO2"]).sum() / 1000.0
df = df[df["Capacity"] != 0]
df["ISO_3"] = convert_country_codes(df.index.values, 'alpha_2', 'alpha_3')
fig = go.Figure(data=go.Choropleth(
locations=df['ISO_3'], # Spatial coordinates
z=df['Capacity'], # Data to be color-coded
text=[f"{cap} GW" for cap in df["Capacity"].values],
colorscale='Reds',
colorbar_title=f"Capacity (GW)"))
fig.update_layout(geo=dict(lonaxis=dict(range=lon_range, ),
lataxis=dict(range=lat_range, ),
scope='europe'))
fig.show()
def get_legacy_capacity_in_regions_from_non_open(tech: str, regions_shapes: pd.Series, countries: List[str],
spatial_res: float, include_operating: bool,
match_distance: float = 50., raise_error: bool = True) -> pd.Series:
"""
Return the total existing capacity (in GW) for the given tech for a set of geographical regions.
This function is using proprietary data.
Parameters
----------
tech: str
Technology name.
regions_shapes: pd.Series [Union[Polygon, MultiPolygon]]
Geographical regions
countries: List[str]
List of ISO codes of countries in which the regions are situated
spatial_res: float
Spatial resolution of data
include_operating: bool
Include or not the legacy capacity of already operating units.
match_distance: float (default: 50)
Distance threshold (in km) used when associating points to shape.
raise_error: bool (default: True)
Whether to raise an error if no legacy data is available for this technology.
Returns
-------
capacities: pd.Series
Legacy capacities (in GW) of technology 'tech' for each region
"""
path_legacy_data = f"{data_path}generation/vres/legacy/source/"
path_gdp_data = f"{data_path}indicators/gdp/source"
path_pop_data = f"{data_path}indicators/population/source"
capacities = pd.Series(0., index=regions_shapes.index)
plant, plant_type = get_config_values(tech, ["plant", "type"])
if (plant, plant_type) in [("Wind", "Onshore"), ("Wind", "Offshore"), ("PV", "Utility")]:
if plant == "Wind":
data = pd.read_excel(f"{path_legacy_data}Windfarms_Europe_20200127.xls", sheet_name='Windfarms',
header=0, usecols=[2, 5, 9, 10, 18, 22, 23], skiprows=[1], na_values='#ND')
data = data.dropna(subset=['Latitude', 'Longitude', 'Total power'])
if include_operating:
plant_status = ['Planned', 'Approved', 'Construction', 'Production']
else:
plant_status = ['Planned', 'Approved', 'Construction']
data = data.loc[data['Status'].isin(plant_status)]
if countries is not None:
data = data[data['ISO code'].isin(countries)]
if len(data) == 0:
return capacities
# Converting from kW to GW
data['Total power'] *= 1e-6
data["Location"] = data[["Longitude", "Latitude"]].apply(lambda x: (x.Longitude, x.Latitude), axis=1)
# Keep only onshore or offshore point depending on technology
if plant_type == 'Onshore':
data = data[data['Area'] != 'Offshore']
else: # Offshore
data = data[data['Area'] == 'Offshore']
if len(data) == 0:
return capacities
else: # plant == "PV":
data = pd.read_excel(f"{path_legacy_data}Solarfarms_Europe_20200208.xlsx", sheet_name='ProjReg_rpt',
header=0, usecols=[0, 4, 5, 6, 8])
data = data[pd.notnull(data['Coords'])]
if include_operating:
plant_status = ['Building', 'Planned', 'Active']
else:
plant_status = ['Building', 'Planned']
data = data.loc[data['Status'].isin(plant_status)]
data["Location"] = data["Coords"].apply(lambda x: (float(x.split(',')[1]), float(x.split(',')[0])))
if countries is not None:
data['Country'] = convert_country_codes(data['Country'].values, 'name', 'alpha_2')
data = data[data['Country'].isin(countries)]
if len(data) == 0:
return capacities
# Converting from MW to GW
data['Total power'] = data['MWac']*1e-3
data = data[["Location", "Total power"]]
points_region = match_points_to_regions(data["Location"].values, regions_shapes,
distance_threshold=match_distance).dropna()
for region in regions_shapes.index:
points_in_region = points_region[points_region == region].index.values
capacities[region] = data[data["Location"].isin(points_in_region)]["Total power"].sum()
elif (plant, plant_type) == ("PV", "Residential"):
legacy_capacity_fn = join(path_legacy_data, 'SolarEurope_Residential_deployment.xlsx')
data = pd.read_excel(legacy_capacity_fn, header=0, index_col=0, usecols=[0, 4], squeeze=True).sort_index()
data = data[data.index.isin(countries)]
if len(data) == 0:
return capacities
# TODO: where is this file ?
gdp_data_fn = join(path_gdp_data, "GDP_per_capita_PPP_1990_2015_v2.nc")
gdp_data = xr.open_dataset(gdp_data_fn)
gdp_2015 = gdp_data.sel(time='2015.0')
pop_data_fn = join(path_pop_data, "gpw_v4_population_count_adjusted_rev11_15_min.nc")
pop_data = xr.open_dataset(pop_data_fn)
pop_2020 = pop_data.sel(raster=5)
# Temporary, to reduce the size of this ds, which is anyway read in each iteration.
min_lon, max_lon, min_lat, max_lat = -11., 32., 35., 80.
mask_lon = (gdp_2015.longitude >= min_lon) & (gdp_2015.longitude <= max_lon)
mask_lat = (gdp_2015.latitude >= min_lat) & (gdp_2015.latitude <= max_lat)
new_lon = np.arange(min_lon, max_lon+spatial_res, spatial_res)
new_lat = np.arange(min_lat, max_lat+spatial_res, spatial_res)
gdp_ds = gdp_2015.where(mask_lon & mask_lat, drop=True)['GDP_per_capita_PPP']
pop_ds = pop_2020.where(mask_lon & mask_lat, drop=True)['UN WPP-Adjusted Population Count, v4.11 (2000,'
' 2005, 2010, 2015, 2020): 15 arc-minutes']
gdp_ds = gdp_ds.reindex(longitude=new_lon, latitude=new_lat, method='nearest')\
.stack(locations=('longitude', 'latitude'))
pop_ds = pop_ds.reindex(longitude=new_lon, latitude=new_lat, method='nearest')\
.stack(locations=('longitude', 'latitude'))
all_sites = [(idx[0], idx[1]) for idx in regions_shapes.index]
total_gdp_per_capita = gdp_ds.sel(locations=all_sites).sum().values
total_population = pop_ds.sel(locations=all_sites).sum().values
df_metrics = pd.DataFrame(index=regions_shapes.index, columns=['gdp', 'pop'])
for region_id, region_shape in regions_shapes.items():
lon, lat = region_id[0], region_id[1]
df_metrics.loc[region_id, 'gdp'] = gdp_ds.sel(longitude=lon, latitude=lat).values/total_gdp_per_capita
df_metrics.loc[region_id, 'pop'] = pop_ds.sel(longitude=lon, latitude=lat).values/total_population
df_metrics['gdppop'] = df_metrics['gdp'] * df_metrics['pop']
df_metrics['gdppop_norm'] = df_metrics['gdppop']/df_metrics['gdppop'].sum()
capacities = df_metrics['gdppop_norm'] * data[countries[0]]
capacities = capacities.reset_index()['gdppop_norm']
else:
if raise_error:
raise ValueError(f"Error: No legacy data exists for tech {tech} with plant {plant} and type {plant_type}.")
else:
warnings.warn(f"Warning: No legacy data exists for tech {tech}.")
return capacities.astype(float)
def get_powerplants(tech_name: str, country_codes: List[str]) -> pd.DataFrame:
"""
Return power plants filtered by technology and country list.
Parameters
----------
tech_name: str
Name of one of the technologies defined in the system.
country_codes: List[str]
List of target ISO2 country codes.
Returns
-------
pp_df: pd.DataFrame
List of powerplants with the following attributes: name, capacity (in MW), ISO2 code, longitude and latitude.
"""
assert len(country_codes) != 0, "Error: List of country must be non-empty."
assert all([len(c) == 2 for c in country_codes]), "Error: Countries must be identified with ISO2 codes which" \
" are of length 2. Found code of different length than 2."
tech_config = get_config_dict([tech_name])[tech_name]
assert 'jrc_type' in tech_config, "Error: Capacities cannot be retrieved for this technology."
jrc_dir = f"{data_path}generation/misc/source/JRC/"
if tech_name in ['ror', 'sto', 'phs']:
# Hydro entries read from richer hydro-only database.
pp_fn = f"{jrc_dir}hydro-power-database-master/data/jrc-hydro-power-plant-database.csv"
pp_df = pd.read_csv(pp_fn, index_col=0)
pp_df.rename(columns={
'installed_capacity_MW': 'Capacity',
'name': 'Name',
'country_code': 'ISO2'
},
inplace=True)
# Replace ISO2 codes.
pp_df["ISO2"] = pp_df["ISO2"].map(lambda x: replace_iso2_codes([x])[0])
# Filter out plants outside target countries, of other tech than the target tech, whose capacity is missing.
pp_df = pp_df.loc[(pp_df["ISO2"].isin(country_codes))
& (pp_df['type'] == tech_config['jrc_type']) &
(~pp_df['Capacity'].isnull())]
else:
# All other technologies read from JRC's PPDB.
pp_fn = f"{jrc_dir}JRC-PPDB-OPEN.ver1.0/JRC_OPEN_UNITS.csv"
pp_df = pd.read_csv(pp_fn, sep=';')
pp_df["ISO2"] = convert_country_codes(pp_df['country'], 'name',
'alpha_2', True)
# Plants in the PPDB are listed per generator (multiple per plant), duplicates are hereafter dropped.
pp_df = pp_df.drop_duplicates(subset='eic_p',
keep='first').set_index('eic_p')
# Filter out plants outside target countries, of other tech than the target tech, which are decommissioned.
pp_df = pp_df.loc[(pp_df["ISO2"].isin(country_codes))
& (pp_df['type_g'] == tech_config['jrc_type']) &
(pp_df["status_g"] == 'COMMISSIONED')]
# Remove plants whose commissioning year goes back further than specified year.
if 'comm_year_threshold' in tech_config:
pp_df = pp_df[~(
pp_df['year_commissioned'] < tech_config['comm_year_threshold']
)]
# Column renaming for consistency across different datasets.
pp_df.rename(columns={
'capacity_p': 'Capacity',
'name_p': 'Name'
},
inplace=True)
# Filter out plants in countries with additional constraints (e.g., nuclear decommissioning in DE)
if 'countries_out' in tech_config:
pp_df = pp_df[~pp_df['ISO2'].isin(tech_config['countries_out'])]
pp_df['Name'] = pp_df['Name'].apply(unidecode)
return pp_df[['Name', 'Capacity', 'ISO2', 'lon', 'lat']]
def get_nuts_storage_distribution_from_grand(nuts_codes: List[str]) -> pd.Series:
"""
Estimating STO energy storage distribution per NUTS sub-divisions.
Parameters
----------
nuts_codes: List[str]
List of NUTS (e.g., "NUTS2", "NUTS3") codes for which data is retrieved.
Returns
-------
storage_distribution_ds: pd.DataFrame
DataFrame containing STO energy storage distribution keys per NUTS regions.
"""
assert len(nuts_codes) != 0, "Error: Empty list of NUTS codes."
# Read GRanD database
source_dir = f"{data_path}generation/hydro/source/GDW/GRanD_Version_1_3/"
grand_reservoirs_fn = f"{source_dir}GRanD_reservoirs_v1_3.shp"
reservoirs_df = pd.DataFrame(gpd.read_file(grand_reservoirs_fn)).set_index('GRAND_ID')
# A particular reservoir is manually removed (others could follow). The Vanern lake (SE) is labeled as a reservoir
# with hydro power activities, though information online suggests otherwise. Its presence in the associated NUTS
# region leads to inconsistencies in the distribution of Swedish storage potential across the country.
reservoirs_df = reservoirs_df[reservoirs_df['RES_NAME'] != 'Vanern']
# Get NUTS0, ISO2 and countries names of the countries which NUTS regions are part of
nuts0_codes = list(set([nuts[:2] for nuts in nuts_codes]))
iso2_codes = replace_iso2_codes(nuts0_codes)
countries_names = convert_country_codes(iso2_codes, 'alpha_2', 'name', True)
# Get NUTS region shapes
shapes = get_nuts_shapes(str(len(nuts_codes[0]) - 2), nuts_codes)
shapes_countries = replace_iso2_codes([c[:2] for c in shapes.index])
# Filtering out reservoirs whose purpose is not for hydro power generation.
reservoirs_df["COUNTRY"] = reservoirs_df["COUNTRY"].apply(convert_old_country_names)
reservoirs_hydropower_df = reservoirs_df[(reservoirs_df['USE_ELEC'].isin(['Main', 'Sec', 'Major'])) &
(reservoirs_df['COUNTRY'].isin(countries_names))].copy()
# Associating each plant to the corresponding NUTS region
reservoirs_hydropower_df["ISO2"] = \
convert_country_codes(reservoirs_hydropower_df['COUNTRY'], 'name', 'alpha_2', True)
reservoirs_hydropower_df["region_code"] = \
match_powerplants_to_regions(reservoirs_hydropower_df.rename(columns={'LONG_DD': 'lon', 'LAT_DD': 'lat'}),
shapes, shapes_countries)
reservoirs_hydropower_df = reservoirs_hydropower_df[~reservoirs_hydropower_df['region_code'].isnull()]
# Aggregating storage capacity per NUTS region
storage_by_nuts_ds = reservoirs_hydropower_df.groupby(by=reservoirs_hydropower_df['region_code'])['CAP_MCM'].sum()
# Computing storage distribution keys per NUTS by dividing the capacity
# per NUTS by the total capacity of all NUTS in the same country.
storage_distribution_ds = pd.Series()
for nuts0_code, iso2_code in zip(nuts0_codes, iso2_codes):
storage_sum_per_country = \
reservoirs_hydropower_df[reservoirs_hydropower_df['ISO2'] == iso2_code]['CAP_MCM'].sum()
storage_ds_temp = storage_by_nuts_ds[storage_by_nuts_ds.index.str.contains(nuts0_code)]
storage_ds_temp /= storage_sum_per_country
storage_distribution_ds = storage_distribution_ds.append(storage_ds_temp)
return storage_distribution_ds