def preprocess(plot: bool = False):
# TODO:
# - figure out what we really need
# - converters?
# - transformers?
# - figure out if lines and/or links can be extended
# - see prepare_network
# - Probably best to do all these steps once in a preprocess function and then just
# remove unwanted components at run time
# Load main components
# buses_df = load_buses_from_eg(countries, voltages)
buses_df = load_buses_from_eg()
# buses_df, links_df = load_links_from_eg(buses_df, config['links'], countries)
buses_df, links_df = load_links_from_eg(buses_df)
# converters_df = load_converters_from_eg(buses_df.index, config["links"])
converters_df = load_converters_from_eg(buses_df.index)
# lines_df = load_lines_from_eg(buses_df.index, config["lines"], voltages, net.line_types)
lines_df = load_lines_from_eg(buses_df.index)
# transformers_df = load_transformers_from_eg(buses_df.index, config["transformers"])
transformers_df = load_transformers_from_eg(buses_df.index)
# Add everything to the network
net = Network()
net.import_components_from_dataframe(buses_df, "Bus")
net.import_components_from_dataframe(lines_df, "Line")
net.import_components_from_dataframe(transformers_df, "Transformer")
net.import_components_from_dataframe(links_df, "Link")
net.import_components_from_dataframe(converters_df, "Link")
# Update a bunch of parameters for given components according to parameters_correction.yaml
apply_parameter_corrections(net)
# Remove subnetworks with less than a given number of components
net = remove_unconnected_components(net)
# Determine to which country each bus (onshore or offshore) belong and do some stuff with substations
# set_countries_and_substations(net, countries)
set_countries_and_substations(net)
# Set what portion of the link is under water
# set_links_underwater_fraction(net, countries)
set_links_underwater_fraction(net)
# I have no clue what this is for...
replace_b2b_converter_at_country_border_by_link(net)
# Save base network
net.export_to_csv_folder(
f"{data_path}topologies/pypsa_entsoe_gridkit/generated/base_network/")
if plot:
import matplotlib.pyplot as plt
net.plot(bus_sizes=0.001)
plt.show()
return net
def get_topology(network: pypsa.Network,
countries: List[str] = None,
p_nom_extendable: bool = True,
extension_multiplier: float = None,
extension_base: str = 'GCA',
use_ex_line_cap: bool = True,
p_max_pu: float = 1.0,
plot: bool = False) -> pypsa.Network:
"""
Load the e-highway network topology (buses and links) using PyPSA.
Parameters
----------
network: pypsa.Network
Network instance
countries: List[str] (default: None)
List of ISO codes of countries for which we want the tyndp topology.
p_nom_extendable: bool (default: True)
Whether line capacity is allowed to be expanded
extension_multiplier: float (default: None)
By how much the capacity can be extended if extendable. If None, no limit on expansion.
extension_base: str (default: GCA)
TYNDP 2040 scenario to use as the basis for computing the max potential of links, can be one of:
- ST (Sustainable Transition): targets reached by national regulation, emission trading schemes and subsidies,
maximising the use of existing infrastructure
~ 180 GW and 94 TWkm
- DG (Distributed Generation): prosumers at the centre - small-scale generation, batteries and fuel-switching
society engaged and empowered
~ 190 GW and 99 TWkm
- GCA (Global Climate Action): full-speed global decarbonisation, large-scale renewables
~ 200 GW and 103 TWkm
The three scenarios are quite similar in terms of NTCs with GCA being the most generous.
use_ex_line_cap: bool (default True)
Whether to use existing line capacity
p_max_pu: float (default: 1.0)
Maximal dispatch per unit of p_nom
plot: bool (default: False)
Whether to show loaded topology or not
Returns
-------
network: pypsa.Network
Updated network
"""
assert countries is None or len(countries) != 0, "Error: Countries list must not be empty. If you want to " \
"obtain, the full topology, don't pass anything as argument."
assert extension_base in ['ST', 'DG', 'GCA'], f"Error: extension_base must be one of ST, DG or GCA, " \
f"received {extension_base}."
topology_dir = f"{data_path}topologies/tyndp2018/generated/"
buses_fn = f"{topology_dir}buses.csv"
assert isfile(
buses_fn), f"Error: Buses are undefined. Please run 'preprocess'."
buses = pd.read_csv(buses_fn, index_col='id')
links_fn = f"{topology_dir}links.csv"
assert isfile(
links_fn), f"Error: Links are undefined. Please run 'preprocess'."
links = pd.read_csv(links_fn, index_col='id')
if countries is not None:
# Check if there is a bus for each country considered
missing_countries = set(countries) - set(buses.index)
assert not missing_countries, f"Error: No buses exist for the following countries: {missing_countries}"
# Remove buses that are not associated with the considered countries
buses = buses.loc[buses.index.isin(countries)]
countries = buses.index
# Converting polygons strings to Polygon object
for region_type in ["onshore_region", "offshore_region"]:
regions = buses[region_type].values
# Convert strings
for i, region in enumerate(regions):
if isinstance(region, str):
regions[i] = shapely.wkt.loads(region)
# If we have only one bus, add it to the network and return
if len(buses) == 1:
network.import_components_from_dataframe(buses, "Bus")
return network
# Remove links for which one of the two end buses has been removed
links = pd.DataFrame(links.loc[links.bus0.isin(buses.index)
& links.bus1.isin(buses.index)])
# Removing offshore buses that are not connected anymore
connected_buses = sorted(list(
set(links["bus0"]).union(set(links["bus1"]))))
buses = buses.loc[connected_buses]
disconnected_onshore_bus = set(countries) - set(buses.index)
assert not disconnected_onshore_bus, f"Error: Buses {disconnected_onshore_bus} were disconnected."
if not use_ex_line_cap:
links['p_nom'] = 0
links['p_nom_min'] = links['p_nom']
links['p_max_pu'] = p_max_pu
links['p_min_pu'] = -p_max_pu # Making the link bi-directional
links['p_nom_extendable'] = p_nom_extendable
if p_nom_extendable:
# Choose p_nom_max based on some TYNDP 2040 scenario
if extension_base == 'ST':
links['p_nom_max'] = links['p_nom_st']
elif extension_base == 'DG':
links['p_nom_max'] = links['p_nom_dg']
else:
links['p_nom_max'] = links['p_nom_gca']
links = links.drop(['p_nom_st', 'p_nom_dg', 'p_nom_gca'], axis=1)
if extension_multiplier is not None:
links['p_nom_max'] = (links['p_nom_max'] *
extension_multiplier).round(3)
links['p_nom_max'] = links[['p_nom_max', 'p_nom_min']].max(axis=1)
else:
links['p_nom_max'] = "inf"
links['capital_cost'] = pd.Series(index=links.index)
for idx in links.index:
carrier = links.loc[idx].carrier
cap_cost, _ = get_costs(carrier,
sum(network.snapshot_weightings['objective']))
links.loc[idx, ('capital_cost', )] = cap_cost * links.length.loc[idx]
network.import_components_from_dataframe(buses, "Bus")
network.import_components_from_dataframe(links, "Link")
if plot:
from epippy.topologies.core.plot import plot_topology
plot_topology(buses, links)
plt.show()
return network
def to_pypsa(network, mode, timesteps):
"""
Translate graph based grid representation to PyPSA Network
For details from a user perspective see API documentation of
:meth:`~.grid.network.EDisGo.analyze` of the API class
:class:`~.grid.network.EDisGo`.
Translating eDisGo's grid topology to PyPSA representation is structured
into tranlating the topology and adding time series for components of the
grid. In both cases translation of MV grid only (`mode='mv'`), LV grid only
(`mode='lv'`), MV and LV (`mode=None`) share some code. The
code is organized as follows
* Medium-voltage only (`mode='mv'`): All medium-voltage grid components are
exported by :func:`mv_to_pypsa` including the LV station. LV grid load
and generation is considered using :func:`add_aggregated_lv_components`.
Time series are collected by `_pypsa_load_timeseries` (as example
for loads, generators and buses) specifying `mode='mv'`). Timeseries
for aggregated load/generation at substations are determined individually.
* Low-voltage only (`mode='lv'`): LV grid topology including the MV-LV
transformer is exported. The slack is defind at primary side of the MV-LV
transformer.
* Both level MV+LV (`mode=None`): The entire grid topology is translated to
PyPSA in order to perform a complete power flow analysis in both levels
together. First, both grid levels are translated seperately using
:func:`mv_to_pypsa` and :func:`lv_to_pypsa`. Those are merge by
:func:`combine_mv_and_lv`. Time series are obtained at once for both grid
levels.
This PyPSA interface is aware of translation errors and performs so checks
on integrity of data converted to PyPSA grid representation
* Sub-graphs/ Sub-networks: It is ensured the grid has no islanded parts
* Completeness of time series: It is ensured each component has a time
series
* Buses available: Each component (load, generator, line, transformer) is
connected to a bus. The PyPSA representation is check for completeness of
buses.
* Duplicate labels in components DataFrames and components' time series
DataFrames
Parameters
----------
network : Network
eDisGo grid container
mode : str
Determines grid levels that are translated to
`PyPSA grid representation
<https://www.pypsa.org/doc/components.html#network>`_. Specify
* None to export MV and LV grid levels. None is the default.
* ('mv' to export MV grid level only. This includes cumulative load and
generation from underlying LV grid aggregated at respective LV
station. This option is implemented, though the rest of edisgo does
not handle it yet.)
* ('lv' to export LV grid level only. This option is not yet
implemented)
timesteps : :pandas:`pandas.DatetimeIndex<datetimeindex>` or :pandas:`pandas.Timestamp<timestamp>`
Timesteps specifies which time steps to export to pypsa representation
and use in power flow analysis.
Returns
-------
PyPSA Network
"""
# check if timesteps is array-like, otherwise convert to list (necessary
# to obtain a dataframe when using .loc in time series functions)
if not hasattr(timesteps, "__len__"):
timesteps = [timesteps]
# get topology and time series data
if mode is None:
mv_components = mv_to_pypsa(network)
lv_components = lv_to_pypsa(network)
components = combine_mv_and_lv(mv_components, lv_components)
if list(components['Load'].index.values):
timeseries_load_p_set = _pypsa_load_timeseries(network,
mode=mode,
timesteps=timesteps)
if len(list(components['Generator'].index.values)) > 1:
timeseries_gen_p_min, timeseries_gen_p_max = \
_pypsa_generator_timeseries(
network, mode=mode, timesteps=timesteps)
timeseries_storage_p_min, timeseries_storage_p_max = \
_pypsa_storage_timeseries(
network, mode=mode, timesteps=timesteps)
if list(components['Bus'].index.values):
timeseries_bus_v_set = _pypsa_bus_timeseries(
network, components['Bus'].index.tolist(), timesteps=timesteps)
else:
raise ValueError("Provide proper mode or leave it empty to export "
"entire grid topology.")
# check topology
_check_topology(components)
# create power flow problem
pypsa_network = PyPSANetwork()
pypsa_network.edisgo_mode = mode
pypsa_network.set_snapshots(timesteps)
# import grid topology to PyPSA network
# buses are created first to avoid warnings
pypsa_network.import_components_from_dataframe(components['Bus'], 'Bus')
for k, comps in components.items():
if k is not 'Bus' and not comps.empty:
pypsa_network.import_components_from_dataframe(comps, k)
# import time series to PyPSA network
if len(list(components['Generator'].index.values)) > 1:
import_series_from_dataframe(pypsa_network, timeseries_gen_p_min,
'Generator', 'p_min_pu')
import_series_from_dataframe(pypsa_network, timeseries_gen_p_max,
'Generator', 'p_max_pu')
import_series_from_dataframe(pypsa_network, timeseries_storage_p_min,
'Generator', 'p_min_pu')
import_series_from_dataframe(pypsa_network, timeseries_storage_p_max,
'Generator', 'p_max_pu')
if list(components['Load'].index.values):
import_series_from_dataframe(pypsa_network, timeseries_load_p_set,
'Load', 'p_set')
if list(components['Bus'].index.values):
import_series_from_dataframe(pypsa_network, timeseries_bus_v_set,
'Bus', 'v_mag_pu_set')
_check_integrity_of_pypsa(pypsa_network)
return pypsa_network
def get_topology(network: pypsa.Network,
countries: List[str] = None,
add_offshore: bool = True,
extend_line_cap: bool = True,
use_ex_line_cap: bool = True,
plot: bool = False) -> pypsa.Network:
"""
Load the e-highway network topology (buses and links) using PyPSA.
Parameters
----------
network: pypsa.Network
Network instance
countries: List[str] (default: None)
List of ISO codes of countries for which we want the e-highway topology
add_offshore: bool (default: True)
Whether to include offshore nodes
extend_line_cap: bool (default True)
Whether line capacity is allowed to be expanded
use_ex_line_cap: bool (default True)
Whether to use existing line capacity
plot: bool (default: False)
Whether to show loaded topology or not
Returns
-------
network: pypsa.Network
Updated network
"""
assert countries is None or len(countries) != 0, "Error: Countries list must not be empty. If you want to " \
"obtain, the full topology, don't pass anything as argument."
topology_dir = f"{data_path}topologies/e-highways/generated/"
buses_fn = f"{topology_dir}buses.csv"
assert isfile(
buses_fn), f"Error: Buses are undefined. Please run 'preprocess'."
buses = pd.read_csv(buses_fn, index_col='id')
lines_fn = f"{topology_dir}lines.csv"
assert isfile(
lines_fn), f"Error: Lines are undefined. Please run 'preprocess'."
lines = pd.read_csv(lines_fn, index_col='id')
# Remove offshore buses if not considered
if not add_offshore:
buses = buses.dropna(subset=["onshore_region"])
if countries is not None:
# In e-highway, GB is referenced as UK
iso_to_ehighway = {"GB": "UK"}
ehighway_countries = [
iso_to_ehighway[c] if c in iso_to_ehighway else c
for c in countries
]
# Remove onshore buses that are not in the considered region,
# keep also buses that are offshore (i.e. with a country name that is not a string)
def filter_buses(bus):
return (not isinstance(
bus.country, str)) or (bus.name[2:] in ehighway_countries)
buses = buses.loc[buses.apply(filter_buses, axis=1)]
else:
countries = replace_iso2_codes(
list(
set([
idx[2:]
for idx in buses.dropna(subset=["onshore_region"]).index
])))
# Converting polygons strings to Polygon object
for region_type in ["onshore_region", "offshore_region"]:
regions = buses[region_type].values
# Convert strings
for i, region in enumerate(regions):
if isinstance(region, str):
regions[i] = shapely.wkt.loads(region)
# Remove lines for which one of the two end buses has been removed
lines = pd.DataFrame(lines.loc[lines.bus0.isin(buses.index)
& lines.bus1.isin(buses.index)])
# Removing offshore buses that are not connected anymore
connected_buses = sorted(list(
set(lines["bus0"]).union(set(lines["bus1"]))))
buses = buses.loc[connected_buses]
assert len(
buses
) != 0, "Error: No buses are located in the given list of countries."
# Add offshore polygons to remaining offshore buses
if add_offshore:
offshore_shapes = get_shapes(countries, which='offshore',
save=True)["geometry"]
if len(offshore_shapes) != 0:
offshore_zones_shape = unary_union(offshore_shapes.values)
offshore_bus_indexes = buses[
buses["onshore_region"].isnull()].index
offshore_buses = buses.loc[offshore_bus_indexes]
# Use a home-made 'voronoi' partition to assign a region to each offshore bus
buses.loc[offshore_bus_indexes,
"offshore_region"] = voronoi_special(
offshore_zones_shape, offshore_buses[["x", "y"]])
# Setting line parameters
""" For DC-opf
lines['s_nom'] *= 1000.0 # PyPSA uses MW
lines['s_nom_min'] = lines['s_nom']
# Define reactance # TODO: do sth more clever
lines['x'] = pd.Series(0.00001, index=lines.index)
lines['s_nom_extendable'] = pd.Series(True, index=lines.index) # TODO: parametrize
lines['capital_cost'] = pd.Series(index=lines.index)
for idx in lines.index:
carrier = lines.loc[idx].carrier
cap_cost, _ = get_costs(carrier, sum(network.snapshot_weightings['objective']))
lines.loc[idx, ('capital_cost', )] = cap_cost * lines.length.loc[idx]
"""
lines['p_nom'] = lines["s_nom"]
if not use_ex_line_cap:
lines['p_nom'] = 0
lines['p_nom_min'] = lines['p_nom']
lines['p_min_pu'] = -1. # Making the link bi-directional
lines = lines.drop('s_nom', axis=1)
lines['p_nom_extendable'] = extend_line_cap
lines['capital_cost'] = pd.Series(index=lines.index)
for idx in lines.index:
carrier = lines.loc[idx].carrier
cap_cost, _ = get_costs(carrier,
sum(network.snapshot_weightings['objective']))
lines.loc[idx, ('capital_cost', )] = cap_cost * lines.length.loc[idx]
network.import_components_from_dataframe(buses, "Bus")
network.import_components_from_dataframe(lines, "Link")
# network.import_components_from_dataframe(lines, "Line") for dc-opf
if plot:
from epippy.topologies.core.plot import plot_topology
plot_topology(buses, lines)
plt.show()
return network
def get_topology(network: pypsa.Network,
countries: List[str] = None,
add_offshore: bool = False,
extend_line_cap: bool = True,
extension_multiplier: float = None,
use_ex_line_cap: bool = True,
plot: bool = False) -> pypsa.Network:
"""
Load the e-highway network topology (buses and links) using PyPSA.
Parameters
----------
network: pypsa.Network
Network instance
countries: List[str] (default: None)
List of ISO codes of countries for which we want the tyndp topology.
add_offshore: bool (default: False)
Whether to include offshore nodes
extend_line_cap: bool (default: True)
Whether line capacity is allowed to be expanded
extension_multiplier: float (default: 1.)
By how much the capacity can be extended if extendable
use_ex_line_cap: bool (default True)
Whether to use existing line capacity
plot: bool (default: False)
Whether to show loaded topology or not
Returns
-------
network: pypsa.Network
Updated network
"""
assert countries is None or len(countries) != 0, "Error: Countries list must not be empty. If you want to " \
"obtain, the full topology, don't pass anything as argument."
topology_dir = f"{data_path}topologies/tyndp2018/generated/"
buses_fn = f"{topology_dir}buses.csv"
assert isfile(
buses_fn), f"Error: Buses are undefined. Please run 'preprocess'."
buses = pd.read_csv(buses_fn, index_col='id')
links_fn = f"{topology_dir}links.csv"
assert isfile(
links_fn), f"Error: Links are undefined. Please run 'preprocess'."
links = pd.read_csv(links_fn, index_col='id')
# Remove offshore buses if not considered
if not add_offshore:
buses = buses.loc[buses['onshore']]
if countries is not None:
# Check if there is a bus for each country considered
missing_countries = set(countries) - set(buses.index)
assert not missing_countries, f"Error: No buses exist for the following countries: {missing_countries}"
# Remove onshore buses that are not in the considered countries, keep also buses that are offshore
def filter_buses(bus):
return not bus.onshore or bus.name in countries
buses = buses.loc[buses.apply(filter_buses, axis=1)]
countries = buses.index
# Converting polygons strings to Polygon object
regions = buses.region.values
# Convert strings
for i, region in enumerate(regions):
if isinstance(region, str):
regions[i] = shapely.wkt.loads(region)
# If we have only one bus, add it to the network and return
if len(buses) == 1:
network.import_components_from_dataframe(buses, "Bus")
return network
# Remove links for which one of the two end buses has been removed
links = pd.DataFrame(links.loc[links.bus0.isin(buses.index)
& links.bus1.isin(buses.index)])
# Removing offshore buses that are not connected anymore
connected_buses = sorted(list(
set(links["bus0"]).union(set(links["bus1"]))))
buses = buses.loc[connected_buses]
disconnected_onshore_bus = set(countries) - set(buses.index)
assert not disconnected_onshore_bus, f"Error: Buses {disconnected_onshore_bus} were disconnected."
if not use_ex_line_cap:
links['p_nom'] = 0
links['p_nom_min'] = links['p_nom']
links['p_min_pu'] = -1. # Making the link bi-directional
links['p_nom_extendable'] = extend_line_cap
if extend_line_cap and extension_multiplier is not None:
links['p_nom_max'] = links['p_nom'] * extension_multiplier
links['capital_cost'] = pd.Series(index=links.index)
for idx in links.index:
carrier = links.loc[idx].carrier
cap_cost, _ = get_costs(carrier, len(network.snapshots))
links.loc[idx, ('capital_cost', )] = cap_cost * links.length.loc[idx]
network.import_components_from_dataframe(buses, "Bus")
network.import_components_from_dataframe(links, "Link")
if plot:
from iepy.topologies.core.plot import plot_topology
plot_topology(buses, links)
plt.show()
return network