def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None,
line_length_factor=1.25, potential_mode='simple',
solver_name="cbc", algorithm="kmeans",
extended_link_costs=0, focus_weights=None):
if potential_mode == 'simple':
p_nom_max_strategy = np.sum
elif potential_mode == 'conservative':
p_nom_max_strategy = np.min
else:
raise AttributeError("potential_mode should be one of 'simple' or 'conservative', "
"but is '{}'".format(potential_mode))
clustering = get_clustering_from_busmap(
n, busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm),
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=True,
aggregate_generators_carriers=aggregate_carriers,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=line_length_factor,
generator_strategies={'p_nom_max': p_nom_max_strategy},
scale_link_capital_costs=False)
nc = clustering.network
nc.links['underwater_fraction'] = (n.links.eval('underwater_fraction * length')
.div(nc.links.length).dropna())
nc.links['capital_cost'] = (nc.links['capital_cost']
.add((nc.links.length - n.links.length)
.clip(lower=0).mul(extended_link_costs),
fill_value=0))
return clustering
def clustering_for_n_clusters(n,
n_clusters,
aggregate_carriers=None,
line_length_factor=1.25,
potential_mode='simple',
solver_name="cbc",
algorithm="kmeans"):
if potential_mode == 'simple':
p_nom_max_strategy = np.sum
elif potential_mode == 'conservative':
p_nom_max_strategy = np.min
else:
raise AttributeError(
"potential_mode should be one of 'simple' or 'conservative', "
"but is '{}'".format(potential_mode))
clustering = get_clustering_from_busmap(
n,
busmap_for_n_clusters(n, n_clusters, solver_name, algorithm),
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=True,
aggregate_generators_carriers=aggregate_carriers,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=line_length_factor,
generator_strategies={'p_nom_max': p_nom_max_strategy})
return clustering
def clustering_for_n_clusters(n,
n_clusters,
custom_busmap=False,
aggregate_carriers=None,
line_length_factor=1.25,
potential_mode='simple',
solver_name="cbc",
algorithm="kmeans",
extended_link_costs=0,
focus_weights=None):
if potential_mode == 'simple':
p_nom_max_strategy = np.sum
elif potential_mode == 'conservative':
p_nom_max_strategy = np.min
else:
raise AttributeError(
f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'"
)
if custom_busmap:
busmap = pd.read_csv(snakemake.input.custom_busmap,
index_col=0,
squeeze=True)
busmap.index = busmap.index.astype(str)
logger.info(
f"Imported custom busmap from {snakemake.input.custom_busmap}")
else:
busmap = busmap_for_n_clusters(n, n_clusters, solver_name,
focus_weights, algorithm)
clustering = get_clustering_from_busmap(
n,
busmap,
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=True,
aggregate_generators_carriers=aggregate_carriers,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=line_length_factor,
generator_strategies={'p_nom_max': p_nom_max_strategy},
scale_link_capital_costs=False)
if not n.links.empty:
nc = clustering.network
nc.links['underwater_fraction'] = (
n.links.eval('underwater_fraction * length').div(
nc.links.length).dropna())
nc.links['capital_cost'] = (nc.links['capital_cost'].add(
(nc.links.length -
n.links.length).clip(lower=0).mul(extended_link_costs),
fill_value=0))
return clustering
def clustering_for_n_clusters(n,
n_clusters,
aggregate_renewables=True,
line_length_factor=1.25):
aggregate_generators_carriers = (None if aggregate_renewables else (
pd.Index(n.generators.carrier.unique()).difference(
['onwind', 'offwind', 'solar'])))
clustering = get_clustering_from_busmap(
n,
busmap_for_n_clusters(n, n_clusters),
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=True,
aggregate_generators_carriers=aggregate_generators_carriers,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=line_length_factor)
return clustering
def aggregate_to_substations(n, buses_i=None):
# can be used to aggregate a selection of buses to electrically closest neighbors
# if no buses are given, nodes that are no substations or without offshore connection are aggregated
if buses_i is None:
logger.info(
"Aggregating buses that are no substations or have no valid offshore connection"
)
buses_i = list(
set(n.buses.index) - set(n.generators.bus) - set(n.loads.bus))
weight = pd.concat({
'Line': n.lines.length / n.lines.s_nom.clip(1e-3),
'Link': n.links.length / n.links.p_nom.clip(1e-3)
})
adj = n.adjacency_matrix(branch_components=['Line', 'Link'],
weights=weight)
bus_indexer = n.buses.index.get_indexer(buses_i)
dist = pd.DataFrame(dijkstra(adj, directed=False, indices=bus_indexer),
buses_i, n.buses.index)
dist[
buses_i] = np.inf # bus in buses_i should not be assigned to different bus in buses_i
for c in n.buses.country.unique():
incountry_b = n.buses.country == c
dist.loc[incountry_b, ~incountry_b] = np.inf
busmap = n.buses.index.to_series()
busmap.loc[buses_i] = dist.idxmin(1)
clustering = get_clustering_from_busmap(
n,
busmap,
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=True,
aggregate_generators_carriers=None,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=1.0,
generator_strategies={'p_nom_max': 'sum'},
scale_link_capital_costs=False)
return clustering.network, busmap
def kmean_clustering(network, n_clusters=10):
"""
Implement k-mean clustering in existing network
Parameters
----------
network : :class:`pypsa.Network
Overall container of PyPSA
Returns
-------
network : pypsa.Network object
Container for all network components.
"""
def weighting_for_scenario(x):
b_i = x.index
g = normed(gen.reindex(b_i, fill_value=0))
l = normed(load.reindex(b_i, fill_value=0))
w = g + l
return (w * (100000. / w.max())).astype(int)
def normed(x):
return (x / x.sum()).fillna(0.)
print('start k-mean clustering')
# prepare k-mean
# k-means clustering (first try)
network.generators.control = "PV"
network.buses['v_nom'] = 380.
# problem our lines have no v_nom. this is implicitly defined by the connected buses:
network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom)
# adjust the x of the lines which are not 380.
lines_v_nom_b = network.lines.v_nom != 380
network.lines.loc[lines_v_nom_b,
'x'] *= (380. /
network.lines.loc[lines_v_nom_b, 'v_nom'])**2
network.lines.loc[lines_v_nom_b, 'v_nom'] = 380.
trafo_index = network.transformers.index
transformer_voltages = pd.concat([
network.transformers.bus0.map(network.buses.v_nom),
network.transformers.bus1.map(network.buses.v_nom)
],
axis=1)
network.import_components_from_dataframe(
network.transformers.loc[:, ['bus0', 'bus1', 'x', 's_nom']].assign(
x=network.transformers.x *
(380. /
transformer_voltages.max(axis=1))**2).set_index('T' +
trafo_index),
'Line')
network.transformers.drop(trafo_index, inplace=True)
for attr in network.transformers_t:
network.transformers_t[attr] = network.transformers_t[attr].reindex(
columns=[])
#define weighting based on conventional 'old' generator spatial distribution
non_conv_types = {
'biomass', 'wind', 'solar', 'geothermal', 'load shedding',
'extendable_storage'
}
# Attention: network.generators.carrier.unique()
gen = (
network.generators.loc[(network.generators.carrier.isin(non_conv_types)
== False)].groupby('bus').p_nom.sum().reindex(
network.buses.index, fill_value=0.) +
network.storage_units.loc[
(network.storage_units.carrier.isin(non_conv_types)
== False)].groupby('bus').p_nom.sum().reindex(network.buses.index,
fill_value=0.))
load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum()
# k-mean clustering
# busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1,
# len(network.buses)), index=network.buses.index) , n_clusters= 10)
weight = weighting_for_scenario(network.buses).reindex(network.buses.index,
fill_value=1)
busmap = busmap_by_kmeans(network,
bus_weightings=pd.Series(weight),
n_clusters=n_clusters)
# ToDo change function in order to use bus_strategies or similar
network.generators['weight'] = 1
aggregate_one_ports = components.one_port_components.copy()
aggregate_one_ports.discard('Generator')
clustering = get_clustering_from_busmap(
network,
busmap,
aggregate_generators_weighted=True,
aggregate_one_ports=aggregate_one_ports)
network = clustering.network
#network = cluster_on_extra_high_voltage(network, busmap, with_time=True)
return network
def kmean_clustering(network,
n_clusters=10,
load_cluster=False,
line_length_factor=1.25,
remove_stubs=False,
use_reduced_coordinates=False,
bus_weight_tocsv=None,
bus_weight_fromcsv=None,
n_init=10,
max_iter=300,
tol=1e-4,
n_jobs=1):
""" Main function of the k-mean clustering approach. Maps an original
network to a new one with adjustable number of nodes and new coordinates.
Parameters
----------
network : :class:`pypsa.Network
Container for all network components.
n_clusters : int
Desired number of clusters.
load_cluster : boolean
Loads cluster coordinates from a former calculation.
line_length_factor : float
Factor to multiply the crow-flies distance between new buses in order
to get new line lengths.
remove_stubs: boolean
Removes stubs and stubby trees (i.e. sequentially reducing dead-ends).
use_reduced_coordinates: boolean
If True, do not average cluster coordinates, but take from busmap.
bus_weight_tocsv : str
Creates a bus weighting based on conventional generation and load
and save it to a csv file.
bus_weight_fromcsv : str
Loads a bus weighting from a csv file to apply it to the clustering
algorithm.
Returns
-------
network : pypsa.Network object
Container for all network components.
"""
def weighting_for_scenario(x, save=None):
"""
"""
b_i = x.index
g = normed(gen.reindex(b_i, fill_value=0))
l = normed(load.reindex(b_i, fill_value=0))
w = g + l
weight = ((w * (100000. / w.max())).astype(int)).reindex(
network.buses.index, fill_value=1)
if save:
weight.to_csv(save)
return weight
def normed(x):
return (x / x.sum()).fillna(0.)
print('start k-mean clustering')
# prepare k-mean
# k-means clustering (first try)
network.generators.control = "PV"
network.storage_units.control[network.storage_units.carrier == \
'extendable_storage'] = "PV"
network.buses['v_nom'] = 380.
# problem our lines have no v_nom. this is implicitly defined by the
# connected buses:
network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom)
# adjust the x of the lines which are not 380.
lines_v_nom_b = network.lines.v_nom != 380
network.lines.loc[lines_v_nom_b, 'x'] *= \
(380. / network.lines.loc[lines_v_nom_b, 'v_nom'])**2
network.lines.loc[lines_v_nom_b, 'v_nom'] = 380.
trafo_index = network.transformers.index
transformer_voltages = \
pd.concat([network.transformers.bus0.map(network.buses.v_nom),
network.transformers.bus1.map(network.buses.v_nom)], axis=1)
network.import_components_from_dataframe(
network.transformers.
loc[:, ['bus0', 'bus1', 'x', 's_nom', 'capital_cost']].assign(
x=network.transformers.x *
(380. / transformer_voltages.max(axis=1))**2,
length=1).set_index('T' + trafo_index), 'Line')
network.transformers.drop(trafo_index, inplace=True)
for attr in network.transformers_t:
network.transformers_t[attr] = network.transformers_t[attr]\
.reindex(columns=[])
# remove stubs
if remove_stubs:
network.determine_network_topology()
busmap = busmap_by_stubs(network)
network.generators['weight'] = network.generators['p_nom']
aggregate_one_ports = components.one_port_components.copy()
aggregate_one_ports.discard('Generator')
# reset coordinates to the new reduced guys, rather than taking an
# average (copied from pypsa.networkclustering)
if use_reduced_coordinates:
# TODO : FIX THIS HACK THAT HAS UNEXPECTED SIDE-EFFECTS,
# i.e. network is changed in place!!
network.buses.loc[busmap.index, ['x', 'y']] = network.buses.loc[
busmap, ['x', 'y']].values
clustering = get_clustering_from_busmap(
network,
busmap,
aggregate_generators_weighted=True,
aggregate_one_ports=aggregate_one_ports,
line_length_factor=line_length_factor)
network = clustering.network
# define weighting based on conventional 'old' generator spatial
# distribution
non_conv_types = {
'biomass', 'wind_onshore', 'wind_offshore', 'solar', 'geothermal',
'load shedding', 'extendable_storage'
}
# Attention: network.generators.carrier.unique()
gen = (
network.generators.loc[(network.generators.carrier.isin(non_conv_types)
== False)].groupby('bus').p_nom.sum().reindex(
network.buses.index, fill_value=0.) +
network.storage_units.loc[
(network.storage_units.carrier.isin(non_conv_types)
== False)].groupby('bus').p_nom.sum().reindex(network.buses.index,
fill_value=0.))
load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum()
# k-mean clustering
# busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1,
# len(network.buses)), index=network.buses.index) , n_clusters= 10)
# State whether to create a bus weighting and save it, create or not save
# it, or use a bus weighting from a csv file
if bus_weight_tocsv is not None:
weight = weighting_for_scenario(x=network.buses, save=bus_weight_tocsv)
elif bus_weight_fromcsv is not None:
weight = pd.Series.from_csv(bus_weight_fromcsv)
weight.index = weight.index.astype(str)
else:
weight = weighting_for_scenario(x=network.buses, save=False)
busmap = busmap_by_kmeans(network,
bus_weightings=pd.Series(weight),
n_clusters=n_clusters,
load_cluster=load_cluster,
n_init=n_init,
max_iter=max_iter,
tol=tol,
n_jobs=n_jobs)
# ToDo change function in order to use bus_strategies or similar
network.generators['weight'] = network.generators['p_nom']
aggregate_one_ports = components.one_port_components.copy()
aggregate_one_ports.discard('Generator')
clustering = get_clustering_from_busmap(
network,
busmap,
aggregate_generators_weighted=True,
aggregate_one_ports=aggregate_one_ports)
return clustering
def clustering_for_n_clusters(
n,
n_clusters,
custom_busmap=False,
aggregate_carriers=None,
line_length_factor=1.25,
potential_mode="simple",
solver_name="cbc",
algorithm="kmeans",
extended_link_costs=0,
focus_weights=None,
):
if potential_mode == "simple":
p_nom_max_strategy = np.sum
elif potential_mode == "conservative":
p_nom_max_strategy = np.min
else:
raise AttributeError(
f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'"
)
if custom_busmap:
busmap = pd.read_csv(snakemake.input.custom_busmap,
index_col=0,
squeeze=True)
busmap.index = busmap.index.astype(str)
logger.info(
f"Imported custom busmap from {snakemake.input.custom_busmap}")
else:
if snakemake.config["cluster_options"]["alternative_clustering"]:
n, busmap = busmap_for_gadm_clusters(
n, snakemake.config["build_shape_options"]["gadm_layer_id"]
) # TODO make func only return busmap, and get level from config
else:
busmap = busmap_for_n_clusters(n, n_clusters, solver_name,
focus_weights, algorithm)
weighted_agg_gens = True
clustering = get_clustering_from_busmap(
n,
busmap,
bus_strategies=dict(country=_make_consense("Bus", "country")),
aggregate_generators_weighted=weighted_agg_gens,
aggregate_generators_carriers=aggregate_carriers,
aggregate_one_ports=["Load", "StorageUnit"],
line_length_factor=line_length_factor,
generator_strategies={
"p_nom_max": p_nom_max_strategy,
"p_nom_min": np.sum
},
scale_link_capital_costs=False,
)
if not n.links.empty:
nc = clustering.network
nc.links["underwater_fraction"] = (
n.links.eval("underwater_fraction * length").div(
nc.links.length).dropna())
nc.links["capital_cost"] = nc.links["capital_cost"].add(
(nc.links.length -
n.links.length).clip(lower=0).mul(extended_link_costs),
fill_value=0,
)
return clustering
def kmean_clustering(network):
""" Implement k-mean clustering in existing network
----------
network : :class:`pypsa.Network
Overall container of PyPSA
Returns
-------
"""
def weighting_for_scenario(x):
b_i = x.index
g = normed(gen.reindex(b_i, fill_value=0))
l = normed(load.reindex(b_i, fill_value=0))
w = g + l
return (w * (100. / w.max())).astype(int)
def normed(x):
return (x / x.sum()).fillna(0.)
print('start k-mean clustering')
# prepare k-mean
# k-means clustering (first try)
network.generators.control = "PV"
network.buses['v_nom'] = 380.
# problem our lines have no v_nom. this is implicitly defined by the connected buses:
network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom)
# adjust the x of the lines which are not 380.
lines_v_nom_b = network.lines.v_nom != 380
network.lines.loc[lines_v_nom_b,
'x'] *= (380. /
network.lines.loc[lines_v_nom_b, 'v_nom'])**2
network.lines.loc[lines_v_nom_b, 'v_nom'] = 380.
trafo_index = network.transformers.index
transformer_voltages = pd.concat([
network.transformers.bus0.map(network.buses.v_nom),
network.transformers.bus1.map(network.buses.v_nom)
],
axis=1)
network.import_components_from_dataframe(
network.transformers.loc[:, ['bus0', 'bus1', 'x', 's_nom']].assign(
x=network.transformers.x *
(380. /
transformer_voltages.max(axis=1))**2).set_index('T' +
trafo_index),
'Line')
network.transformers.drop(trafo_index, inplace=True)
for attr in network.transformers_t:
network.transformers_t[attr] = network.transformers_t[attr].reindex(
columns=[])
#ToDo: change conv to types minus wind and solar
conv_types = {
'biomass', 'run_of_river', 'gas', 'oil', 'coal', 'waste', 'uranium'
}
# Attention: network.generators.carrier.unique()
# conv_types only for SH scenario defined!
gen = (network.generators.loc[network.generators.carrier.isin(
conv_types)].groupby('bus').p_nom.sum().reindex(network.buses.index,
fill_value=0.) +
network.storage_units.loc[network.storage_units.carrier.isin(
conv_types)].groupby('bus').p_nom.sum().reindex(
network.buses.index, fill_value=0.))
load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum()
# k-mean clustering
# busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1,
# len(network.buses)), index=network.buses.index) , n_clusters= 10)
weight = weighting_for_scenario(network.buses).reindex(network.buses.index,
fill_value=1)
busmap = busmap_by_kmeans(network,
bus_weightings=pd.Series(weight),
buses_i=network.buses.index,
n_clusters=10)
# ToDo change function in order to use bus_strategies or similar
clustering = get_clustering_from_busmap(network, busmap)
network = clustering.network
#network = cluster_on_extra_high_voltage(network, busmap, with_time=True)
return network