def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): if potential_mode == 'simple': p_nom_max_strategy = np.sum elif potential_mode == 'conservative': p_nom_max_strategy = np.min else: raise AttributeError("potential_mode should be one of 'simple' or 'conservative', " "but is '{}'".format(potential_mode)) clustering = get_clustering_from_busmap( n, busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm), bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, generator_strategies={'p_nom_max': p_nom_max_strategy}, scale_link_capital_costs=False) nc = clustering.network nc.links['underwater_fraction'] = (n.links.eval('underwater_fraction * length') .div(nc.links.length).dropna()) nc.links['capital_cost'] = (nc.links['capital_cost'] .add((nc.links.length - n.links.length) .clip(lower=0).mul(extended_link_costs), fill_value=0)) return clustering
def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', solver_name="cbc", algorithm="kmeans"): if potential_mode == 'simple': p_nom_max_strategy = np.sum elif potential_mode == 'conservative': p_nom_max_strategy = np.min else: raise AttributeError( "potential_mode should be one of 'simple' or 'conservative', " "but is '{}'".format(potential_mode)) clustering = get_clustering_from_busmap( n, busmap_for_n_clusters(n, n_clusters, solver_name, algorithm), bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, generator_strategies={'p_nom_max': p_nom_max_strategy}) return clustering
def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): if potential_mode == 'simple': p_nom_max_strategy = np.sum elif potential_mode == 'conservative': p_nom_max_strategy = np.min else: raise AttributeError( f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'" ) if custom_busmap: busmap = pd.read_csv(snakemake.input.custom_busmap, index_col=0, squeeze=True) busmap.index = busmap.index.astype(str) logger.info( f"Imported custom busmap from {snakemake.input.custom_busmap}") else: busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) clustering = get_clustering_from_busmap( n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, generator_strategies={'p_nom_max': p_nom_max_strategy}, scale_link_capital_costs=False) if not n.links.empty: nc = clustering.network nc.links['underwater_fraction'] = ( n.links.eval('underwater_fraction * length').div( nc.links.length).dropna()) nc.links['capital_cost'] = (nc.links['capital_cost'].add( (nc.links.length - n.links.length).clip(lower=0).mul(extended_link_costs), fill_value=0)) return clustering
def clustering_for_n_clusters(n, n_clusters, aggregate_renewables=True, line_length_factor=1.25): aggregate_generators_carriers = (None if aggregate_renewables else ( pd.Index(n.generators.carrier.unique()).difference( ['onwind', 'offwind', 'solar']))) clustering = get_clustering_from_busmap( n, busmap_for_n_clusters(n, n_clusters), bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_generators_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor) return clustering
def aggregate_to_substations(n, buses_i=None): # can be used to aggregate a selection of buses to electrically closest neighbors # if no buses are given, nodes that are no substations or without offshore connection are aggregated if buses_i is None: logger.info( "Aggregating buses that are no substations or have no valid offshore connection" ) buses_i = list( set(n.buses.index) - set(n.generators.bus) - set(n.loads.bus)) weight = pd.concat({ 'Line': n.lines.length / n.lines.s_nom.clip(1e-3), 'Link': n.links.length / n.links.p_nom.clip(1e-3) }) adj = n.adjacency_matrix(branch_components=['Line', 'Link'], weights=weight) bus_indexer = n.buses.index.get_indexer(buses_i) dist = pd.DataFrame(dijkstra(adj, directed=False, indices=bus_indexer), buses_i, n.buses.index) dist[ buses_i] = np.inf # bus in buses_i should not be assigned to different bus in buses_i for c in n.buses.country.unique(): incountry_b = n.buses.country == c dist.loc[incountry_b, ~incountry_b] = np.inf busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) clustering = get_clustering_from_busmap( n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, generator_strategies={'p_nom_max': 'sum'}, scale_link_capital_costs=False) return clustering.network, busmap
def kmean_clustering(network, n_clusters=10): """ Implement k-mean clustering in existing network Parameters ---------- network : :class:`pypsa.Network Overall container of PyPSA Returns ------- network : pypsa.Network object Container for all network components. """ def weighting_for_scenario(x): b_i = x.index g = normed(gen.reindex(b_i, fill_value=0)) l = normed(load.reindex(b_i, fill_value=0)) w = g + l return (w * (100000. / w.max())).astype(int) def normed(x): return (x / x.sum()).fillna(0.) print('start k-mean clustering') # prepare k-mean # k-means clustering (first try) network.generators.control = "PV" network.buses['v_nom'] = 380. # problem our lines have no v_nom. this is implicitly defined by the connected buses: network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) # adjust the x of the lines which are not 380. lines_v_nom_b = network.lines.v_nom != 380 network.lines.loc[lines_v_nom_b, 'x'] *= (380. / network.lines.loc[lines_v_nom_b, 'v_nom'])**2 network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. trafo_index = network.transformers.index transformer_voltages = pd.concat([ network.transformers.bus0.map(network.buses.v_nom), network.transformers.bus1.map(network.buses.v_nom) ], axis=1) network.import_components_from_dataframe( network.transformers.loc[:, ['bus0', 'bus1', 'x', 's_nom']].assign( x=network.transformers.x * (380. / transformer_voltages.max(axis=1))**2).set_index('T' + trafo_index), 'Line') network.transformers.drop(trafo_index, inplace=True) for attr in network.transformers_t: network.transformers_t[attr] = network.transformers_t[attr].reindex( columns=[]) #define weighting based on conventional 'old' generator spatial distribution non_conv_types = { 'biomass', 'wind', 'solar', 'geothermal', 'load shedding', 'extendable_storage' } # Attention: network.generators.carrier.unique() gen = ( network.generators.loc[(network.generators.carrier.isin(non_conv_types) == False)].groupby('bus').p_nom.sum().reindex( network.buses.index, fill_value=0.) + network.storage_units.loc[ (network.storage_units.carrier.isin(non_conv_types) == False)].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.)) load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() # k-mean clustering # busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, # len(network.buses)), index=network.buses.index) , n_clusters= 10) weight = weighting_for_scenario(network.buses).reindex(network.buses.index, fill_value=1) busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(weight), n_clusters=n_clusters) # ToDo change function in order to use bus_strategies or similar network.generators['weight'] = 1 aggregate_one_ports = components.one_port_components.copy() aggregate_one_ports.discard('Generator') clustering = get_clustering_from_busmap( network, busmap, aggregate_generators_weighted=True, aggregate_one_ports=aggregate_one_ports) network = clustering.network #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) return network
def kmean_clustering(network, n_clusters=10, load_cluster=False, line_length_factor=1.25, remove_stubs=False, use_reduced_coordinates=False, bus_weight_tocsv=None, bus_weight_fromcsv=None, n_init=10, max_iter=300, tol=1e-4, n_jobs=1): """ Main function of the k-mean clustering approach. Maps an original network to a new one with adjustable number of nodes and new coordinates. Parameters ---------- network : :class:`pypsa.Network Container for all network components. n_clusters : int Desired number of clusters. load_cluster : boolean Loads cluster coordinates from a former calculation. line_length_factor : float Factor to multiply the crow-flies distance between new buses in order to get new line lengths. remove_stubs: boolean Removes stubs and stubby trees (i.e. sequentially reducing dead-ends). use_reduced_coordinates: boolean If True, do not average cluster coordinates, but take from busmap. bus_weight_tocsv : str Creates a bus weighting based on conventional generation and load and save it to a csv file. bus_weight_fromcsv : str Loads a bus weighting from a csv file to apply it to the clustering algorithm. Returns ------- network : pypsa.Network object Container for all network components. """ def weighting_for_scenario(x, save=None): """ """ b_i = x.index g = normed(gen.reindex(b_i, fill_value=0)) l = normed(load.reindex(b_i, fill_value=0)) w = g + l weight = ((w * (100000. / w.max())).astype(int)).reindex( network.buses.index, fill_value=1) if save: weight.to_csv(save) return weight def normed(x): return (x / x.sum()).fillna(0.) print('start k-mean clustering') # prepare k-mean # k-means clustering (first try) network.generators.control = "PV" network.storage_units.control[network.storage_units.carrier == \ 'extendable_storage'] = "PV" network.buses['v_nom'] = 380. # problem our lines have no v_nom. this is implicitly defined by the # connected buses: network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) # adjust the x of the lines which are not 380. lines_v_nom_b = network.lines.v_nom != 380 network.lines.loc[lines_v_nom_b, 'x'] *= \ (380. / network.lines.loc[lines_v_nom_b, 'v_nom'])**2 network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. trafo_index = network.transformers.index transformer_voltages = \ pd.concat([network.transformers.bus0.map(network.buses.v_nom), network.transformers.bus1.map(network.buses.v_nom)], axis=1) network.import_components_from_dataframe( network.transformers. loc[:, ['bus0', 'bus1', 'x', 's_nom', 'capital_cost']].assign( x=network.transformers.x * (380. / transformer_voltages.max(axis=1))**2, length=1).set_index('T' + trafo_index), 'Line') network.transformers.drop(trafo_index, inplace=True) for attr in network.transformers_t: network.transformers_t[attr] = network.transformers_t[attr]\ .reindex(columns=[]) # remove stubs if remove_stubs: network.determine_network_topology() busmap = busmap_by_stubs(network) network.generators['weight'] = network.generators['p_nom'] aggregate_one_ports = components.one_port_components.copy() aggregate_one_ports.discard('Generator') # reset coordinates to the new reduced guys, rather than taking an # average (copied from pypsa.networkclustering) if use_reduced_coordinates: # TODO : FIX THIS HACK THAT HAS UNEXPECTED SIDE-EFFECTS, # i.e. network is changed in place!! network.buses.loc[busmap.index, ['x', 'y']] = network.buses.loc[ busmap, ['x', 'y']].values clustering = get_clustering_from_busmap( network, busmap, aggregate_generators_weighted=True, aggregate_one_ports=aggregate_one_ports, line_length_factor=line_length_factor) network = clustering.network # define weighting based on conventional 'old' generator spatial # distribution non_conv_types = { 'biomass', 'wind_onshore', 'wind_offshore', 'solar', 'geothermal', 'load shedding', 'extendable_storage' } # Attention: network.generators.carrier.unique() gen = ( network.generators.loc[(network.generators.carrier.isin(non_conv_types) == False)].groupby('bus').p_nom.sum().reindex( network.buses.index, fill_value=0.) + network.storage_units.loc[ (network.storage_units.carrier.isin(non_conv_types) == False)].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.)) load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() # k-mean clustering # busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, # len(network.buses)), index=network.buses.index) , n_clusters= 10) # State whether to create a bus weighting and save it, create or not save # it, or use a bus weighting from a csv file if bus_weight_tocsv is not None: weight = weighting_for_scenario(x=network.buses, save=bus_weight_tocsv) elif bus_weight_fromcsv is not None: weight = pd.Series.from_csv(bus_weight_fromcsv) weight.index = weight.index.astype(str) else: weight = weighting_for_scenario(x=network.buses, save=False) busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(weight), n_clusters=n_clusters, load_cluster=load_cluster, n_init=n_init, max_iter=max_iter, tol=tol, n_jobs=n_jobs) # ToDo change function in order to use bus_strategies or similar network.generators['weight'] = network.generators['p_nom'] aggregate_one_ports = components.one_port_components.copy() aggregate_one_ports.discard('Generator') clustering = get_clustering_from_busmap( network, busmap, aggregate_generators_weighted=True, aggregate_one_ports=aggregate_one_ports) return clustering
def clustering_for_n_clusters( n, n_clusters, custom_busmap=False, aggregate_carriers=None, line_length_factor=1.25, potential_mode="simple", solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None, ): if potential_mode == "simple": p_nom_max_strategy = np.sum elif potential_mode == "conservative": p_nom_max_strategy = np.min else: raise AttributeError( f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'" ) if custom_busmap: busmap = pd.read_csv(snakemake.input.custom_busmap, index_col=0, squeeze=True) busmap.index = busmap.index.astype(str) logger.info( f"Imported custom busmap from {snakemake.input.custom_busmap}") else: if snakemake.config["cluster_options"]["alternative_clustering"]: n, busmap = busmap_for_gadm_clusters( n, snakemake.config["build_shape_options"]["gadm_layer_id"] ) # TODO make func only return busmap, and get level from config else: busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) weighted_agg_gens = True clustering = get_clustering_from_busmap( n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=weighted_agg_gens, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, generator_strategies={ "p_nom_max": p_nom_max_strategy, "p_nom_min": np.sum }, scale_link_capital_costs=False, ) if not n.links.empty: nc = clustering.network nc.links["underwater_fraction"] = ( n.links.eval("underwater_fraction * length").div( nc.links.length).dropna()) nc.links["capital_cost"] = nc.links["capital_cost"].add( (nc.links.length - n.links.length).clip(lower=0).mul(extended_link_costs), fill_value=0, ) return clustering
def kmean_clustering(network): """ Implement k-mean clustering in existing network ---------- network : :class:`pypsa.Network Overall container of PyPSA Returns ------- """ def weighting_for_scenario(x): b_i = x.index g = normed(gen.reindex(b_i, fill_value=0)) l = normed(load.reindex(b_i, fill_value=0)) w = g + l return (w * (100. / w.max())).astype(int) def normed(x): return (x / x.sum()).fillna(0.) print('start k-mean clustering') # prepare k-mean # k-means clustering (first try) network.generators.control = "PV" network.buses['v_nom'] = 380. # problem our lines have no v_nom. this is implicitly defined by the connected buses: network.lines["v_nom"] = network.lines.bus0.map(network.buses.v_nom) # adjust the x of the lines which are not 380. lines_v_nom_b = network.lines.v_nom != 380 network.lines.loc[lines_v_nom_b, 'x'] *= (380. / network.lines.loc[lines_v_nom_b, 'v_nom'])**2 network.lines.loc[lines_v_nom_b, 'v_nom'] = 380. trafo_index = network.transformers.index transformer_voltages = pd.concat([ network.transformers.bus0.map(network.buses.v_nom), network.transformers.bus1.map(network.buses.v_nom) ], axis=1) network.import_components_from_dataframe( network.transformers.loc[:, ['bus0', 'bus1', 'x', 's_nom']].assign( x=network.transformers.x * (380. / transformer_voltages.max(axis=1))**2).set_index('T' + trafo_index), 'Line') network.transformers.drop(trafo_index, inplace=True) for attr in network.transformers_t: network.transformers_t[attr] = network.transformers_t[attr].reindex( columns=[]) #ToDo: change conv to types minus wind and solar conv_types = { 'biomass', 'run_of_river', 'gas', 'oil', 'coal', 'waste', 'uranium' } # Attention: network.generators.carrier.unique() # conv_types only for SH scenario defined! gen = (network.generators.loc[network.generators.carrier.isin( conv_types)].groupby('bus').p_nom.sum().reindex(network.buses.index, fill_value=0.) + network.storage_units.loc[network.storage_units.carrier.isin( conv_types)].groupby('bus').p_nom.sum().reindex( network.buses.index, fill_value=0.)) load = network.loads_t.p_set.mean().groupby(network.loads.bus).sum() # k-mean clustering # busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(np.repeat(1, # len(network.buses)), index=network.buses.index) , n_clusters= 10) weight = weighting_for_scenario(network.buses).reindex(network.buses.index, fill_value=1) busmap = busmap_by_kmeans(network, bus_weightings=pd.Series(weight), buses_i=network.buses.index, n_clusters=10) # ToDo change function in order to use bus_strategies or similar clustering = get_clustering_from_busmap(network, busmap) network = clustering.network #network = cluster_on_extra_high_voltage(network, busmap, with_time=True) return network