def electricity(buses, datapackage_dir, raw_data_path): """ Parameters ---------- buses: dict Dictionary with two keys: decentral and central and their values beeing the names of the buses datapackage_dir: str Directory of datapackage where resources are stored raw_data_path: str Path to directory where raw data can be found """ filepath = building.download_data( "http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/" "NUTS_2013_10M_SH.zip", unzip_file="NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.shp", directory=raw_data_path, ) building.download_data( "http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/" "NUTS_2013_10M_SH.zip", unzip_file="NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.dbf", directory=raw_data_path, ) if not os.path.exists(filepath): print("Shapefile data not found. Did you download raw data?") # get nuts 1 regions for german neighbours nuts0 = pd.Series(geometry.nuts(filepath, nuts=0, tolerance=0.1)) nuts0.index = [i.replace("UK", "GB") for i in nuts0.index] el_buses = pd.Series(name="geometry") el_buses.index.name = "name" for r in buses: el_buses[r + "-electricity"] = nuts0[r] building.write_geometries( "buses.csv", el_buses, os.path.join(datapackage_dir, "data", "geometries"), ) # Add electricity buses bus_elements = {} for b in el_buses.index: bus_elements[b] = { "type": "bus", "carrier": "electricity", "geometry": b, "balanced": True, } building.write_elements( "bus.csv", pd.DataFrame.from_dict(bus_elements, orient="index"), os.path.join(datapackage_dir, "data", "elements"), )
def ninja_pv_profiles(buses, weather_year, scenario_year, datapackage_dir, raw_data_path): """ Parameter --------- buses: array like List with buses represented by iso country code weather_year: integer or string Year to select from raw data source scenario_year: integer or string Year to use for timeindex in tabular resource datapackage_dir: string Directory for tabular resource raw_data_path: string Path where raw data file `ninja_pv_europe_v1.1_merra2.csv` is located """ filepath = building.download_data( "https://www.renewables.ninja/static/downloads/ninja_europe_pv_v1.1.zip", unzip_file="ninja_pv_europe_v1.1_merra2.csv", directory=raw_data_path, ) year = str(weather_year) countries = buses raw_data = pd.read_csv(filepath, index_col=[0], parse_dates=True) # for leap year... raw_data = raw_data[~((raw_data.index.month == 2) & (raw_data.index.day == 29))] df = raw_data.loc[year] sequences_df = pd.DataFrame(index=df.index) for c in countries: sequence_name = c + "-pv-profile" sequences_df[sequence_name] = raw_data.loc[year][c].values sequences_df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", sequences_df, directory=os.path.join(datapackage_dir, "data", "sequences"), )
def ninja_offshore_wind_profiles(buses, weather_year, scenario_year, datapackage_dir, raw_data_path): """ Parameter --------- buses: array like List with buses represented by iso country code weather_year: integer or string Year to select from raw data source scenario_year: integer or string Year to use for timeindex in tabular resource datapackage_dir: string Directory for tabular resource raw_data_path: string Path where raw data file `ninja_wind_europe_v1.1_current_national.csv` and `ninja_wind_europe_v1.1_current_national.csv` is located """ onoff_filepath = building.download_data( "https://www.renewables.ninja/static/downloads/ninja_europe_wind_v1.1.zip", unzip_file="ninja_wind_europe_v1.1_future_nearterm_on-offshore.csv", directory=raw_data_path, ) year = str(weather_year) on_off_data = pd.read_csv(onoff_filepath, index_col=[0], parse_dates=True) on_off_data = on_off_data[~((on_off_data.index.month == 2) & (on_off_data.index.day == 29))] sequences_df = pd.DataFrame(index=on_off_data.loc[year].index) for c in buses: if c + "_OFF" in on_off_data.columns: sequences_df[c + "-offshore-profile"] = on_off_data[c + "_OFF"] elif c == "PL": sequences_df[c + "-offshore-profile"] = on_off_data["SE_OFF"] sequences_df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", sequences_df, directory=os.path.join(datapackage_dir, "data", "sequences"), )
def emhires_pv_profiles(buses, weather_year, scenario_year, datapackage_dir, raw_data_path): """ Gonzalez Aparicio, Iratxe (2017): Solar hourly generation time series at country, NUTS 1, NUTS 2 level and bidding zones. European Commission, Joint Research Centre (JRC) [Dataset] PID: http://data.europa.eu/89h/jrc-emhires-solar-generation-time-series EU Commission, DG ENER, Unit A4 - ENERGY STATISTICS, https://ec.europa.eu/energy/sites/ener/files/documents/countrydatasheets_june2018.xlsx """ year = str(weather_year) countries = buses date_parser = lambda y: datetime.strptime(y, "%Y %m %d %H") date_columns = ["Year", "Month", "Day", "Hour"] df = (pd.read_excel( building.download_data( "https://setis.ec.europa.eu/sites/default/files/EMHIRES_DATA/Solar/EMHIRESPV_country_level.zip", unzip_file="EMHIRESPV_TSh_CF_Country_19862015.xlsx", directory=raw_data_path, ), parse_dates={ "i": date_columns }, date_parser=date_parser, index_col="i", ).reindex(columns=countries).dropna(axis=1).loc[year, countries]) renames = {c: c + "-pv-profile" for c in countries} df.rename(columns=renames, inplace=True) df = df[~((df.index.month == 2) & (df.index.day == 29))] df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", df, directory=os.path.join(datapackage_dir, "data", "sequences"), )
def tyndp(buses, scenario, datapackage_dir, raw_data_path, sensitivities): """ """ filepath = building.download_data( "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP2018/" "Scenarios%20Data%20Sets/Input%20Data.xlsx", directory=raw_data_path, sensitivities=None, ) df = pd.read_excel(filepath, sheet_name="Demand") df["countries"] = [i[0:2] for i in df.index] # for aggregation by country elements = df.groupby("countries").sum()[scenario].to_frame() elements.index.name = "bus" elements = elements.loc[buses] elements.reset_index(inplace=True) elements["name"] = elements.apply( lambda row: row.bus + "-electricity-load", axis=1) elements["profile"] = elements.apply( lambda row: row.bus + "-electricity-load-profile", axis=1) elements["type"] = "load" elements["carrier"] = "electricity" elements.set_index("name", inplace=True) elements.bus = [b + "-electricity" for b in elements.bus] elements["amount"] = elements[scenario] * 1000 # MWh -> GWh if sensitivities is not None: for k in sensitivities: if "load" in k: elements.loc[k, "amount"] = sensitivities[k] * 1000 building.write_elements( "load.csv", elements, directory=os.path.join(datapackage_dir, "data", "elements"), )
def german_heat_system( heat_buses, weather_year, scenario, scenario_year, wacc, decentral_heat_flex_share, sensitivities, datapackage_dir, raw_data_path, ): """ """ technologies = pd.DataFrame( # Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/technology/datapackage.json" ).get_resource("heat").read(keyed=True)).set_index( ["year", "parameter", "carrier", "tech"]) data = (pd.DataFrame( Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/capacities/datapackage.json"). get_resource("german-heat-system").read(keyed=True)).set_index( ["scenario", "year", "carrier", "tech"]).loc[(scenario, scenario_year)]) filepath = building.download_data( "https://data.open-power-system-data.org/when2heat/" "opsd-when2heat-2019-08-06.zip", directory=raw_data_path, unzip_file="opsd-when2heat-2019-08-06/", ) df = pd.read_csv( os.path.join(filepath, "opsd-when2heat-2019-08-06", "when2heat.csv"), index_col=[0], parse_dates=True, sep=";", ) cop = pd.read_csv( os.path.join(filepath, "opsd-when2heat-2019-08-06", "when2heat.csv"), decimal=",", index_col=[0], parse_dates=True, sep=";", ) df = df[~((df.index.month == 2) & (df.index.day == 29))] cop = cop[~((cop.index.month == 2) & (cop.index.day == 29))] data["country"] = "DE" data.set_index("country", append=True, inplace=True) if sensitivities is not None: for k, v in sensitivities.items(): k = k.split("-") data.at[(k[1], k[2], k[0]), "value"] = v elements = [] sequences = {} weather_year = str(weather_year) gshp_cop = cop.loc[ weather_year, ["DE_COP_GSHP_floor", "DE_COP_GSHP_radiator", "DE_COP_GSHP_water"], ].mean(axis=1) ashp_cop = cop.loc[ weather_year, ["DE_COP_ASHP_floor", "DE_COP_ASHP_radiator", "DE_COP_ASHP_water"], ].mean(axis=1) el_buses = building.read_elements("bus.csv", directory=os.path.join( datapackage_dir, "data/elements")) heat_demand_total = (float(data.loc[("decentral_heat", "load"), "value"]) * 1000) # MWh for bustype, buses in heat_buses.items(): carrier = bustype + "_heat" for b in buses: heat_bus = "-".join([b, carrier, "bus"]) flex_peak_demand_heat = ( df.loc[weather_year][b + "_heat_demand_total"] / df.loc[weather_year][b + "_heat_demand_total"].sum() # MW * heat_demand_total).max() * decentral_heat_flex_share peak_demand_heat = ( df.loc[weather_year][b + "_heat_demand_total"] / df.loc[weather_year][b + "_heat_demand_total"].sum() # MW * heat_demand_total).max() * (1 - decentral_heat_flex_share) el_buses.loc[heat_bus] = [True, "heat", None, "bus"] profile_name = "-".join([b, carrier, "load", "profile"]) if "flex" in bustype: elements.append({ "name": "-".join([b, carrier, "load"]), "type": "load", "bus": heat_bus, "amount": heat_demand_total * decentral_heat_flex_share, "profile": profile_name, "carrier": carrier, }) elements.append({ "name": "-".join([b, carrier, "gshp"]), "type": "conversion", "to_bus": heat_bus, "capacity_cost": ( float(technologies.loc[(2050, "fom", "decentral_heat", "gshp"), "value", ]) + annuity( float(technologies.loc[( 2050, "capex", "decentral_heat", "gshp", ), "value", ]), float(technologies.loc[( 2050, "lifetime", "decentral_heat", "gshp", ), "value", ]), wacc, ) * 1000, # €/kW -> €/MW )[0], "from_bus": "DE-electricity", "expandable": True, "capacity": flex_peak_demand_heat, "efficiency": "DE-gshp-profile", "carrier": carrier, "tech": "gshp", }) name = "-".join([b, carrier, "tes"]) if sensitivities is not None: if name in sensitivities.keys(): capacity = sensitivities[name] else: capacity = flex_peak_demand_heat else: capacity = flex_peak_demand_heat carrier = carrier.replace("flex-", "") elements.append({ "name": name, "type": "storage", "bus": heat_bus, # "capacity": capacity, "capacity_cost": 0, "storage_capacity_cost": (float(technologies.loc[(2050, "fom", "decentral_heat", "tes"), "value", ]) * 1000) + ( annuity( float(technologies.loc[( 2050, "capex_energy", "decentral_heat", "tes", ), "value", ]), float(technologies.loc[( 2050, "lifetime", "decentral_heat", "tes", ), "value", ]), wacc, ) * 1000, # €/kWh -> €/MWh )[0], "expandable": True, # "storage_capacity": capacity * float(technologies.loc[ # (2050, "max_hours", carrier, "tes"), # "value" # ]), "efficiency": float( technologies.loc[(2050, "efficiency", carrier, "tes"), "value"])**0.5, # rountrip conversion "loss": technologies.loc[(2050, "loss", carrier, "tes"), "value"], "marginal_cost": 0.001, "carrier": carrier, "tech": "tes", }) else: elements.append({ "name": "-".join([b, carrier, "load"]), "type": "load", "bus": heat_bus, "amount": heat_demand_total * (1 - decentral_heat_flex_share), "profile": profile_name, "carrier": carrier, }) elements.append({ "name": "-".join([b, carrier, "gshp"]), "type": "conversion", "to_bus": heat_bus, "capacity_cost": 0, "expandable": False, "from_bus": "DE-electricity", "capacity": peak_demand_heat, "efficiency": "DE-gshp-profile", "carrier": carrier, "tech": "gshp", }) sequences[profile_name] = ( df.loc[weather_year][b + "_heat_demand_total"] / df.loc[weather_year][b + "_heat_demand_total"].sum()) sequences_df = pd.DataFrame(sequences) sequences_df.index.name = "timeindex" sequences_df.index = building.timeindex(year=str(scenario_year)) sequences_cop = pd.concat([gshp_cop, ashp_cop], axis=1) sequences_cop.columns = ["DE-gshp-profile", "DE-ashp-profile"] sequences_cop.index.name = "timeindex" sequences_cop.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "efficiency_profile.csv", sequences_cop, directory=os.path.join(datapackage_dir, "data/sequences"), ) if "NEPC" in scenario: must_run_sequences = {} must_run_sequences["DE-must-run-profile"] = ( df.loc[weather_year][b + "_heat_demand_total"] / df.loc[weather_year][b + "_heat_demand_total"].max()) must_run_sequences_df = pd.DataFrame(must_run_sequences) must_run_sequences_df = (must_run_sequences_df * 3 * 8300).clip( upper=8300) / 8300 # calibrate for 2030NEPC must_run_sequences_df.index.name = "timeindex" must_run_sequences_df.index = building.timeindex( year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", must_run_sequences_df, directory=os.path.join(datapackage_dir, "data/sequences"), ) building.write_elements( "heat_load.csv", pd.DataFrame([i for i in elements if i["type"] == "load"]).set_index("name"), directory=os.path.join(datapackage_dir, "data/elements"), ) building.write_elements( "heatpump.csv", pd.DataFrame([i for i in elements if i["type"] == "conversion"]).set_index("name"), directory=os.path.join(datapackage_dir, "data/elements"), ) building.write_elements( "heat_storage.csv", pd.DataFrame([i for i in elements if i["type"] == "storage"]).set_index("name"), directory=os.path.join(datapackage_dir, "data/elements"), ) building.write_elements( "bus.csv", el_buses, directory=os.path.join(datapackage_dir, "data/elements"), replace=True, ) building.write_sequences( "heat_load_profile.csv", sequences_df, directory=os.path.join(datapackage_dir, "data/sequences"), )
Open Power System Data. 2017. Data Package Time series. Version 2017-07-09. https://data.open-power-system-data.org/time_series/2017-07-09/. (Primary data from various sources, for a complete list see URL) https://data.open-power-system-data.org/time_series/2017-07-09/time_series_60min_singleindex.csv https://data.open-power-system-data.org/time_series/2017-07-09/time_series_60min_singleindex.csv """ import pandas as pd from oemof.tabular.datapackage import building config = building.read_build_config('config.toml') countries, year = config['countries'], str(config['year']) raw_data = pd.read_csv(building.download_data( 'https://data.open-power-system-data.org/time_series/2017-07-09/' 'time_series_60min_singleindex.csv'), index_col=[0], parse_dates=True, low_memory=False) suffix = '_load_old' column_names = [c + suffix for c in countries] timeseries = raw_data.loc[year, column_names] # replace missing last hour with previous timeseries.loc['2015-12-31 23:00:00', :] = timeseries.loc[ '2015-12-31 22:00:00', :] timeseries['DE_load_old'] = timeseries['DE_load_old'] * ( 596.3e6 / timeseries['DE_load_old'].sum())
def opsd_profile(buses, demand_year, scenario_year, datapackage_dir, raw_data_path): """ Parameter --------- buses: array like List with buses represented by iso country code demand_year: integer or string Demand year to select scenario_year: integer or string Year of scenario to use for timeindex to resource datapackage_dir: string Directory for tabular resource raw_data_path: string Path where raw data file is located """ filepath = building.download_data( "https://data.open-power-system-data.org/time_series/2018-06-30/" "time_series_60min_singleindex.csv", directory=raw_data_path, ) if os.path.exists(filepath): raw_data = pd.read_csv(filepath, index_col=[0], parse_dates=True) else: raise FileNotFoundError( "File for OPSD loads does not exist. Did you download data?") suffix = "_load_entsoe_power_statistics" countries = buses columns = [c + suffix for c in countries] timeseries = raw_data[str(demand_year)][columns] if timeseries.isnull().values.any(): raise ValueError("Timeseries for load has NaN values. Select " + "another demand year or use another data source.") load_total = timeseries.sum() load_profile = timeseries / load_total sequences_df = pd.DataFrame(index=load_profile.index) elements = building.read_elements("load.csv", directory=os.path.join( datapackage_dir, "data", "elements")) for c in countries: # get sequence name from elements edge_parameters # (include re-exp to also check for 'elec' or similar) sequence_name = elements.at[ elements.index[elements.index.str.contains(c)][0], "profile"] sequences_df[sequence_name] = load_profile[c + suffix].values if sequences_df.index.is_leap_year[0]: sequences_df = sequences_df.loc[~((sequences_df.index.month == 2) & (sequences_df.index.day == 29))] sequences_df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "load_profile.csv", sequences_df, directory=os.path.join(datapackage_dir, "data/sequences"), )
https://www.sciencedirect.com/science/article/pii/S0960148118303677 """ import os import pandas as pd from oemof.tabular.datapackage import building from datetime import datetime config = building.read_build_config('config.toml') countries, year = config['countries'], str(config['year']) filepath = building.download_data( "https://www.renewables.ninja/static/downloads/ninja_europe_pv_v1.1.zip", unzip_file="ninja_pv_europe_v1.1_merra2.csv") raw_data = pd.read_csv(filepath, index_col=[0], parse_dates=True) df = raw_data.loc[year] sequences_df = pd.DataFrame(index=df.index) for c in countries: sequence_name = c + "-pv-profile" sequences_df[sequence_name] = raw_data.loc[year][c].values sequences_df.index = building.timeindex(year) building.write_sequences("volatile_profile.csv", sequences_df)
import os from oemof.tabular.datapackage import building # set raw data path to the default fuchur raw raw_data_path # which is: 'home/user/oemof-raw-data' # change this if you have your raw data stored somewhere else raw_data_path = os.path.join(os.path.expanduser("~"), "oemof-raw-data") if not os.path.exists(raw_data_path): os.makedirs(raw_data_path) building.download_data( "https://zenodo.org/record/3549531/files/angus-raw-data.zip?download=1", directory=raw_data_path, unzip_file="", )
technologies = pd.DataFrame( #Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package( 'https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/features/add-2015-data/datapackage.json' ).get_resource('electricity').read(keyed=True)).set_index( ['year', 'carrier', 'tech', 'parameter']) carriers = pd.DataFrame( #Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package( 'https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/features/add-2015-data/datapackage.json' ).get_resource('carrier').read(keyed=True)).set_index( ['year', 'carrier', 'parameter', 'unit']).sort_index() df = pd.read_csv(building.download_data( "https://data.open-power-system-data.org/conventional_power_plants/" "2018-12-20/conventional_power_plants_DE.csv"), encoding='utf-8') cond1 = df['country_code'] == 'DE' cond2 = df['fuel'].isin(['Hydro']) cond3 = (df['fuel'] == 'Other fuels') & (df['technology'] == 'Storage technologies') df = df.loc[cond1 & ~cond2 & ~cond3, :].copy() mapper = { ('Biomass and biogas', 'Steam turbine'): ('biomass', 'biomass'), ('Biomass and biogas', 'Combustion Engine'): ('biomass', 'biomass'), ('Hard coal', 'Steam turbine'): ('coal', 'st'), ('Hard coal', 'Combined cycle'): ('coal', 'ccgt'),
def ehighway_generation( countries, cost_scenario, scenario="100% RES", datapackage_dir=None, raw_data_path=None, ccgt_share=0.66, scenario_year=2050, ): """ """ scenario_mapper = {"100% RES": "T54"} filename = "e-Highway_database_per_country-08022016.xlsx" data = pd.read_excel( building.download_data( "http://www.e-highway2050.eu/fileadmin/documents/Results/" + filename, directory=raw_data_path, ), sheet_name=scenario_mapper[scenario], index_col=[1], skiprows=3, encoding="utf-8", ) data = data.loc[countries] technologies = pd.DataFrame( # Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/technology/datapackage.json" ).get_resource("technology").read(keyed=True)).set_index( ["year", "parameter", "carrier", "tech"]) storage_capacities = (pd.DataFrame( Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/capacities/datapackage.json"). get_resource("storage-capacities").read(keyed=True)).set_index( ["year", "country"]).loc[scenario_year]) storage_capacities.drop("phs", axis=1, inplace=True) storage_capacities.rename( columns={ "acaes": ("cavern", "acaes"), "redox": ("redox", "battery"), "lithium": ("lithium", "battery"), "hydrogen": ("hydrogen", "storage"), }, inplace=True, ) carrier_package = Package( "https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/carrier/datapackage.json") carrier_cost = (pd.DataFrame( carrier_package.get_resource("carrier-cost").read( keyed=True)).set_index(["scenario", "carrier"]).sort_index()) emission_factors = (pd.DataFrame( carrier_package.get_resource("emission-factor").read( keyed=True)).set_index(["carrier"]).sort_index()) data["CCGT"] = data["TOTAL GAS"] * ccgt_share data["OCGT"] = data["TOTAL GAS"] * (1 - ccgt_share) rename_cols = { "Wind": ("wind", "onshore"), "Wind North Sea": ("wind", "offshore"), "PV": ("solar", "pv"), "OCGT": ("gas", "ocgt"), "CCGT": ("gas", "ccgt"), "TOTAL Biomass": ("biomass", "st"), "RoR": ("hydro", "ror"), "PSP": ("hydro", "phs"), "Hydro with reservoir": ("hydro", "rsv"), } data.rename(columns=rename_cols, inplace=True) data = data[[i for i in rename_cols.values()]] data = pd.concat([data, storage_capacities], axis=1, sort=True) elements = _elements( countries, data, technologies, carrier_cost, emission_factors, cost_scenario, scenario_year, ) load = _load(countries, data) for k, v in elements.items(): v.update(load[k]) df = pd.DataFrame.from_dict(elements, orient="index") df = df[df.capacity != 0] for element_type in [ "dispatchable", "volatile", "conversion", "storage", "reservoir", "load", ]: building.write_elements( element_type + ".csv", df.loc[df["type"] == element_type].dropna(how="all", axis=1), directory=os.path.join(datapackage_dir, "data", "elements"), overwrite=True, )
def tyndp_generation_2018( countries, vision, scenario, scenario_year, datapackage_dir, raw_data_path, ccgt_share=0.66, sensitivities=None, ): """Extracts TYNDP2018 generation data and writes to datapackage for oemof tabular usage Parameters ----------- countries: list List with countries to extract (Names in country codes) vision: str TYNDP Vision (one of 2040 GCA, 2030 DG, etc.) scenario: str Name of scenario to be used for cost assumptions etc scenario_year: str Year of scenario datapackage_dir: string Directory for tabular resource raw_data_path: string Path where raw data file `ENTSO%20Scenario%202018%20Generation%20Capacities.xlsm` is located ccgt_share: Share of ccgt generation of total gas generation """ storage_capacities = (pd.DataFrame( Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/capacities/datapackage.json"). get_resource("storage-capacities").read(keyed=True)).set_index( ["year", "country"]).loc[scenario_year]) storage_capacities.drop("phs", axis=1, inplace=True) storage_capacities.rename( columns={ "acaes": ("cavern", "acaes"), "redox": ("redox", "battery"), "lithium": ("lithium", "battery"), "hydrogen": ("hydrogen", "storage"), }, inplace=True, ) filepath = building.download_data( "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP2018/" "Scenarios%20Data%20Sets/ENTSO%20Scenario%202018%20Generation%20Capacities.xlsm", directory=raw_data_path, ) df = pd.read_excel(filepath, sheet_name=vision, index_col=0, skiprows=[0, 1]) colnames = [ "Biofuels", "Gas", "Hard coal", "Hydro-pump", "Hydro-run", "Hydro-turbine", "Lignite", "Nuclear", "Oil", "Othernon-RES", "Other RES", "Solar-thermal", "Solar-\nPV", "Wind-\non-shore", "Wind-\noff-shore", ] newnames = [ ("biomass", "st"), ("gas", "ocgt"), ("coal", "st"), ("hydro", "phs"), ("hydro", "ror"), ("hydro", "rsv"), ("lignite", "st"), ("uranium", "st"), ("oil", "ocgt"), ("mixed", "st"), ("other", "res"), ("solar", "thermal"), ("solar", "pv"), ("wind", "onshore"), ("wind", "offshore"), ] df = df.rename(columns=dict(zip(colnames, newnames))) df[("biomass", "st")] += df[("other", "res")] df.drop([("other", "res")], axis=1, inplace=True) df.index.name = "zones" df.reset_index(inplace=True) df = pd.concat( [ pd.DataFrame( df["zones"].apply(lambda row: [row[0:2], row[2::]]).tolist(), columns=["country", "zone"], ), df, ], axis=1, sort=True, ) df = df.groupby("country").sum() df[("gas", "ccgt")] = df[("gas", "ocgt")] * ccgt_share df[("gas", "ocgt")] = df[("gas", "ocgt")] * (1 - ccgt_share) # as raw data is divided in turbine and pump (where turbine is also from # pump storages as well as reservoirs) df[("hydro", "rsv")] = (df[("hydro", "rsv")] - df[("hydro", "phs")]).clip(0) technologies = pd.DataFrame( # Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/technology/datapackage.json" ).get_resource("technology").read(keyed=True)).set_index( ["year", "parameter", "carrier", "tech"]) carrier_package = Package( "https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/carrier/datapackage.json") carrier_cost = (pd.DataFrame( carrier_package.get_resource("carrier-cost").read( keyed=True)).set_index(["scenario", "carrier"]).sort_index()) emission_factors = (pd.DataFrame( carrier_package.get_resource("emission-factor").read( keyed=True)).set_index(["carrier"]).sort_index()) df = pd.concat([df, storage_capacities], axis=1, sort=True) elements = _elements( countries, df, technologies, carrier_cost, emission_factors, scenario, scenario_year, sensitivities, ) # load = _load(countries, df) # for k,v in load.items(): # v.update(load[k]) df = pd.DataFrame.from_dict(elements, orient="index") df = df[df.capacity != 0] # write elements to CSV-files for element_type in ["dispatchable", "volatile", "conversion", "storage"]: building.write_elements( element_type + ".csv", df.loc[df["type"] == element_type].dropna(how="all", axis=1), directory=os.path.join(datapackage_dir, "data", "elements"), )
def emhires_wind_profiles(buses, weather_year, scenario_year, datapackage_dir, raw_data_path): """ Gonzalez Aparicio, Iratxe; Zucker, Andreas; Careri, Francesco; Monforti Ferrario, Fabio; Huld, Thomas; Badger, Jake (2016): Wind hourly generation time series at country, NUTS 1, NUTS 2 level and bidding zones. European Commission, Joint Research Centre (JRC) [Dataset] PID: http://data.europa.eu/89h/jrc-emhires-wind-generation-time-series """ year = str(weather_year) countries = buses date_parser = lambda y: datetime.strptime(y, "%Y %m %d %H") date_columns = ["Year", "Month", "Day", "Hour"] urls = [ "http://setis.ec.europa.eu/sites/default/files/EMHIRES_DATA/EMHIRES_WIND_COUNTRY_June2019.zip", "http://setis.ec.europa.eu/sites/default/files/EMHIRES_DATA/TS_CF_OFFSHORE_30yr_date.zip", ] filenames = [ "EMHIRES_WIND_COUNTRY_June2019.xlsx", "TS.CF.OFFSHORE.30yr.date.txt", ] technologies = ["onshore", "offshore"] for url, fname, tech in zip(urls, filenames, technologies): if fname.endswith(".xlsx"): df = (pd.read_excel( building.download_data(url, unzip_file=fname, directory=raw_data_path), parse_dates={ "i": date_columns }, date_parser=date_parser, index_col="i", ).reindex(columns=countries).dropna(axis=1).loc[year, :]) else: df = (pd.read_csv( building.download_data(url, unzip_file=fname, directory=raw_data_path), sep="\t", parse_dates={ "i": date_columns }, date_parser=date_parser, index_col="i", ).reindex(columns=countries).dropna(axis=1).loc[year, :]) renames = {c: c + "-" + tech + "-profile" for c in countries} df.rename(columns=renames, inplace=True) df = df[~((df.index.month == 2) & (df.index.day == 29))] df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", df, directory=os.path.join(datapackage_dir, "data", "sequences"), )
def generation(config, scenario_year, datapackage_dir, raw_data_path): """ """ countries, scenario_year = ( config["buses"]["electricity"], config["scenario"]["year"], ) building.download_data( "https://zenodo.org/record/804244/files/Hydro_Inflow.zip?download=1", directory=raw_data_path, unzip_file="Hydro_Inflow/", ) technologies = pd.DataFrame( Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/technology/datapackage.json"). get_resource("technology").read(keyed=True)).set_index( ["year", "parameter", "carrier", "tech"]) hydro_data = pd.DataFrame( Package("https://raw.githubusercontent.com/ZNES-datapackages/" "angus-input-data/master/hydro/datapackage.json").get_resource( "hydro").read(keyed=True)).set_index(["year", "country"]) hydro_data.rename(index={"UK": "GB"}, inplace=True) # for iso code inflows = _get_hydro_inflow( inflow_dir=os.path.join(raw_data_path, "Hydro_Inflow")) inflows = inflows.loc[inflows.index.year == config["scenario"]["weather_year"], :] inflows["DK"], inflows["LU"] = 0, inflows["BE"] for c in hydro_data.columns: if c != "source": hydro_data[c] = hydro_data[c].astype(float) capacities = hydro_data.loc[scenario_year].loc[countries][[ "ror", "rsv", "phs" ]] ror_shares = hydro_data.loc[scenario_year].loc[countries]["ror-share"] max_hours = hydro_data.loc[scenario_year].loc[countries][[ "rsv-max-hours", "phs-max-hours" ]] rsv_factor = hydro_data.loc[scenario_year].loc[countries]["rsv-factor"] # ror elements = {} for country in countries: name = country + "-hydro-ror" capacity = capacities.loc[country, "ror"] # eta = technologies.loc[ # (scenario_year, "efficiency", "hydro", "ror"), "value" # ] if capacity > 0: elements[name] = { "type": "volatile", "tech": "ror", "carrier": "hydro", "bus": country + "-electricity", "capacity": capacity, "profile": country + "-ror-profile", "efficiency": 1, # as already included in inflow profile } building.write_elements( "ror.csv", pd.DataFrame.from_dict(elements, orient="index"), directory=os.path.join(datapackage_dir, "data", "elements"), ) sequences = (inflows[countries] * ror_shares * 1000) / capacities["ror"] col = list(set(countries) - set(["NO", "SE"])) sequences[col] = sequences[col] * 1.5 # correction factor sequences = sequences[countries].copy() sequences.dropna(axis=1, inplace=True) sequences.clip(upper=1, inplace=True) sequences.columns = sequences.columns.astype(str) + "-ror-profile" building.write_sequences( "ror_profile.csv", sequences.set_index(building.timeindex(str(scenario_year))), directory=os.path.join(datapackage_dir, "data", "sequences"), ) # reservoir elements = {} for country in countries: name = country + "-hydro-reservoir" capacity = capacities.loc[country, "rsv"] rsv_max_hours = max_hours.loc[country, "rsv-max-hours"] # eta = technologies.loc[ # (scenario_year, "efficiency", "hydro", "rsv"), "value" # ] if capacity > 0: elements[name] = { "type": "reservoir", "tech": "rsv", "carrier": "hydro", "bus": country + "-electricity", "capacity": capacity, "storage_capacity": capacity * rsv_max_hours, "profile": country + "-reservoir-profile", "efficiency": 1, # as already included in inflow profile "marginal_cost": 0.0000001, } building.write_elements( "reservoir.csv", pd.DataFrame.from_dict(elements, orient="index"), directory=os.path.join(datapackage_dir, "data", "elements"), ) sequences = inflows[countries] * (1 - ror_shares) * 1000 sequences[["NO", "SE"]] = (sequences[["NO", "SE"]] * 1.6) # correction factor sequences = sequences[countries].copy() sequences.dropna(axis=1, inplace=True) sequences.columns = sequences.columns.astype(str) + "-reservoir-profile" building.write_sequences( "reservoir_profile.csv", sequences.set_index(building.timeindex(str(scenario_year))), directory=os.path.join(datapackage_dir, "data", "sequences"), ) # phs elements = {} for country in countries: name = country + "-hydro-phs" capacity = capacities.loc[country, "phs"] phs_max_hours = max_hours.loc[country, "phs-max-hours"] eta = technologies.loc[(scenario_year, "efficiency", "hydro", "phs"), "value"] if capacity > 0: elements[name] = { "type": "storage", "tech": "phs", "carrier": "hydro", "bus": country + "-electricity", "capacity": capacity, "loss": 0, "marginal_cost": 1, "storage_capacity": capacity * phs_max_hours, "storage_capacity_initial": 0.5, "efficiency": float(eta)**(0.5), # rountrip to input/output eta } building.write_elements( "phs.csv", pd.DataFrame.from_dict(elements, orient="index"), directory=os.path.join(datapackage_dir, "data", "elements"), )
# -*- coding: utf-8 -*- """ Define hub regions """ import pandas as pd from oemof.tabular.datapackage import building from oemof.tabular.tools import geometry config = building.read_build_config('config.toml') filepath = building.download_data( 'http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/' 'NUTS_2013_10M_SH.zip', unzip_file='NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.shp') building.download_data( 'http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/' 'NUTS_2013_10M_SH.zip', unzip_file='NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.dbf') # get nuts 1 regions for german neighbours nuts0 = pd.Series(geometry.nuts(filepath, nuts=0, tolerance=0.1)) hubs = pd.Series(name='geometry') hubs.index.name = 'name' # add hubs and their geometry for r in config['countries']: hubs[r + '-electricity'] = nuts0[r]
import os import pandas as pd from datapackage import Package from oemof.tabular.datapackage import building from oemof.tabular.tools import geometry from atlite import Cutout config = building.read_build_config('config.toml') countries, year = config['countries'], config['year'] filepath = building.download_data( 'http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/' 'NUTS_2013_10M_SH.zip', unzip_file='NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.shp') nuts0 = pd.Series(geometry.nuts(filepath, nuts=0, tolerance=0.1))[countries] countrynames = pd.DataFrame(Package( 'https://raw.githubusercontent.com/datasets/country-codes/5b645f4ea861be1362539d06641e5614353c9895/datapackage.json' ).get_resource('country-codes').read(keyed=True)).set_index(['official_name_en'])\ ['ISO3166-1-Alpha-2'].to_dict() hyd = pd.read_csv(os.path.join( config['directories']['archive'], 'EIA-annual-hydro-generation.csv'), skiprows=4, index_col=1 ).drop(['Unnamed: 0', 'Unnamed: 2'], axis=1).dropna().loc[:, str(year)] hyd = hyd.rename(index={'Czech Republic': 'Czechia'}).\ rename(index=countrynames).T
def tyndp(buses, grid_loss, scenario, datapackage_dir, raw_data_path): """ Parameter --------- buses: array like List with buses represented by iso country code grid_loss: numeric Loss for transshipment model (oemof.tabular.facades.Link component attribute) scenario: str Scenario name (e.g. 2040GCA) datapackage_dir: string Directory for tabular resource raw_data_path: string Path where raw data file is located """ filepath = building.download_data( "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP2018/" "Scenarios%20Data%20Sets/Input%20Data.xlsx", directory=raw_data_path, ) mapper = { "2030": ["CBA Capacities", "Unnamed: 3"], "2040GCA": ["Unnamed: 8", "Unnamed: 9"], "2040ST": ["Unnamed: 5", "Unnamed: 6"], "2040DG": ["Unnamed: 6", "Unnamed: 7"], } df = pd.read_excel( filepath, sheet_name="NTC", index_col=[0], skiprows=[1, 2] )[mapper[scenario]] df.columns = ["=>", "<="] df["links"] = df.index.astype(str) df["links"] = df["links"].apply( lambda row: (row.split("-")[0][0:2], row.split("-")[1][0:2]) ) df = df.groupby(df["links"]).sum() df.reset_index(inplace=True) df = pd.concat( [ pd.DataFrame( df["links"].apply(lambda row: [row[0], row[1]]).tolist(), columns=["from", "to"], ), df[["=>", "<="]], ], axis=1, ) elements = {} for idx, row in df.iterrows(): if (row["from"] in buses and row["to"] in buses) and row[ "from" ] != row["to"]: predecessor = row["from"] + "-electricity" successor = row["to"] + "-electricity" element_name = predecessor + "-" + successor element = { "type": "link", "loss": grid_loss, "from_bus": predecessor, "to_bus": successor, "tech": "transshipment", "from_to_capacity": row["=>"], # still need to think how to "to_from_capacity": row["<="], "marginal_cost": 0.0001, } elements[element_name] = element building.write_elements( "link.csv", pd.DataFrame.from_dict(elements, orient="index"), directory=os.path.join(datapackage_dir, "data", "elements"), )
carriers = pd.DataFrame( #Package('/home/planet/data/datapackages/technology-cost/datapackage.json') Package('https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/features/add-2015-data/datapackage.json') .get_resource('carrier').read(keyed=True)).set_index( ['year', 'carrier', 'parameter', 'unit']).sort_index() isocodes = dict(pd.DataFrame( Package('https://raw.githubusercontent.com/datasets/country-codes/master/datapackage.json') .get_resource('country-codes').read(keyed=True)) [['ISO3166-1-Alpha-2', 'official_name_en']].values) isocodes['CZ'] = 'Czech Republic' isocodes['GB'] = 'United Kingdom' df = pd.read_csv(building.download_data( 'https://media.githubusercontent.com/media/FRESNA/powerplantmatching/master/data/out/default/powerplants.csv'), encoding='utf-8', converters={'projectID': literal_eval}) df['Country'] = df['Country'].map({y:x for x, y in isocodes.items()}) cond1 = df['Fueltype'].isin(['Wind', 'Solar', 'Hydro', 'Geothermal']) cond2 = (df['Fueltype'] == 'Natural Gas') & df['Technology'].isin(['Pv', 'Storage Technologies', 'Caes']) cond3 = (df['Fueltype'] == 'Other') & (df['Technology'] == 'Storage Technologies') cond4 = (df['Fueltype'] == 'Bioenergy') & (df['Technology'] == 'Pv') cond5 = df['Country'].isin(countries) df = df.loc[~cond1 & ~cond2 & ~cond3 & ~cond4 & cond5, :].copy() df.fillna({'Technology': 'Unknown'}, inplace=True) # Other
def ehighway( buses, year, grid_loss, scenario="100% RES", datapackage_dir=None, raw_data_path=None, ): """ Parameter --------- buses: array like List with buses represented by iso country code year: integer Scenario year to select. One of: 2030, 2050. If year is 2030, the starting grid will be used, meaning the scenario argument will have no impact datapackage_dir: string Directory for tabular resource scenario: Name of ehighway scenario to select. One of: ["Large Scale RES", "100% RES", "Big & Market", "Fossil & Nuclear", "Small & Local"], default: "100% RES" raw_data_path: string Path where raw data file `e-Highway_database_per_country-08022016.xlsx` is located """ filename = "e-Highway_database_per_country-08022016.xlsx" filepath = building.download_data(filename, directory=raw_data_path) if os.path.exists(filepath): df_2030 = pd.read_excel( filepath, sheet_name="T93", index_col=[1], skiprows=[0, 1, 3] ).fillna(0) df_2050 = pd.read_excel( filepath, sheet_name="T94", index_col=[1], skiprows=[0, 1, 3] ).fillna(0) else: raise FileNotFoundError( "File for e-Highway capacities does not exist. Did you download?" ) df_2050 = _prepare_frame(df_2050).set_index(["Links"]) df_2030 = _prepare_frame(df_2030).set_index(["Links"]) elements = {} for idx, row in df_2030.iterrows(): if row["from"] in buses and row["to"] in buses: predecessor = row["from"] + "-electricity" successor = row["to"] + "-electricity" element_name = predecessor + "-" + successor if year == 2030: capacity = row[scenario] elif year == 2050: capacity = row[scenario] + df_2050.to_dict()[scenario].get( idx, 0 ) element = { "type": "link", "loss": grid_loss, "from_bus": predecessor, "to_bus": successor, "tech": "transshipment", "from_to_capacity": capacity, "to_from_capacity": capacity, "marginal_cost": 0.0001, # "length": row["Length"], } elements[element_name] = element building.write_elements( "link.csv", pd.DataFrame.from_dict(elements, orient="index"), directory=os.path.join(datapackage_dir, "data/elements"), )
""" EU Commission, DG ENER, Unit A4 - ENERGY STATISTICS, https://ec.europa.eu/energy/sites/ener/files/documents/countrydatasheets_june2018.xlsx """ import pandas as pd from oemof.tabular.datapackage import building config = building.read_build_config('config.toml') countries, year = config['countries'], str(config['year']) xl = pd.ExcelFile( building.download_data( 'https://ec.europa.eu/energy/sites/ener/files/documents/countrydatasheets_june2018.xlsx')) stored_capacities = pd.read_csv('archive/capacities.csv', sep=';', index_col=[0, 1, 2]) idx = pd.IndexSlice offshore_capacities = stored_capacities.loc[idx[:, :, 'wind-offshore'], :].\ reset_index(level=['year', 'technology']) # volatile profile Sweden not available offshore_capacities.drop(index=['SE'], inplace=True) elements = {} for country in countries: if country in ['NO', 'CH']: wind_capacity = stored_capacities.loc[(int(year), country, 'wind-total'), 'value']
def eGo_offshore_wind_profiles( buses, weather_year, scenario_year, datapackage_dir, raw_data_path, correction_factor=0.8, ): """ Parameter --------- buses: array like List with buses represented by iso country code weather_year: integer or string Year to select from raw data source scenario_year: integer or string Year to use for timeindex in tabular resource datapackage_dir: string Directory for tabular resource raw_data_path: string """ filepath = building.download_data( "https://github.com/znes/FlEnS/archive/master.zip", unzip_file="FlEnS-master/open_eGo/NEP_2035/nep_2035_seq.csv", directory=raw_data_path, ) wind = pd.read_csv(filepath, parse_dates=True, index_col=0, header=[0, 1, 2, 3, 4]) wind.columns = wind.columns.droplevel([0, 2, 3, 4]) wind.reset_index(inplace=True) sequences_df = pd.DataFrame() # use vernetzen data filepath_2050 = building.download_data( "https://github.com/znes/FlEnS/archive/master.zip", unzip_file="FlEnS-master/Socio-ecologic/2050_seq.csv", directory=raw_data_path, ) wind_2050 = pd.read_csv(filepath_2050, parse_dates=True, index_col=0, header=[0, 1, 2, 3, 4]) wind_2050.columns = wind_2050.columns.droplevel([0, 2, 3, 4]) wind_2050["DE_wind_offshore"] = (wind_2050["DEdr19_wind_offshore"] * 0.2 + wind_2050["DEdr20_wind_offshore"] * 0.4 + wind_2050["DEdr21_wind_offshore"] * 0.4) wind_2050.reset_index(inplace=True) wind_2050["DE_wind_onshore"] = wind["DE_wind_onshore"] wind = wind_2050 for c in buses: if c + "_wind_offshore" in wind.columns: sequences_df[c + "-offshore-profile"] = ( wind[c + "_wind_offshore"] * correction_factor ) # correction factor sequences_df.index = building.timeindex(year=str(scenario_year)) building.write_sequences( "volatile_profile.csv", sequences_df, directory=os.path.join(datapackage_dir, "data", "sequences"), )