def add_pp_limit(table_collection, year):
if len(cfg.get_dict('limited_transformer').keys()) > 0:
repp = reegis_tools.bmwi.bmwi_re_energy_capacity()
trsf = table_collection['transformer']
for limit_trsf in cfg.get_dict('limited_transformer').keys():
trsf = table_collection['transformer']
try:
limit = repp.loc[year, (limit_trsf, 'energy')]
except KeyError:
msg = "Cannot calculate limit for {0} in {1}."
raise ValueError(msg.format(limit_trsf, year))
cap_sum = trsf.loc['capacity',
(slice(None), slice(limit_trsf))].sum()
for region in trsf.columns.get_level_values(level=0).unique():
trsf.loc['limit_elec_pp',
(region,
limit_trsf)] = round(trsf.loc['capacity',
(region, limit_trsf)] /
cap_sum * limit + 0.5)
trsf.loc['limit_elec_pp'] = trsf.loc['limit_elec_pp'].fillna(
float('inf'))
table_collection['transformer'] = trsf
return table_collection
def scenario_feedin_pv(year, my_index):
pv_types = cfg.get_dict('pv_types')
pv_orientation = cfg.get_dict('pv_orientation')
pv = de21.feedin.get_de21_feedin(year, 'solar')
# combine different pv-sets to one feedin time series
feedin_ts = pd.DataFrame(columns=my_index, index=pv.index)
orientation_fraction = pd.Series(pv_orientation)
pv.sort_index(1, inplace=True)
orientation_fraction.sort_index(inplace=True)
base_set_column = 'coastdat_{0}_solar_{1}'.format(year, '{0}')
for reg in pv.columns.levels[0]:
feedin_ts[reg, 'solar'] = 0
for mset in pv_types.keys():
set_col = base_set_column.format(mset)
feedin_ts[reg, 'solar'] += pv[reg, set_col].multiply(
orientation_fraction).sum(1).multiply(pv_types[mset])
# feedin_ts[reg, 'solar'] = rt
# print(f.sum())
# from matplotlib import pyplot as plt
# f.plot()
# plt.show()
# exit(0)
return feedin_ts.sort_index(1)
def powerplants(table_collection, year, round_values=None):
"""Get power plants for the scenario year
"""
pp = de21.powerplants.get_de21_pp_by_year(year, overwrite_capacity=True)
logging.info("Adding power plants to your scenario.")
replace_names = cfg.get_dict('source_names')
replace_names.update(cfg.get_dict('source_groups'))
pp['energy_source_level_2'].replace(replace_names, inplace=True)
pp['model_classes'] = pp['energy_source_level_2'].replace(
cfg.get_dict('model_classes'))
pp = pp.groupby(['model_classes', 'de21_region', 'energy_source_level_2'
]).sum()[['capacity', 'capacity_in']]
for model_class in pp.index.get_level_values(level=0).unique():
pp_class = pp.loc[model_class]
if model_class != 'volatile_source':
pp_class['efficiency'] = (pp_class['capacity'] /
pp_class['capacity_in'] * 100)
del pp_class['capacity_in']
if round_values is not None:
pp_class = pp_class.round(round_values)
if 'efficiency' in pp_class:
pp_class['efficiency'] = pp_class['efficiency'].div(100)
pp_class = pp_class.transpose()
pp_class.index.name = 'parameter'
table_collection[model_class] = pp_class
table_collection = add_pp_limit(table_collection, year)
return table_collection
def get_conversion_balance(year):
fn = os.path.join(
cfg.get('paths', 'static_sources'),
cfg.get('energy_balance', 'energy_balance_states_conversion'))
eb = pd.read_csv(fn, index_col=[0, 1, 2, 3])
eb.rename(columns=cfg.get_dict('COLUMN_TRANSLATION'), inplace=True)
eb.sort_index(0, inplace=True)
eb = eb.apply(lambda x: pd.to_numeric(x, errors='coerce')).fillna(0)
eb = eb.groupby(by=cfg.get_dict('FUEL_GROUPS'), axis=1).sum()
eb = fix_states(year, eb)
return eb
def feedin_windpowerlib(weather, turbine, installed_capacity=1):
"""Use the windpowerlib to generate normalised feedin time series.
Parameters
----------
turbine : dict
Parameters of the wind turbine (hub height, diameter of the rotor,
identifier of the turbine to get cp-series, nominal power).
weather : pandas.DataFrame
Weather data set. See module header.
installed_capacity : float
Overall installed capacity for the given wind turbine. The installed
capacity is set to 1 by default for normalised time series.
Returns
-------
pandas.DataFrame
"""
wpp = WindTurbine(**turbine)
modelchain_data = cfg.get_dict('windpowerlib')
mc = ModelChain(wpp, **modelchain_data)
mcwpp = mc.run_model(weather)
return mcwpp.power_output.div(
turbine['nominal_power']).multiply(installed_capacity)
def share_houses_flats(key=None):
"""
Parameters
----------
key str
Valid keys are: 'total_area', 'avg_area', 'share_area', 'total_number',
'share_number'.
Returns
-------
dict or pd.DataFrame
"""
size = pd.Series([1, 25, 50, 70, 90, 110, 130, 150, 170, 190, 210])
infile = os.path.join(cfg.get('paths', 'data_de21'),
cfg.get('general_sources', 'zensus_flats'))
whg = pd.read_csv(infile,
delimiter=';',
index_col=[0],
header=[0, 1],
skiprows=5)
whg = whg.loc[whg['Insgesamt', 'Insgesamt'].notnull()]
new_index = []
states = cfg.get_dict('STATES')
for i in whg.index:
new_index.append(states[i[3:-13]])
whg.index = new_index
flat = {
'total_area': pd.DataFrame(),
'total_number': pd.DataFrame(),
}
for f in whg.columns.get_level_values(0).unique():
df = pd.DataFrame(whg[f].values * size.values,
columns=whg[f].columns,
index=whg.index)
flat['total_area'][f] = df.sum(1) - df['Insgesamt']
flat['total_number'][f] = df['Insgesamt']
flat['total_area']['1 + 2 Wohnungen'] = (flat['total_area']['1 Wohnung'] +
flat['total_area']['2 Wohnungen'])
flat['total_number']['1 + 2 Wohnungen'] = (
flat['total_number']['1 Wohnung'] +
flat['total_number']['2 Wohnungen'])
flat['avg_area'] = flat['total_area'].div(flat['total_number'])
flat['share_area'] = (flat['total_area'].transpose().div(
flat['total_area']['Insgesamt'])).transpose().round(3)
flat['share_number'] = (flat['total_number'].transpose().div(
flat['total_number']['Insgesamt'])).transpose().round(3)
if key is None:
return flat
elif key in flat:
return flat[key].sort_index()
else:
logging.warning(
"'{0}' is an invalid key for function 'share_houses_flats'".format(
key))
return None
def scenario_feedin_pv(year, my_index):
pv_types = cfg.get_dict('pv_types')
pv_orientation = cfg.get_dict('pv_orientation')
pv = feedin.get_berlin_feedin(2014, 'solar')
# combine different pv-sets to one feedin time series
feedin_ts = pd.DataFrame(columns=my_index, index=pv.index)
orientation_fraction = pd.Series(pv_orientation)
pv.sort_index(1, inplace=True)
orientation_fraction.sort_index(inplace=True)
base_set_column = 'coastdat_{0}_solar_{1}'.format(year, '{0}')
for reg in pv.columns.levels[0]:
feedin_ts['feedin', 'solar'] = 0
for mset in pv_types.keys():
set_col = base_set_column.format(mset)
feedin_ts['feedin', 'solar'] += pv[reg, set_col].multiply(
orientation_fraction).sum(1).multiply(pv_types[mset])
return feedin_ts.sort_index(1)
def get_de_balance(year=None, grouped=False):
fname_de = os.path.join(
cfg.get('paths', 'static_sources'),
cfg.get('energy_balance', 'energy_balance_de_original'))
deb = pd.read_csv(fname_de, index_col=[0, 1, 2]).fillna(0)
deb.rename(columns=cfg.get_dict('COLUMN_TRANSLATION'), inplace=True)
deb.sort_index(0, inplace=True)
deb = deb.apply(lambda x: pd.to_numeric(x, errors='coerce')).fillna(0)
new_index_values = list()
sector = cfg.get_dict('SECTOR')
for value in deb.index.get_level_values(2):
new_index_values.append(sector[value])
deb.index.set_levels(new_index_values, level=2, inplace=True)
if grouped:
deb = deb.groupby(by=cfg.get_dict('FUEL_GROUPS'), axis=1).sum()
deb.index = deb.index.set_names(['year', 'state', 'sector'])
deb.sort_index(0, inplace=True)
if year is not None:
deb = deb.loc[year]
return deb
def get_states_balance(year=None, grouped=False, overwrite=False):
fname = os.path.join(cfg.get('paths', 'energy_balance'),
cfg.get('energy_balance', 'energy_balance_edited'))
if not os.path.isfile(fname) or overwrite:
edit_balance()
eb = pd.read_csv(fname, index_col=[0, 1, 2])
if grouped:
eb = eb.groupby(by=cfg.get_dict('FUEL_GROUPS'), axis=1).sum()
eb.index = eb.index.set_names(['year', 'state', 'sector'])
if year is not None:
eb = eb.loc[year]
return eb
def get_heat_profiles_by_state(year, to_csv=False, divide_domestic=False):
building_class = {}
for (k, v) in cfg.get_dict('building_class').items():
for s in v.split(', '):
building_class[s] = int(k)
demand_state = heat_demand(year).sort_index()
if divide_domestic:
house_flats = share_houses_flats('share_area')
for state in demand_state.index.get_level_values(0).unique():
dom = demand_state.loc[state, 'domestic']
demand_state.loc[(state, 'domestic_efh'), ] = (
dom * house_flats.loc[state, '1 + 2 Wohnungen'])
demand_state.sort_index(0, inplace=True)
dom = demand_state.loc[state, 'domestic']
demand_state.loc[(state, 'domestic_mfh'), ] = (
dom * house_flats.loc[state, '3 und mehr Wohnungen'])
demand_state.sort_index(0, inplace=True)
demand_state.sort_index(inplace=True)
demand_state.drop('domestic', level=1, inplace=True)
temperatures = reegis_tools.coastdat.federal_state_average_weather(
year, 'temp_air')
temperatures = temperatures.tz_localize('UTC').tz_convert('Europe/Berlin')
my_columns = pd.MultiIndex(levels=[[], [], []], labels=[[], [], []])
heat_profiles = pd.DataFrame(columns=my_columns)
for region in demand_state.index.get_level_values(0).unique():
logging.info("Creating heat profile for {}".format(region))
tmp = demand_state.loc[region].groupby(level=0).sum()
temperature = temperatures[region] - 273
for fuel in tmp.columns:
logging.debug("{0} - {1} ({2})".format(region, fuel,
building_class[region]))
for sector in tmp.index:
heat_profiles[(region, sector,
fuel)] = (get_heat_profile_from_demandlib(
temperature, tmp.loc[sector, fuel], sector,
year, building_class[region]))
heat_profiles.sort_index(1, inplace=True)
if to_csv:
heat_profiles.to_csv(
os.path.join(
cfg.get('paths', 'demand'),
cfg.get('demand', 'heat_profile_state').format(year=year)))
return heat_profiles
def scenario_commodity_sources(year, use_znes_2014=True):
cs = reegis_tools.commodity_sources.get_commodity_sources()
rename_cols = {
key.lower(): value
for key, value in cfg.get_dict('source_names').items()
}
cs = cs.rename(columns=rename_cols)
cs_year = cs.loc[year]
if use_znes_2014:
before = len(cs_year[cs_year.isnull()])
cs_year = cs_year.fillna(cs.loc[2014])
after = len(cs_year[cs_year.isnull()])
if before - after > 0:
logging.warning("Values were replaced with znes2014 data.")
cs_year.sort_index(inplace=True)
return cs_year
def create_windpowerlib_sets():
"""Create parameter sets for the windpowerlib from wind.ini.
Returns
-------
dict
"""
windpowerlib_sets = cfg.get_list('wind', 'set_list')
# Only one subset is created but following the pvlib sets it is possible
# to create subsets.
windsets = {}
for windpowerlib_set in windpowerlib_sets:
w_set = {1: cfg.get_dict(windpowerlib_set)}
set_name = w_set[1].pop('set_name')
windsets[set_name] = w_set
return windsets
def reshape_conversion_balance(year):
# get conversion balance for the federal states
eb = reegis_tools.energy_balance.get_conversion_balance(year)
# create empty DataFrame to take the conversion balance for the regions
my_index = pd.MultiIndex(levels=[[], [], [], []], labels=[[], [], [], []])
eb21 = pd.DataFrame(index=my_index, columns=eb.columns)
# Use the number of inhabitants to reshape the balance to the new regions
logging.info("Fetching inhabitants table.")
inhabitants = de21.inhabitants.get_ew_by_de21_subregions(year)
inhabitants = inhabitants.replace({'state': cfg.get_dict('STATES')})
inhabitants_by_state = inhabitants.groupby('state').sum()
# Calculate the share of inhabitants of a state that is within a specific
# model region.
logging.info(
"Rearrange state table of the conversion balance to the de21 regions")
for subregion in inhabitants.index:
inhabitants.loc[subregion, 'share_state'] = float(
inhabitants.loc[subregion, 'ew'] /
inhabitants_by_state.loc[inhabitants.loc[subregion, 'state']])
# Loop over the de21 regions
for de21_region in sorted(inhabitants.region.unique()):
# Get all states that intersects with the current de21-region
states = inhabitants.loc[inhabitants.region == de21_region].state
# Sum up the fraction of each state-table to get the new region table
for idx in eb.loc[year, states[0]].index:
eb21.loc[year, de21_region, idx[0], idx[1]] = 0
for state in states:
share = inhabitants.loc[
(inhabitants['region'] == de21_region)
& (inhabitants['state'] == state)]['share_state']
eb21.loc[year, de21_region, idx[0],
idx[1]] += (eb.loc[year, state, idx[0], idx[1]] *
float(share))
eb21.rename(columns={'re': cfg.get('chp', 'renewable_source')},
inplace=True)
return eb21
def normalised_feedin_for_each_data_set(year, wind=True, solar=True,
overwrite=False):
"""
Loop over all weather data sets (regions) and calculate a normalised time
series for each data set with the given parameters of the power plants.
This file could be more elegant and shorter but it will be rewritten soon
with the new feedinlib features.
year : int
The year of the weather data set to use.
wind : boolean
Set to True if you want to create wind feed-in time series.
solar : boolean
Set to True if you want to create solar feed-in time series.
Returns
-------
"""
# Get coordinates of the coastdat data points.
data_points = pd.read_csv(
os.path.join(cfg.get('paths', 'geometry'),
cfg.get('coastdat', 'coastdatgrid_centroid')),
index_col='gid')
# Open coastdat-weather data hdf5 file for the given year or try to
# download it if the file is not found.
weather_file_name = os.path.join(
cfg.get('paths', 'coastdat'),
cfg.get('coastdat', 'file_pattern').format(year=year))
if not os.path.isfile(weather_file_name):
get_coastdat_data(year, weather_file_name)
weather = pd.HDFStore(weather_file_name, mode='r')
# Fetch coastdat data heights from ini file.
data_height = cfg.get_dict('coastdat_data_height')
# Create basic file and path pattern for the resulting files
coastdat_path = os.path.join(cfg.get('paths_pattern', 'coastdat'))
feedin_file = os.path.join(coastdat_path,
cfg.get('feedin', 'file_pattern'))
# Fetch coastdat region-keys from weather file.
key_file_path = coastdat_path.format(year='', type='')[:-2]
key_file = os.path.join(key_file_path, 'coastdat_keys.csv')
if not os.path.isfile(key_file):
coastdat_keys = weather.keys()
if not os.path.isdir(key_file_path):
os.makedirs(key_file_path)
pd.Series(coastdat_keys).to_csv(key_file)
else:
coastdat_keys = pd.read_csv(key_file, index_col=[0],
squeeze=True, header=None)
txt_create = "Creating normalised {0} feedin time series for {1}."
hdf = {'wind': {}, 'solar': {}}
if solar:
logging.info(txt_create.format('solar', year))
# Add directory if not present
os.makedirs(coastdat_path.format(year=year, type='solar'),
exist_ok=True)
# Create the pv-sets defined in the solar.ini
pv_sets = feedin.create_pvlib_sets()
# Open a file for each main set (subsets are stored in columns)
for pv_key, pv_set in pv_sets.items():
filename = feedin_file.format(
type='solar', year=year, set_name=pv_key)
if not os.path.isfile(filename) or overwrite:
hdf['solar'][pv_key] = pd.HDFStore(filename, mode='w')
else:
pv_sets = {}
if wind:
logging.info(txt_create.format('wind', year))
# Add directory if not present
os.makedirs(coastdat_path.format(year=year, type='wind'),
exist_ok=True)
# Create the pv-sets defined in the wind.ini
wind_sets = feedin.create_windpowerlib_sets()
# Open a file for each main set (subsets are stored in columns)
for wind_key, wind_set in wind_sets.items():
filename = feedin_file.format(
type='wind', year=year, set_name=wind_key)
if not os.path.isfile(filename) or overwrite:
hdf['wind'][wind_key] = pd.HDFStore(filename, mode='w')
else:
wind_sets = {}
# Define basic variables for time logging
remain = len(coastdat_keys)
done = 0
start = datetime.datetime.now()
# Loop over all regions
for coastdat_key in coastdat_keys:
# Get weather data set for one location
local_weather = weather[coastdat_key]
# Adapt the coastdat weather format to the needs of pvlib.
# The expression "len(list(hdf['solar'].keys()))" returns the number
# of open hdf5 files. If no file is open, there is nothing to do.
if solar and len(list(hdf['solar'].keys())) > 0:
# Get coordinates for the weather location
local_point = data_points.loc[int(coastdat_key[2:])]
# Create a pvlib Location object
location = pvlib.location.Location(
latitude=local_point['lat'], longitude=local_point['lon'])
# Adapt weather data to the needs of the pvlib
local_weather_pv = adapt_coastdat_weather_to_pvlib(
local_weather, location)
# Create one DataFrame for each pv-set and store into the file
for pv_key, pv_set in pv_sets.items():
if pv_key in hdf['solar']:
hdf['solar'][pv_key][coastdat_key] = feedin.feedin_pv_sets(
local_weather_pv, location, pv_set)
# Create one DataFrame for each wind-set and store into the file
if wind and len(list(hdf['wind'].keys())) > 0:
local_weather_wind = adapt_coastdat_weather_to_windpowerlib(
local_weather, data_height)
for wind_key, wind_set in wind_sets.items():
if wind_key in hdf['wind']:
hdf['wind'][wind_key][coastdat_key] = (
feedin.feedin_wind_sets(
local_weather_wind, wind_set))
# Start- time logging *******
remain -= 1
done += 1
if divmod(remain, 10)[1] == 0:
elapsed_time = (datetime.datetime.now() - start).seconds
remain_time = elapsed_time / done * remain
end_time = datetime.datetime.now() + datetime.timedelta(
seconds=remain_time)
msg = "Actual time: {:%H:%M}, estimated end time: {:%H:%M}, "
msg += "done: {0}, remain: {1}".format(done, remain)
logging.info(msg.format(datetime.datetime.now(), end_time))
# End - time logging ********
for k1 in hdf.keys():
for k2 in hdf[k1].keys():
hdf[k1][k2].close()
weather.close()
logging.info("All feedin time series for {0} are stored in {1}".format(
year, coastdat_path.format(year=year, type='')))
def get_map_config():
maps = {}
keys = cfg.get_list('fis_broker', 'maps')
for key in keys:
maps[key] = cfg.get_dict(key)
return maps
def nodes_from_table_collection(table_collection):
# Create a special dictionary that will raise an error if a key is
# updated. This avoids the
nodes = reegis_tools.scenario_tools.NodeDict()
# Global commodity sources
cs = table_collection['commodity_sources']['DE']
for fuel in cs.columns:
bus_label = 'bus_cs_{0}'.format(fuel.replace(' ', '_'))
nodes[bus_label] = solph.Bus(label=bus_label)
cs_label = 'source_cs_{0}'.format(fuel.replace(' ', '_'))
nodes[cs_label] = solph.Source(
label=cs_label,
outputs={
nodes[bus_label]:
solph.Flow(variable_costs=cs.loc['costs', fuel],
emission=cs.loc['emission', fuel])
})
# Create additional bus for electricity as source. This bus can be
# connected to the electricity bus for future scenarios.
nodes['bus_cs_electricity'] = solph.Bus(label='bus_cs_electricity')
# Create electricity Bus
elec_bus_label = 'bus_elec_{0}'.format('BE')
nodes[elec_bus_label] = solph.Bus(label=elec_bus_label)
# Local volatile electricity sources
vs = table_collection['volatile_source']
ts = table_collection['time_series']
for vs_type in vs['BE'].columns:
vs_label = 'source_{0}_{1}'.format(vs_type, 'BE')
capacity = vs.loc['capacity', ('BE', vs_type)]
try:
feedin = ts['feedin', vs_type.lower()]
except KeyError:
if capacity > 0:
msg = "Missing time series for {0} (capacity: {1}) in {2}."
raise ValueError(msg.format(vs_type, capacity, 'BE'))
if capacity * sum(feedin) > 0:
nodes[vs_label] = solph.Source(label=vs_label,
outputs={
nodes[elec_bus_label]:
solph.Flow(
actual_value=feedin,
nominal_value=capacity,
fixed=True,
emission=0)
})
# Decentralised heating systems
dh = table_collection['decentralised_heating']
for fuel in ts['decentralised_demand'].columns:
src = dh.loc['source', ('BE_demand', fuel)]
if src == 'elec':
bus_label = elec_bus_label
else:
bus_label = 'bus_cs_{0}'.format(src.replace(' ', '_'))
# Check if source bus exists
if bus_label not in nodes:
msg = "Bus '{0}' not found for source '{1}'. Node without Bus!"
raise ValueError(msg.format(bus_label, src))
# Create heating bus as Bus
heat_bus_label = 'bus_dectrl_heating_{0}'.format(fuel.replace(
' ', '_'))
nodes[heat_bus_label] = solph.Bus(label=heat_bus_label)
# Create heating system as Transformer
trsf_label = 'trsf_dectrl_heating_{0}'.format(fuel.replace(' ', '_'))
efficiency = float(dh.loc['efficiency', ('BE_demand', fuel)])
nodes[trsf_label] = solph.Transformer(
label=trsf_label,
inputs={nodes[bus_label]: solph.Flow()},
outputs={nodes[heat_bus_label]: solph.Flow()},
conversion_factors={nodes[heat_bus_label]: efficiency})
# Create demand as Sink
d_heat_demand_label = 'demand_dectrl_heating_{0}'.format(
fuel.replace(' ', '_'))
nodes[d_heat_demand_label] = solph.Sink(
label=d_heat_demand_label,
inputs={
nodes[heat_bus_label]:
solph.Flow(actual_value=ts['decentralised_demand', fuel],
nominal_value=1,
fixed=True)
})
# Electricity demand
elec_demand_label = 'demand_elec_{0}'.format('BE')
nodes[elec_demand_label] = solph.Sink(
label=elec_demand_label,
inputs={
nodes[elec_bus_label]:
solph.Flow(actual_value=ts['electricity', 'demand'],
nominal_value=1,
fixed=True)
})
# District heating demand
for system in ts['district_heating_demand'].columns:
if ts['district_heating_demand'][system].sum() > 0:
bus_label = 'bus_distr_heat_{0}'.format(system)
if bus_label not in nodes:
nodes[bus_label] = solph.Bus(label=bus_label)
dh_demand_label = 'demand_distr_heat_{0}'.format(system)
nodes[dh_demand_label] = solph.Sink(
label=dh_demand_label,
inputs={
nodes[bus_label]:
solph.Flow(actual_value=ts['district_heating_demand',
system],
nominal_value=1,
fixed=True)
})
# Prepare the input table for power plants
pp = table_collection['powerplants']['BE'].copy()
pp = pp.fillna(0)
pp['capacity_in'] = (pp.capacity_elec + pp.capacity_heat) / pp.efficiency
pp['eff_cond_elec'] = pp.capacity_elec / pp.capacity_in
pp['eff_chp_heat'] = pp.capacity_heat / pp.capacity_in
pp['capacity_elec_chp'] = (pp.eff_cond_elec - pp.elec_loss_factor *
pp.eff_chp_heat) * pp.capacity_in
pp['eff_chp_elec'] = pp.capacity_elec_chp / pp.capacity_in
pp = pp.groupby(['type', 'network', 'fuel']).sum()
pp['eff_cond_elec'] = pp.capacity_elec / pp.capacity_in
pp['eff_chp_heat'] = pp.capacity_heat / pp.capacity_in
pp['eff_chp_elec'] = pp.capacity_elec_chp / pp.capacity_in
district_heating_systems = cfg.get_dict('district_heating_systems')
fuel_dict = cfg.get_dict('fuel_dict')
# Create chp plants with extraction turbine
for ext in pp.loc['EXT'].iterrows():
heat_sys = district_heating_systems[ext[0][0]]
src = ext[0][1].replace(' ', '_')
if src in fuel_dict:
src = fuel_dict[src]
src_bus_label = 'bus_cs_{0}'.format(src)
heat_bus_label = 'bus_distr_heat_{0}'.format(heat_sys)
chp_label = 'transf_chp_ext_{0}_{1}'.format(src, heat_sys)
bel = nodes[elec_bus_label]
bth = nodes[heat_bus_label]
nodes[chp_label] = solph.components.ExtractionTurbineCHP(
label=chp_label,
inputs={
nodes[src_bus_label]:
solph.Flow(nominal_value=ext[1].capacity_in)
},
outputs={
bel: solph.Flow(),
bth: solph.Flow()
},
conversion_factors={
bel: ext[1].eff_chp_elec,
bth: ext[1].eff_chp_heat
},
conversion_factor_full_condensation={bel: ext[1].eff_cond_elec})
# Create chp plants with fixed heat ratio (e.g. backpressure)
for fix in pp.loc['FIX'].iterrows():
heat_sys = district_heating_systems[fix[0][0]]
src = fix[0][1].replace(' ', '_')
if src in fuel_dict:
src = fuel_dict[src]
src_bus_label = 'bus_cs_{0}'.format(src)
heat_bus_label = 'bus_distr_heat_{0}'.format(heat_sys)
chp_label = 'transf_chp_fix_{0}_{1}'.format(src, heat_sys)
bel = nodes[elec_bus_label]
bth = nodes[heat_bus_label]
nodes[chp_label] = solph.Transformer(
label=chp_label,
inputs={
nodes[src_bus_label]:
solph.Flow(nominal_value=fix[1].capacity_in)
},
outputs={
bel: solph.Flow(),
bth: solph.Flow()
},
conversion_factors={
bel: fix[1].eff_chp_elec,
bth: fix[1].eff_chp_heat
})
# Create heat plants (hp) without power production
for hp in pp.loc['HP'].iterrows():
heat_sys = district_heating_systems[hp[0][0]]
src = hp[0][1].replace(' ', '_')
if src in fuel_dict:
src = fuel_dict[src]
src_bus_label = 'bus_cs_{0}'.format(src)
heat_bus_label = 'bus_distr_heat_{0}'.format(heat_sys)
hp_label = 'transf_hp_{0}_{1}'.format(src, heat_sys)
bth = nodes[heat_bus_label]
nodes[hp_label] = solph.Transformer(
label=hp_label,
inputs={
nodes[src_bus_label]:
solph.Flow(nominal_value=hp[1].capacity_in)
},
outputs={bth: solph.Flow()},
conversion_factors={bth: hp[1].eff_chp_heat})
# Create power plants without heat extraction
for pp in pp.loc['PP'].iterrows():
heat_sys = district_heating_systems[pp[0][0]]
src = pp[0][1].replace(' ', '_')
if src in fuel_dict:
src = fuel_dict[src]
src_bus_label = 'bus_cs_{0}'.format(src)
pp_label = 'transf_pp_{0}_{1}'.format(src, heat_sys)
bel = nodes[elec_bus_label]
nodes[pp_label] = solph.Transformer(
label=pp_label,
inputs={nodes[src_bus_label]: solph.Flow()},
outputs={bel: solph.Flow(nominal_value=pp[1].capacity_elec)},
conversion_factors={bel: pp[1].efficiency})
# # Storages
# storages = table_collection['storages']
# storages.columns = storages.columns.swaplevel()
# for region in storages['phes'].columns:
# storage_label = 'phe_storage_{0}'.format(region)
# bus_label = 'bus_elec_{0}'.format(region)
# params = storages['phes'][region]
# nodes[storage_label] = solph.components.GenericStorage(
# label=storage_label,
# inputs={nodes[bus_label]: solph.Flow(
# nominal_value=params.pump)},
# outputs={nodes[bus_label]: solph.Flow(
# nominal_value=params.turbine)},
# nominal_capacity=params.energy,
# capacity_loss=0,
# initial_capacity=None,
# inflow_conversion_factor=params.pump_eff,
# outflow_conversion_factor=params.turbine_eff)
# Add shortage excess to every bus
bus_keys = [key for key in nodes.keys() if 'bus' in key]
for key in bus_keys:
excess_label = 'excess_{0}'.format(key)
nodes[excess_label] = solph.Sink(label=excess_label,
inputs={nodes[key]: solph.Flow()})
shortage_label = 'shortage_{0}'.format(key)
nodes[shortage_label] = solph.Source(
label=shortage_label,
outputs={nodes[key]: solph.Flow(variable_costs=9000)})
return nodes
def create_heat_profiles(year):
"""Create heat_profiles for the basic scenario as time series.
- district heating time series for the different district heating systems
- decentralised heating demand time series for different fuels
Parameters
----------
year : int
The year of the basic scenario.
Returns
-------
pandas.DataFrame
"""
logging.info("Starting...")
# allocation of district heating systems (map) to groups (model)
district_heating_groups = cfg.get_dict('district_heating_systems')
# A file with a heat factor for each building type of the alkis
# classification. Buildings like garages etc get the heat-factor 0. It is
# possible to define building factors between 0 and 1.
filename_heat_factor = os.path.join(
cfg.get('paths', 'data_berlin'),
cfg.get('oeq', 'alkis_heat_factor_table'))
heat_factor = pd.read_csv(filename_heat_factor, index_col=[0])
del heat_factor['gebaeude_1']
# heat demand for each building from open_e_quarter
data_oeq = load_heat_data()
# areas of district heating systems in Berlin
distr_heat_areas = get_district_heating_areas()
# Every building has a block id from the block the building is located.
# Every block that touches a district heating area has the STIFT (number)
# of this district heating system. By merging this information every
# building gets the STIFT (number) of the district heating area.
data = data_oeq.merge(distr_heat_areas[['gml_id', 'STIFT']],
left_on='block', right_on='gml_id', how='left')
# Merge the heat-factor for each building type to the alkis types
data = data.merge(heat_factor, left_on='building_function',
right_index=True)
data['total'] = data['my_total']
# Multiply the heat demand of the buildings with the heat factor
data['total'] *= data['heat_factor']
# Level the overall heat demand with the heat demand from the energy
# balance
end_energy_table = get_end_energy_data(year)
factor = (end_energy_table.loc['total', 'district heating'] /
(data['total'].sum() / 1000 / 1000))
data['total'] = data['total'] * factor
# Todo: Prozesswärme
# Create a dictionary for each demand profile group
shlp = {'ghd': {'build_class': 0},
'mfh': {'build_class': 1}}
# Add the annual demand to the profile dictionary.
frac_cols = [x for x in data.columns if 'frac_' in x]
for t in shlp.keys():
# Multiply fraction columns with total heat demand to get the total
# demand for each fuel type
shlp[t]['demand'] = data[frac_cols].multiply(data['total'] * data[t],
axis=0).sum()
# Create the standardised heat load profiles (shlp) for each group
heat_profiles = create_standardised_heat_load_profile(shlp, year)
# Create a summable column for each demand group for district heating
cols = []
for shlp_type in shlp.keys():
name = 'district_' + shlp_type
data[name] = (data['frac_district_heating'] *
data[shlp_type] * data['total'])
cols.append(name)
# Group district heating by district heating systems (STIFT = id)
district_by_stift = data.groupby('STIFT').sum()[cols]
# Create translation Series with STIFT (numeric) and KLASSENNAM (text)
stift2name = distr_heat_areas.groupby(
['STIFT', 'KLASSENNAM']).size().reset_index(
level='KLASSENNAM')['KLASSENNAM']
stift2name[0] = 'unknown' # add description 'unknown' to STIFT 0
# Group district heating by own definition (ini) of district heating
# systems.
district_groups = pd.DataFrame(
pd.concat([district_by_stift, stift2name], axis=1)).set_index(
'KLASSENNAM').groupby(by=district_heating_groups).sum()
# Calculate the fraction of each distric heating group.
frac_district_groups = district_groups.div(district_groups.sum())
# Create standardised heat load profile for each group
for nr in frac_district_groups.index:
heat_profiles[nr] = (
(heat_profiles['district_heating_mfh'] *
frac_district_groups.loc[nr, 'district_mfh']) +
(heat_profiles['district_heating_ghd'] *
frac_district_groups.loc[nr, 'district_ghd']))
del heat_profiles['district_heating_ghd']
del heat_profiles['district_heating_mfh']
return heat_profiles
def heat_demand(year):
eb = reegis_tools.energy_balance.get_states_balance(year)
eb.sort_index(inplace=True)
# get fraction of domestic and retail from the german energy balance
share = reegis_tools.energy_balance.get_domestic_retail_share(year)
# Use 0.5 for both sectors if no value is given
share.fillna(0.5, inplace=True)
# Divide domestic and retail by the value of the german energy balance if
# the sum of domestic and retail does not equal the value given in the
# local energy balance.
check_value = True
for state in eb.index.get_level_values(0).unique():
for col in eb.columns:
check = (eb.loc[(state, 'domestic'), col] +
eb.loc[(state, 'retail'), col] -
eb.loc[(state, 'domestic and retail'), col]).round()
if check < 0:
for sector in ['domestic', 'retail']:
eb.loc[(state, sector), col] = (
eb.loc[(state, 'domestic and retail'), col] *
share.loc[col, sector])
check = (eb.loc[(state, 'domestic'), col] +
eb.loc[(state, 'retail'), col] -
eb.loc[(state, 'domestic and retail'), col]).round()
if check < 0:
logging.error("In {0} the {1} sector results {2}".format(
state, col, check))
check_value = False
if check_value:
logging.debug("Divides 'domestic and retail' without errors.")
# Reduce energy balance to the needed columns and group by fuel groups.
eb = eb.loc[(slice(None), ['industrial', 'domestic', 'retail']), ]
eb = eb.groupby(by=cfg.get_dict('FUEL_GROUPS_HEAT_DEMAND'), axis=1).sum()
# Remove empty columns
for col in eb.columns:
if not (eb.loc[(slice(None), 'domestic'), col].sum() > 0
or eb.loc[(slice(None), 'retail'), col].sum() > 0
or eb.loc[(slice(None), 'industrial'), col].sum() > 0):
del eb[col]
# The use of electricity belongs to the electricity sector. It is possible
# to connect it to the heating sector for future scenarios.
del eb['electricity']
del eb['total'] # if electricity is removed total is not correct anymore.
# get fraction of mechanical energy use and subtract it from the balance to
# get the use of heat only.
share_mech = share_of_mechanical_energy_bmwi(year)
for c in share_mech.columns:
for i in share_mech.index:
eb.loc[(slice(None), c),
i] -= (eb.loc[(slice(None), c), i] * share_mech.loc[i, c])
eb.sort_index(inplace=True)
return eb
def get_heat_profiles_de21(year, time_index=None, keep_unit=False):
heat_demand_state_file = os.path.join(
cfg.get('paths', 'demand'),
cfg.get('demand', 'heat_profile_state').format(year=year))
if os.path.isfile(heat_demand_state_file):
logging.info("Demand profiles by state exist. Reading file.")
demand_state = pd.read_csv(heat_demand_state_file,
index_col=[0],
parse_dates=True,
header=[0, 1, 2])
demand_state = demand_state.tz_localize('UTC').tz_convert(
'Europe/Berlin')
else:
demand_state = get_heat_profiles_by_state(year, to_csv=True)
my_index = demand_state.index
my_columns1 = pd.MultiIndex(levels=[[], []], labels=[[], []])
demand_region = pd.DataFrame(index=my_index, columns=my_columns1)
my_columns2 = pd.MultiIndex(levels=[[], [], [], []],
labels=[[], [], [], []])
district_heat_region = pd.DataFrame(index=my_index, columns=my_columns2)
logging.info("Fetching inhabitants table.")
my_ew = de21.inhabitants.get_ew_by_de21_subregions(year)
my_ew = my_ew.replace({'state': cfg.get_dict('STATES')})
state_ew = my_ew.groupby('state').sum()
for region in my_ew.index:
my_ew.loc[region, 'share_state'] = float(
my_ew.loc[region, 'ew'] / state_ew.loc[my_ew.loc[region, 'state']])
logging.info("Convert demand profile...")
fuels = demand_state.columns.get_level_values(2).unique()
sectors = demand_state.columns.get_level_values(1).unique()
demand_state = demand_state.swaplevel(2, 0, axis=1)
district_heat_state = None
for fuel in fuels:
if fuel != 'district heating':
demand_region['DE_demand', fuel] = demand_state[fuel].sum(axis=1)
else:
district_heat_state = demand_state[fuel]
for subregion in my_ew.index:
state = my_ew.loc[subregion, 'state']
region = my_ew.loc[subregion, 'region']
share = my_ew.loc[subregion, 'share_state']
for sector in sectors:
district_heat_region[region, 'district_heating', sector,
subregion] = (
district_heat_state[sector, state] *
share)
district_heat_region.sort_index(1, inplace=True)
# print(district_heat_region)
district_heat_region = district_heat_region.groupby(level=[0, 1],
axis=1).sum()
de21_demand = pd.concat([district_heat_region, demand_region], axis=1)
if time_index is not None:
de21_demand.index = time_index
if not keep_unit:
msg = ("The unit of the source is 'TJ'. "
"Will be divided by {0} to get 'MWh'.")
converter = 0.0036
de21_demand = de21_demand.div(converter)
logging.warning(msg.format(converter))
return de21_demand
def edit_balance():
"""Fixes the energy balances after analysing them. This is done manually.
"""
# Read energy balance table
ebfile = os.path.join(cfg.get('paths', 'static_sources'),
cfg.get('energy_balance', 'energiebilanzen_laender'))
eb = pd.read_csv(ebfile, index_col=[0, 1, 2]).fillna(0)
eb.rename(columns=cfg.get_dict('COLUMN_TRANSLATION'), inplace=True)
eb.sort_index(0, inplace=True)
eb = eb.apply(lambda x: pd.to_numeric(x, errors='coerce')).fillna(0)
new_index_values = list()
sector = cfg.get_dict('SECTOR')
for value in eb.index.get_level_values(2):
new_index_values.append(sector[value])
eb.index.set_levels(new_index_values, level=2, inplace=True)
# ************************************************************************
# Bavaria (Bayern) - Missing coal values
# Difference between fuel sum and LAK table
missing = {2012: 10529, 2013: 8995, 2014: 9398}
for y in [2012, 2013, 2014]:
fix = missing[y]
# the missing value is added to 'hard coal raw' even though it is not
# specified which hard coal product is missing.
eb.loc[(y, 'BY', 'total'), 'hard coal (raw)'] = fix
# There is a small amount specified in the 'domestic and retail'
# sector.
dom_retail = eb.loc[(y, 'BY', 'domestic and retail'),
'hard coal (raw)']
# The rest of the total hard coal consumption comes from the industrial
# sector.
eb.loc[(y, 'BY', 'industrial'), 'hard coal (raw)'] = fix - dom_retail
# ************************************************************************
# Berlin (Berlin) - corrected values for domestic gas and electricity
# In new publications (e.g. LAK table) these values have changed. The newer
# values will be used.
electricity = {2012: 9150, 2013: 7095}
gas = {2012: -27883, 2013: -13317}
total = {2012: -18733, 2013: -6223}
for row in ['total', 'domestic and retail', 'retail']:
for y in [2012, 2013]:
eb.loc[(y, 'BE', row), 'electricity'] += electricity[y]
eb.loc[(y, 'BE', row), 'natural gas'] += gas[y]
eb.loc[(y, 'BE', row), 'total'] += total[y]
# ************************************************************************
# Saxony-Anhalt (Sachsen Anhalt) - missing values for hard coal, oil and
# other depending on the year. Due to a lack of information the difference
# will be halved between the sectors.
missing = {2012: 5233, 2013: 4396, 2014: 3048}
y = 2012
fix = missing[y]
# the missing value is added to 'hard coal raw' even though it is not
# specified which hard coal product is missing.
eb.loc[(y, 'ST', 'industrial'), 'waste (fossil)'] += fix / 2
eb.loc[(y, 'ST', 'industrial'), 'hard coal (raw)'] += fix / 2
# There is a small amount specified in the 'domestic and retail' sector.
dom_retail_hc = eb.loc[(y, 'ST', 'domestic and retail'), 'hard coal (raw)']
# The rest of the total hard coal consumption comes from the industrial
# sector.
eb.loc[(y, 'ST', 'total'), 'waste (fossil)'] += fix / 2
eb.loc[(y, 'ST', 'total'), 'hard coal (raw)'] += fix / 2 + dom_retail_hc
y = 2013
fix = missing[y]
# the missing value is added to 'hard coal raw' even though it is not
# specified which hard coal product is missing.
eb.loc[(y, 'ST', 'industrial'), 'mineral oil products'] += fix / 2
eb.loc[(y, 'ST', 'industrial'), 'hard coal (raw)'] += fix / 2
# There is a small amount specified in the 'domestic and retail' sector.
dom_retail_hc = eb.loc[(y, 'ST', 'domestic and retail'), 'hard coal (raw)']
dom_retail_oil = eb.loc[(y, 'ST', 'domestic and retail'),
'mineral oil products']
# The rest of the total hard coal consumption comes from the industrial
# sector.
eb.loc[(y, 'ST', 'total'), 'mineral oil products'] += fix / 2 + (
dom_retail_oil)
eb.loc[(y, 'ST', 'total'), 'hard coal (raw)'] += fix / 2 + dom_retail_hc
y = 2014
fix = missing[y]
# the missing value is added to 'hard coal raw' even though it is not
# specified which hard coal product is missing.
eb.loc[(y, 'ST', 'industrial'), 'mineral oil products'] += fix / 2
eb.loc[(y, 'ST', 'industrial'), 'hard coal (coke)'] += fix / 2
# There is a small amount specified in the 'domestic and retail' sector.
dom_retail = eb.loc[(y, 'ST', 'domestic and retail'),
'mineral oil products']
# The rest of the total hard coal consumption comes from the industrial
# sector.
eb.loc[(y, 'ST', 'total'), 'mineral oil products'] += fix / 2 + dom_retail
eb.loc[(y, 'ST', 'total'), 'hard coal (coke)'] += fix / 2
# ************************************************************************
# Write results to table
fname = os.path.join(cfg.get('paths', 'energy_balance'),
cfg.get('energy_balance', 'energy_balance_edited'))
eb.to_csv(fname)
return fname
def check_balance(orig, ebfile):
logging.info('Analyse the energy balances')
years = [2012, 2013, 2014]
# energy balance
file_type = ebfile.split('.')[1]
if file_type == 'xlsx' or file_type == 'xls':
eb = pd.read_excel(ebfile, index_col=[0, 1, 2]).fillna(0)
elif file_type == 'csv':
eb = pd.read_csv(ebfile, index_col=[0, 1, 2]).fillna(0)
else:
logging.error('.{0} is an invalid suffix.'.format(file_type))
logging.error('Cannot load {0}'.format(ebfile))
eb = None
exit(0)
eb.rename(columns=cfg.get_dict('COLUMN_TRANSLATION'), inplace=True)
eb.sort_index(0, inplace=True)
eb = eb.apply(lambda x: pd.to_numeric(x, errors='coerce')).fillna(0)
eb = eb.groupby(by=cfg.get_dict('FUEL_GROUPS'), axis=1).sum()
# sum table (fuel)
ftfile = os.path.join(cfg.get('paths', 'static_sources'),
'sum_table_fuel_groups.csv')
ft = pd.read_csv(ftfile)
states = cfg.get_dict('STATES')
ft['Bundesland'] = ft['Bundesland'].apply(lambda x: states[x])
ft.set_index(['Jahr', 'Bundesland'], inplace=True)
ft.rename(columns=cfg.get_dict('SUM_COLUMNS'), inplace=True)
ft.sort_index(inplace=True)
# sum table (sector)
stfile = os.path.join(cfg.get('paths', 'static_sources'),
'sum_table_sectors.csv')
st = pd.read_csv(stfile)
st['Bundesland'] = st['Bundesland'].apply(lambda x: states[x])
st.set_index(['Jahr', 'Bundesland'], inplace=True)
st.rename(columns=cfg.get_dict('SECTOR_SHORT'), inplace=True)
st.sort_index(inplace=True)
del st['Anm.']
if orig:
outfile = os.path.join(cfg.get('paths', 'messages'),
'energy_balance_check_original.xlsx')
else:
outfile = os.path.join(cfg.get('paths', 'messages'),
'energy_balance_check_edited.xlsx')
writer = pd.ExcelWriter(outfile)
for year in years:
# Compare sum of fuel groups with LAK-table
endenergie_check = pd.DataFrame()
for col in ft.columns:
ft_piece = ft.loc[(year, slice(None)), col]
ft_piece.index = ft_piece.index.droplevel([0])
ft_piece = ft_piece.apply(lambda x: pd.to_numeric(x,
errors='coerce'))
try:
eb_piece = eb.loc[(year, slice(None), 'Endenergieverbrauch'),
col]
except KeyError:
eb_piece = eb.loc[(year, slice(None), 'total'), col]
eb_piece.index = eb_piece.index.droplevel([0, 2])
endenergie_check[col] = ft_piece-eb_piece.round()
endenergie_check['check'] = (endenergie_check.sum(1) -
2 * endenergie_check['total'])
endenergie_check.loc['all'] = endenergie_check.sum()
endenergie_check.to_excel(writer, 'fuel_groups_{0}'.format(year),
freeze_panes=(1, 1))
# Compare subtotal of transport, industrial and domestic and retail
# with the total of end-energy
endenergie_summe = pd.DataFrame()
if orig:
main_cat = [
'Haushalte, Gewerbe, Handel, Dienstleistungen,'
' übrige Verbraucher', 'Verkehr insgesamt',
'Gewinngung und verarbeitendes Gewerbe']
total = 'Endenergieverbrauch'
else:
main_cat = [
'domestic and retail', 'transport',
'industrial']
total = 'total'
for state in eb.index.get_level_values(1).unique():
try:
tmp = pd.DataFrame()
n = 0
for idx in main_cat:
n += 1
tmp[state, n] = eb.loc[year, state, idx]
tmp = (tmp.sum(1) - eb.loc[year, state, total]
).round()
endenergie_summe[state] = tmp
except KeyError:
endenergie_summe[state] = None
endenergie_summe.transpose().to_excel(
writer, 'internal sum check {0}'.format(year), freeze_panes=(1, 1))
# Compare sum of sector groups with LAK-table
eb_fuel = (eb[['hard coal', 'lignite', 'oil', 'gas', 're',
'electricity', 'district heating', 'other']])
eb_fuel = eb_fuel.sum(1)
eb_sector = eb_fuel.round().unstack()
eb_sector.rename(columns=cfg.get_dict('SECTOR_SHORT'), inplace=True)
eb_sector.rename(columns=cfg.get_dict('SECTOR_SHORT_EN'), inplace=True)
try:
del eb_sector['ghd']
del eb_sector['dom']
except KeyError:
del eb_sector['retail']
del eb_sector['domestic']
eb_sector = eb_sector.sort_index(1).loc[year]
st_year = st.sort_index(1).loc[year]
st_year.index = st_year.index
st_year = st_year.apply(
lambda x: pd.to_numeric(x, errors='coerce')).fillna(0)
(eb_sector.astype(int) - st_year.astype(int)).to_excel(
writer, 'sector_groups_{0}'.format(year), freeze_panes=(1, 1))
# Compare the sum of the columns with the "total" column.
sum_check_hrz = pd.DataFrame()
for row in eb.index.get_level_values(2).unique():
eb.sort_index(0, inplace=True)
summe = (eb.loc[(year, slice(None), row)]).sum(1)
ges = (eb.loc[(year, slice(None), row), 'total'])
tmp_check = round(summe - 2 * ges)
tmp_check.index = tmp_check.index.droplevel(0)
tmp_check.index = tmp_check.index.droplevel(1)
sum_check_hrz[row] = tmp_check
sum_check_hrz.to_excel(
writer, 'sum_check_hrz_{0}'.format(year), freeze_panes=(1, 1))
# Check states
for state, abr in states.items():
if abr not in eb.loc[year].index.get_level_values(0).unique():
logging.warning(
'{0} ({1}) not present in the {2} balance.'.format(
state, abr, year))
writer.save()