class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
"""
def __init__(self, scenario_id, api_run=False):
self.scenario_id = scenario_id
self.scenario = Scenario(self.scenario_id)
self.api_run = api_run
self.outputs = Output()
self.demand = Demand(self.scenario)
self.supply = None
self.demand_solved, self.supply_solved = False, False
def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results, rio_scenario):
#try:
self.scenario_id = scenario_id
self.scenario = Scenario(self.scenario_id)
self.rio_scenario = rio_scenario
if solve_demand and not (load_demand or load_supply):
self.calculate_demand(save_models)
if not append_results:
self.remove_old_results()
# it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
# self.demand.create_electricity_reconciliation()
# self.demand.output_subsector_electricity_profiles()
self.export_result_to_csv('demand_outputs')
if solve_supply:
if load_demand:
# if we are loading the demand, we are changing the supply measures and want to reload our scenarios
self.scenario = Scenario(self.scenario_id)
self.supply = Supply(self.scenario, demand_object=self.demand,rio_scenario=rio_scenario)
self.calculate_supply(save_models)
if load_demand and solve_supply and export_results:
# we do this now because we delayed before
self.export_result_to_csv('demand_outputs')
if self.supply_solved and export_results and (load_supply or solve_supply):
self.supply.calculate_supply_outputs()
self.pass_supply_results_back_to_demand()
self.calculate_combined_results()
self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
self.export_result_to_csv('supply_outputs')
self.export_result_to_csv('combined_outputs')
self.export_io()
#except:
# pickle the model in the event that it crashes
#if save_models:
# if cfg.rio_supply_run:
# Output.pickle(self, file_name=self.rio_scenario + cfg.model_error_append_name, path=cfg.workingdir)
# else:
# Output.pickle(self, file_name=str(self.scenario_id) + cfg.model_error_append_name,
# path=cfg.workingdir)
def calculate_demand(self, save_models):
self.demand.setup_and_solve()
self.demand_solved = True
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_d_payback()
self.calculate_d_payback_energy()
if save_models:
if cfg.rio_supply_run:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)
else:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)
def calculate_supply(self, save_models):
if not self.demand_solved:
raise ValueError('demand must be solved first before supply')
logging.info('Configuring energy system supply')
self.supply.add_nodes()
self.supply.add_measures()
self.supply.initial_calculate()
self.supply.calculated_years = []
self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
self.supply.final_calculate()
self.supply_solved = True
if save_models:
if cfg.rio_supply_run:
Output.pickle(self, file_name=self.rio_scenario + cfg.full_model_append_name, path=cfg.workingdir)
else:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
# we don't need the demand side object any more, so we can remove it to save drive space
# if not cfg.rio_supply_run:
# if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
# os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))
def pass_supply_results_back_to_demand(self):
# we need to geomap to the combined output geography
emissions_demand_link = GeoMapper.geo_map(self.supply.emissions_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
demand_emissions_rates = GeoMapper.geo_map(self.supply.demand_emissions_rates, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
energy_demand_link = GeoMapper.geo_map(self.supply.energy_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
cost_demand_link = GeoMapper.geo_map(self.supply.cost_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
logging.info("Calculating link to supply")
self.demand.link_to_supply(emissions_demand_link, demand_emissions_rates, energy_demand_link, cost_demand_link)
if cfg.output_tco == 'true':
if hasattr(self,'d_energy_tco'):
self.demand.link_to_supply_tco(emissions_demand_link, demand_emissions_rates, cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
if cfg.output_payback == 'true':
if hasattr(self,'demand.d_all_energy_demand_payback'):
self.demand.link_to_supply_payback(emissions_demand_link, demand_emissions_rates, cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
def calculate_combined_results(self):
logging.info("Calculating combined emissions results")
self.calculate_combined_emissions_results()
logging.info("Calculating combined cost results")
self.calculate_combined_cost_results()
logging.info("Calculating combined energy results")
self.calculate_combined_energy_results()
if cfg.output_tco == 'true':
if self.demand.d_energy_tco is not None:
self.calculate_tco()
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_payback()
def remove_old_results(self):
folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
for folder_name in folder_names:
folder = os.path.join(cfg.workingdir, folder_name)
if os.path.isdir(folder):
shutil.rmtree(folder)
def export_result_to_csv(self, result_name):
if result_name=='combined_outputs':
res_obj = self.outputs
elif result_name=='demand_outputs':
res_obj = self.demand.outputs
elif result_name=='supply_outputs':
res_obj = self.supply.outputs
else:
raise ValueError('result_name not recognized')
def clean_and_write(result_df, attribute):
"""
:param result_df: pandas dataframe
:param attribute: string
"""
if cfg.rio_supply_run and self.supply is not None:
keys = [self.supply.rio_scenario.upper(), cfg.timestamp]
else:
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO', 'TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df], keys=[key], names=[name])
result_df = result_df.fillna(0)
if attribute in (
'hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
# Special case for hourly dispatch results where we want to write them outside of supply_outputs
Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
else:
Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, result_name))
for attribute in dir(res_obj):
if isinstance(getattr(res_obj, attribute), list):
for df in getattr(res_obj, attribute):
result_df = getattr(res_obj, 'clean_df')(df)
clean_and_write(result_df,attribute)
elif isinstance(getattr(res_obj, attribute), pd.DataFrame):
result_df = getattr(res_obj, 'clean_df')(getattr(res_obj, attribute))
clean_and_write(result_df, attribute)
else:
continue
def calculate_tco(self):
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = self.demand.d_levelized_costs_tco
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
service_demand_df = self.demand.d_service_demand_tco
service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
keys=keys,names=names)
self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
self.outputs.c_tco[self.outputs.c_tco<0]=0
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
self.outputs.c_tco.columns = [cost_unit.upper()]
self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
def calculate_payback(self):
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
supply_side_df = supply_side_df.sort_index()
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
sales_df = sales_df.reindex(supply_side_df.index).sort_index()
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]
self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
self.outputs.c_payback.columns = [cost_unit.upper()]
self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1
self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
def calculate_d_payback(self):
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]
self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
self.demand.outputs.d_payback.columns = [cost_unit.upper()]
self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
def calculate_d_payback_energy(self):
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_all_energy_demand_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
# sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
# sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')
def calc_and_format_export_costs(self):
#calculate and format export costs
if self.supply.export_costs is None:
return None
export_costs = GeoMapper.geo_map(self.supply.export_costs.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
export_costs = Output.clean_df(export_costs)
util.replace_index_name(export_costs, 'FINAL_ENERGY', 'SUPPLY_NODE_EXPORT')
export_costs = util.add_to_df_index(export_costs, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["EXPORT", "SUPPLY"])
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
export_costs.columns = [cost_unit.upper()]
return export_costs
def calc_and_format_embodied_costs(self):
#calculate and format embodied supply costs
embodied_costs_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_energy_costs]
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
for embodied_costs in embodied_costs_list: embodied_costs.columns = [cost_unit.upper()]
embodied_costs_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC","SUPPLY"]) for x in embodied_costs_list]
return embodied_costs_list
def calc_and_format_direct_demand_costs(self):
#calculte and format direct demand costs
if self.demand.outputs.d_levelized_costs is None:
return None
direct_costs = GeoMapper.geo_map(self.demand.outputs.d_levelized_costs.copy(), GeoMapper.demand_primary_geography, GeoMapper.combined_outputs_geography, 'total')
direct_costs = direct_costs[direct_costs.index.get_level_values('year').isin(cfg.combined_years_subset)]
levels_to_keep = [x for x in cfg.output_combined_levels if x in direct_costs.index.names]
direct_costs = direct_costs.groupby(level=levels_to_keep).sum()
direct_costs = Output.clean_df(direct_costs)
direct_costs = util.add_to_df_index(direct_costs, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC","DEMAND"])
return direct_costs
def calculate_combined_cost_results(self):
cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
export_costs = self.calc_and_format_export_costs()
embodied_costs_list = self.calc_and_format_embodied_costs()
direct_costs = self.calc_and_format_direct_demand_costs()
export_costs = util.add_and_set_index(export_costs,['COST_TYPE'],['EXPORTED'])
embodied_costs_list = [util.add_and_set_index(x,['COST_TYPE'],['SUPPLY-SIDE']) for x in embodied_costs_list]
direct_costs = util.add_and_set_index(direct_costs,['COST_TYPE'],['DEMAND-SIDE'])
if export_costs is not None:
for name in [x for x in embodied_costs_list[0].index.names if x not in export_costs.index.names]:
export_costs[name] = "N/A"
export_costs.set_index(name,append=True,inplace=True)
export_costs = export_costs.groupby(level=embodied_costs_list[0].index.names).sum()
if direct_costs is not None:
for name in [x for x in embodied_costs_list[0].index.names if x not in direct_costs.index.names]:
direct_costs[name] = "N/A"
direct_costs.set_index(name, append=True, inplace=True)
direct_costs = direct_costs.groupby(level=embodied_costs_list[0].index.names).sum()
self.outputs.c_costs = embodied_costs_list + [direct_costs] + [export_costs]
self.outputs.c_costs= [x[x.values!=0] for x in self.outputs.c_costs]
for x in self.outputs.c_costs: x.index = x.index.reorder_levels(embodied_costs_list[0].index.names)
def calc_and_format_export_emissions(self):
#calculate and format export emissions
if self.supply.export_emissions is None:
return None
export_emissions = GeoMapper.geo_map(self.supply.export_emissions.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(export_emissions, GeoMapper.supply_primary_geography +'_supply')
export_emissions = Output.clean_df(export_emissions)
util.replace_index_name(export_emissions, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
index_names = export_emissions.index.names
export_emissions = export_emissions.reset_index()
export_emissions['FINAL_ENERGY'] = 'export ' + export_emissions['FINAL_ENERGY']
export_emissions = export_emissions.set_index(index_names).sort_index()
export_emissions = util.add_to_df_index(export_emissions, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["EXPORT", "SUPPLY"])
return export_emissions
def calc_and_format_embodied_supply_emissions(self):
# calculate and format embodied supply emissions
embodied_emissions_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_emissions]
embodied_emissions_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC", "SUPPLY"]) for x in embodied_emissions_list]
return embodied_emissions_list
def calc_and_format_direct_demand_emissions(self):
#calculte and format direct demand emissions
direct_emissions_list = [Output.clean_df(x) for x in self.demand.outputs.demand_direct_emissions]
direct_emissions_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC", "DEMAND"]) for x in direct_emissions_list]
if GeoMapper.combined_outputs_geography + '_supply' in cfg.output_combined_levels:
keys = direct_emissions_list[0].index.get_level_values(GeoMapper.combined_outputs_geography.upper()).values
names = GeoMapper.combined_outputs_geography.upper() + '_SUPPLY'
for x in direct_emissions_list:
x[names] = keys
direct_emissions_list = [x.set_index(names, append=True, inplace=True) for x in direct_emissions_list]
return direct_emissions_list
def calculate_combined_emissions_results(self):
export_emissions = self.calc_and_format_export_emissions()
embodied_emissions_list = self.calc_and_format_embodied_supply_emissions()
direct_emissions_list = self.calc_and_format_direct_demand_emissions()
export_emissions = util.add_and_set_index(export_emissions,['EMISSIONS_TYPE'],['EXPORTED'])
embodied_emissions_list = [util.add_and_set_index(x, ['EMISSIONS_TYPE'], ['SUPPLY_SIDE']) for x in embodied_emissions_list]
direct_emissions_list = [util.add_and_set_index(x,['EMISSIONS_TYPE'],['DEMAND_SIDE']) for x in direct_emissions_list]
if export_emissions is not None:
for name in [x for x in embodied_emissions_list[0].index.names if x not in export_emissions.index.names]:
export_emissions[name] = "N/A"
export_emissions.set_index(name,append=True,inplace=True)
export_emissions = export_emissions.groupby(level=embodied_emissions_list[0].index.names).sum()
if direct_emissions_list is not None:
for df in direct_emissions_list:
for name in [x for x in embodied_emissions_list[0].index.names if x not in df.index.names]:
df[name] = "N/A"
df.set_index(name,append=True,inplace=True)
self.outputs.c_emissions = [export_emissions] + embodied_emissions_list + direct_emissions_list
self.outputs.c_emissions = [util.replace_index_name(x, GeoMapper.combined_outputs_geography.upper() +'-EMITTED', GeoMapper.combined_outputs_geography.upper() +'_SUPPLY',inplace=True) for x in self.outputs.c_emissions]
self.outputs.c_emissions = [util.replace_index_name(x, GeoMapper.combined_outputs_geography.upper() +'-CONSUMED', GeoMapper.combined_outputs_geography.upper(),inplace=True) for x in self.outputs.c_emissions]
self.outputs.c_emissions = [x[x['VALUE']!=0] for x in self.outputs.c_emissions]
emissions_unit = cfg.getParam('mass_unit')
for x in self.outputs.c_emissions:
x.columns = [emissions_unit.upper()]
for x in self.outputs.c_emissions: x.index = x.index.reorder_levels([l for l in embodied_emissions_list[0].index.names if l in x.index.names])
def calc_and_format_export_energy(self):
if self.supply.export_energy is None:
return None
export_energy = GeoMapper.geo_map(self.supply.export_energy.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
export_energy = Output.clean_df(export_energy)
util.replace_index_name(export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
export_energy = util.add_to_df_index(export_energy, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=["EXPORT", "EMBODIED"])
for x in cfg.output_combined_levels:
if x not in export_energy.index.names:
export_energy = util.add_and_set_index(export_energy,[x],["N/A"])
return export_energy
def calc_and_format_embodied_supply_energy(self):
embodied_energy_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_energy]
embodied_energy_list = [x[x['VALUE']!=0] for x in embodied_energy_list]
embodied_energy_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=['DOMESTIC','EMBODIED']) for x in embodied_energy_list]
return embodied_energy_list
def calc_and_format_direct_demand_energy(self):
demand_energy = GeoMapper.geo_map(self.demand.outputs.d_energy.copy(), GeoMapper.demand_primary_geography, GeoMapper.combined_outputs_geography, 'total')
demand_energy = Output.clean_df(demand_energy)
demand_energy = demand_energy[demand_energy.index.get_level_values('YEAR')>=cfg.getParamAsInt('current_year')]
demand_energy = util.add_to_df_index(demand_energy, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=['DOMESTIC','FINAL'])
return demand_energy
def calculate_combined_energy_results(self):
export_energy = self.calc_and_format_export_energy()
embodied_energy_list = self.calc_and_format_embodied_supply_energy()
demand_energy = self.calc_and_format_direct_demand_energy()
# reorder levels so dfs match
for name in [x for x in embodied_energy_list[0].index.names if x not in demand_energy.index.names]:
demand_energy[name] = "N/A"
demand_energy.set_index(name, append=True, inplace=True)
demand_energy = demand_energy.groupby(level=embodied_energy_list[0].index.names).sum()
if export_energy is not None:
for name in [x for x in embodied_energy_list[0].index.names if x not in export_energy.index.names]:
export_energy[name] = "N/A"
export_energy.set_index(name,append=True,inplace=True)
export_energy = export_energy.groupby(level=embodied_energy_list[0].index.names).sum()
self.outputs.c_energy = embodied_energy_list + [demand_energy] + [export_energy]
self.outputs.c_energy = [x[x['VALUE']!=0] for x in self.outputs.c_energy]
energy_unit = cfg.calculation_energy_unit
for x in self.outputs.c_energy: x.columns = [energy_unit.upper()]
for x in self.outputs.c_energy: x.index = x.index.reorder_levels(embodied_energy_list[0].index.names)
def export_io(self):
io_table_write_step = cfg.getParamAsInt('io_table_write_step', 'output_detail')
io_table_years = sorted([min(self.supply.years)] + range(max(self.supply.years), min(self.supply.years), -io_table_write_step))
df_list = []
for year in io_table_years:
keys = self.supply.demand_sectors
year_df = pd.concat([self.supply.io_dict[year][sector] for sector in keys], keys=keys,names=['sector'])
year_df = pd.concat([year_df]*len(keys), keys=keys, names=['sector'], axis=1)
df_list.append(year_df)
keys = io_table_years
name = ['year']
df = pd.concat(df_list,keys=keys,names=name)
for row_sector in self.supply.demand_sectors:
for col_sector in self.supply.demand_sectors:
if row_sector != col_sector:
df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
self.supply.outputs.io = df
result_df = self.supply.outputs.return_cleaned_output('io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
# self.export_stacked_io()
def export_stacked_io(self):
df = copy.deepcopy(self.supply.outputs.io)
df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
df = df.stack(level=df.columns.names).to_frame()
df.columns = ['value']
self.supply.outputs.stacked_io = df
result_df = self.supply.outputs.return_cleaned_output('stacked_io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
"""
def __init__(self, scenario_id, api_run=False):
self.scenario_id = scenario_id
self.scenario = Scenario(self.scenario_id)
self.api_run = api_run
self.outputs = Output()
self.demand = Demand(self.scenario)
self.supply = None
self.demand_solved, self.supply_solved = False, False
def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
try:
if solve_demand and not (load_demand or load_supply):
self.calculate_demand(save_models)
if not append_results:
self.remove_old_results()
# it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
self.export_result_to_csv('demand_outputs')
if solve_supply and not load_supply:
if load_demand:
# if we are loading the demand, we are changing the supply measures and want to reload our scenarios
self.scenario = Scenario(self.scenario_id)
self.supply = Supply(self.scenario, demand_object=self.demand)
self.calculate_supply(save_models)
if load_demand and solve_supply:
# we do this now because we delayed before
self.export_result_to_csv('demand_outputs')
if self.supply_solved and export_results and load_supply or solve_supply:
self.supply.calculate_supply_outputs()
self.pass_supply_results_back_to_demand()
self.calculate_combined_results()
self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
if self.api_run:
self.export_results_to_db()
else:
self.export_result_to_csv('supply_outputs')
self.export_result_to_csv('combined_outputs')
self.export_io()
except:
# pickle the model in the event that it crashes
if save_models:
Output.pickle(self, file_name=str(scenario_id) + cfg.model_error_append_name, path=cfg.workingdir)
raise
def calculate_demand(self, save_models):
self.demand.setup_and_solve()
self.demand_solved = True
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_d_payback()
self.calculate_d_payback_energy()
if save_models:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)
def calculate_supply(self, save_models):
if not self.demand_solved:
raise ValueError('demand must be solved first before supply')
logging.info('Configuring energy system supply')
self.supply.add_nodes()
self.supply.add_measures()
self.supply.initial_calculate()
self.supply.calculated_years = []
self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
self.supply.final_calculate()
self.supply_solved = True
if save_models:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
# we don't need the demand side object any more, so we can remove it to save drive space
if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))
def pass_supply_results_back_to_demand(self):
logging.info("Calculating link to supply")
self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
if cfg.output_tco == 'true':
if hasattr(self,'d_energy_tco'):
self.demand.link_to_supply_tco(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
if cfg.output_payback == 'true':
if hasattr(self,'demand.d_all_energy_demand_payback'):
self.demand.link_to_supply_payback(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
def calculate_combined_results(self):
logging.info("Calculating combined emissions results")
self.calculate_combined_emissions_results()
logging.info("Calculating combined cost results")
self.calculate_combined_cost_results()
logging.info("Calculating combined energy results")
self.calculate_combined_energy_results()
if cfg.output_tco == 'true':
if self.demand.d_energy_tco is not None:
self.calculate_tco()
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_payback()
def remove_old_results(self):
folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
for folder_name in folder_names:
folder = os.path.join(cfg.workingdir, folder_name)
if os.path.isdir(folder):
shutil.rmtree(folder)
def export_result_to_csv(self, result_name):
if result_name=='combined_outputs':
res_obj = self.outputs
elif result_name=='demand_outputs':
res_obj = self.demand.outputs
elif result_name=='supply_outputs':
res_obj = self.supply.outputs
else:
raise ValueError('result_name not recognized')
for attribute in dir(res_obj):
if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
continue
result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df], keys=[key], names=[name])
if attribute in ('hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
# Special case for hourly dispatch results where we want to write them outside of supply_outputs
Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
else:
Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))
def export_results_to_db(self):
scenario_run_id = util.active_scenario_run_id(self.scenario_id)
# Levelized costs
costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 1, costs)
#Energy
energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
# Energy demand by sector
util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
# Transportation Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
# Productive Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))
#Emissions
emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
# Annual emissions by sector
util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
# Transportation Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
# Productive Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))
# Domestic emissions per capita
annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
population_driver.index.name = 'YEAR'
factor = 1E6
df = util.DfOper.divi((annual_emissions, population_driver)) * factor
df.columns = ['TONNE PER CAPITA']
util.write_output_to_db(scenario_run_id, 4, df)
# Electricity supply
electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
.xs('EMBODIED', level='ENERGY ACCOUNTING')\
.groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])
def calculate_combined_cost_results(self):
#calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
if self.supply.export_costs is not None:
setattr(self.outputs,'export_costs',self.supply.export_costs)
self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
del self.outputs.export_costs
util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
self.export_costs_df.columns = [cost_unit.upper()]
else:
self.export_costs_df = None
#calculate and format emobodied supply costs
self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
self.embodied_energy_costs_df.columns = [cost_unit.upper()]
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
#calculte and format direct demand costs
self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
if self.demand_costs_df is not None:
levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
self.demand_costs_df = self.demand_costs_df.groupby(level=levels_to_keep).sum()
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
self.outputs.c_costs[self.outputs.c_costs<0]=0
self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
def calculate_tco(self):
# self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
# del self.demand.outputs.demand_embodied_emissions
#calculte and format direct demand emissions
# self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
## del self.demand.outputs.demand_direct_emissions
# emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
# #calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = self.demand.d_levelized_costs_tco
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
service_demand_df = self.demand.d_service_demand_tco
service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
keys=keys,names=names)
self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
self.outputs.c_tco[self.outputs.c_tco<0]=0
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
self.outputs.c_tco.columns = [cost_unit.upper()]
self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
def calculate_payback(self):
# self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
# del self.demand.outputs.demand_embodied_emissions
#calculte and format direct demand emissions
# self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
## del self.demand.outputs.demand_direct_emissions
# emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
# #calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
supply_side_df = supply_side_df.sort_index()
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
sales_df = sales_df.reindex(supply_side_df.index).sort_index()
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]
self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
self.outputs.c_payback.columns = [cost_unit.upper()]
self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1
self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
def calculate_d_payback(self):
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]
self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
self.demand.outputs.d_payback.columns = [cost_unit.upper()]
self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
def calculate_d_payback_energy(self):
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_all_energy_demand_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
# sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
# sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')
def calculate_combined_emissions_results(self):
#calculate and format export emissions
if self.supply.export_emissions is not None:
setattr(self.outputs,'export_emissions',self.supply.export_emissions)
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(self.outputs.export_emissions, cfg.primary_geography +'_supply')
self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
del self.outputs.export_emissions
util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
else:
self.export_emissions_df = None
#calculate and format emobodied supply emissions
self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
# del self.demand.outputs.demand_embodied_emissions
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])
#calculte and format direct demand emissions
self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
# del self.demand.outputs.demand_direct_emissions
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.direct_emissions_df = pd.concat([self.direct_emissions_df], keys=[key], names=[name])
if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
names = cfg.primary_geography.upper() +'_SUPPLY'
self.direct_emissions_df[names] = keys
self.direct_emissions_df.set_index(names,append=True,inplace=True)
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['EMISSIONS TYPE']
self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
emissions_unit = cfg.cfgfile.get('case','mass_unit')
self.outputs.c_emissions.columns = [emissions_unit.upper()]
def calculate_combined_energy_results(self):
energy_unit = cfg.calculation_energy_unit
if self.supply.export_costs is not None:
setattr(self.outputs,'export_energy',self.supply.export_energy)
self.export_energy = self.outputs.return_cleaned_output('export_energy')
del self.outputs.export_energy
util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","EMBODIED"]
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
else:
self.export_energy = None
self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
keys = ['DOMESTIC','EMBODIED']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
keys = ['DOMESTIC','FINAL']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
# self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
self.final_energy[name] = "N/A"
self.final_energy.set_index(name,append=True,inplace=True)
if self.export_energy is not None:
for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
self.export_energy[name] = "N/A"
self.export_energy.set_index(name,append=True,inplace=True)
self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
self.outputs.c_energy.columns = [energy_unit.upper()]
def export_io(self):
io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
df_list = []
for year in io_table_years:
sector_df_list = []
keys = self.supply.demand_sectors
name = ['sector']
for sector in self.supply.demand_sectors:
sector_df_list.append(self.supply.io_dict[year][sector])
year_df = pd.concat(sector_df_list, keys=keys,names=name)
year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
df_list.append(year_df)
keys = io_table_years
name = ['year']
df = pd.concat(df_list,keys=keys,names=name)
for row_sector in self.supply.demand_sectors:
for col_sector in self.supply.demand_sectors:
if row_sector != col_sector:
df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
self.supply.outputs.io = df
result_df = self.supply.outputs.return_cleaned_output('io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
# self.export_stacked_io()
def export_stacked_io(self):
df = copy.deepcopy(self.supply.outputs.io)
df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
df = df.stack(level=df.columns.names).to_frame()
df.columns = ['value']
self.supply.outputs.stacked_io = df
result_df = self.supply.outputs.return_cleaned_output('stacked_io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
Includes the primary geography of the energy system (i.e. country name) as well as the author.
"""
def __init__(self, db_path, cfgfile_path, custom_pint_definitions_path=None, name=None, author=None):
self.model_config(db_path, cfgfile_path, custom_pint_definitions_path)
self.name = cfg.cfgfile.get("case", "scenario") if name is None else name
self.author = cfg.cfgfile.get("case", "author") if author is None else author
self.demand = Demand()
self.supply = Supply()
def model_config(self, db_path, cfgfile_path, custom_pint_definitions_path):
cfg.init_cfgfile(cfgfile_path)
cfg.init_db(db_path)
cfg.init_pint(custom_pint_definitions_path)
cfg.init_geo()
cfg.init_shapes()
cfg.init_outputs_id_map()
def configure_energy_system(self):
print "configuring energy system"
self.configure_demand()
self.configure_supply()
cfg.init_outputs_id_map()
def populate_energy_system(self):
self.populate_demand_system()
self.populate_supply_system()
def populate_measures(self):
self.populate_demand_measures()
self.populate_supply_measures()
def calculate(self):
self.calculate_demand_only()
self.pass_results_to_supply()
self.calculate_supply()
def configure_demand(self):
"""Read in and initialize data"""
# Drivers must come first
self.demand.add_drivers()
# Sectors requires drivers be read in
self.demand.add_sectors()
for sector in self.demand.sectors.values():
# print 'configuring the %s sector' %sector.name
sector.add_subsectors()
def configure_supply(self):
self.supply.add_nodes()
def populate_demand_system(self):
print "remapping drivers"
self.demand.remap_drivers()
print "populating energy system data"
for sector in self.demand.sectors.values():
print " " + sector.name + " sector"
for subsector in sector.subsectors.values():
print " " + subsector.name
subsector.add_energy_system_data()
self.demand.precursor_dict()
def populate_supply_system(self):
self.supply.add_energy_system_data()
def populate_demand_measures(self):
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
subsector.add_measures()
def populate_supply_measures(self):
self.supply.add_measures()
def calculate_demand_only(self):
self.demand.manage_calculations()
def calculate_supply(self):
self.supply.calculate()
def pass_results_to_supply(self):
for sector in self.demand.sectors.values():
sector.aggregate_subsector_energy_for_supply_side()
self.demand.aggregate_sector_energy_for_supply_side()
self.supply.demand_df = self.demand.energy_demand
def pass_results_to_demand(self):
self.demand.aggregate_results()
self.demand.link_to_supply(
self.supply.emissions_demand_link, self.supply.energy_demand_link, self.supply.cost_demand_link
)
def export_results(self, specified_directory=None):
if specified_directory is None:
specified_directory = os.path.join(os.getcwd())
else:
specified_directory = os.path.join(specified_directory)
attributes = dir(self.demand.outputs)
for att in attributes:
if isinstance(getattr(self.demand.outputs, att), pd.core.frame.DataFrame):
output = self.demand.outputs.return_cleaned_output(att)
ExportMethods.writedataframe(att, output, specified_directory)
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
Includes the primary geography of the energy system (i.e. country name) as well as the author.
"""
def __init__(self, cfgfile_path, custom_pint_definitions_path=None, name=None, author=None):
self.cfgfile_path = cfgfile_path
self.custom_pint_definitions_path = custom_pint_definitions_path
self.model_config(cfgfile_path, custom_pint_definitions_path)
self.name = cfg.cfgfile.get('case', 'scenario') if name is None else name
self.author = cfg.cfgfile.get('case', 'author') if author is None else author
self.scenario_dict = dict(zip(util.sql_read_table('Scenarios','id', return_iterable=True, is_active=True),
util.sql_read_table('Scenarios','name', return_iterable=True, is_active=True)))
self.outputs = Output()
self.geography = cfg.cfgfile.get('case', 'primary_geography')
def model_config(self, cfgfile_path, custom_pint_definitions_path):
cfg.init_cfgfile(cfgfile_path)
cfg.init_db()
cfg.path = custom_pint_definitions_path
cfg.init_pint(custom_pint_definitions_path)
cfg.init_geo()
cfg.init_date_lookup()
if shape.shapes.rerun:
shape.shapes.create_empty_shapes()
# shape.shapes.activate_shape(cfg.electricity_energy_type_shape_id)
cfg.init_outputs_id_map()
def configure_energy_system(self):
print 'configuring energy system'
self.demand = Demand(self.cfgfile_path, self.custom_pint_definitions_path)
self.supply = Supply(os.path.join(os.getcwd(),'outputs'),self.cfgfile_path, self.custom_pint_definitions_path)
self.configure_demand()
self.configure_supply()
def populate_energy_system(self):
self.populate_demand_system()
self.populate_supply_system()
def populate_shapes(self):
print 'processing shapes'
if shape.shapes.rerun:
shape.shapes.initiate_active_shapes()
shape.shapes.process_active_shapes()
def populate_measures(self, scenario_id):
self.scenario_id = scenario_id
self.scenario = self.scenario_dict[self.scenario_id]
self.demand_case_id = util.sql_read_table('Scenarios','demand_case',id=self.scenario_id)
self.populate_demand_measures()
self.supply_case_id = util.sql_read_table('Scenarios','supply_case',id=self.scenario_id)
self.populate_supply_measures()
def calculate(self):
self.calculate_demand_only()
self.pass_results_to_supply()
self.calculate_supply()
def configure_demand(self):
"""Read in and initialize data"""
# Drivers must come first
self.demand.add_drivers()
# Sectors requires drivers be read in
self.demand.add_sectors()
for sector in self.demand.sectors.values():
# print 'configuring the %s sector' %sector.name
sector.add_subsectors()
def configure_supply(self):
self.supply.add_node_list()
def populate_demand_system(self):
print 'remapping drivers'
self.demand.remap_drivers()
print 'populating energy system data'
for sector in self.demand.sectors.values():
print ' '+sector.name+' sector'
# print 'reading energy system data for the %s sector' %sector.name
for subsector in sector.subsectors.values():
print ' '+subsector.name
subsector.add_energy_system_data()
sector.precursor_dict()
def populate_supply_system(self):
self.supply.add_nodes()
def populate_demand_measures(self):
for sector in self.demand.sectors.values():
# print 'reading %s measures' %sector.name
for subsector in sector.subsectors.values():
subsector.add_measures(self.demand_case_id)
def populate_supply_measures(self):
self.supply.add_measures(self.supply_case_id)
def calculate_demand_only(self):
self.demand.calculate_demand()
print "aggregating demand results"
self.demand.aggregate_results()
def calculate_supply(self):
self.supply.initial_calculate()
self.supply.calculate_loop()
self.supply.final_calculate()
def pass_results_to_supply(self):
for sector in self.demand.sectors.values():
sector.aggregate_subsector_energy_for_supply_side()
self.demand.aggregate_sector_energy_for_supply_side()
self.supply.demand_object = self.demand
def pass_results_to_demand(self):
print "calculating link to supply"
self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
def calculate_combined_results(self):
print "calculating combined emissions results"
self.calculate_combined_emissions_results()
print "calculating combined cost results"
self.calculate_combined_cost_results()
print "calculating combined energy results"
self.calculate_combined_energy_results()
def export_results(self):
for attribute in dir(self.outputs):
if isinstance(getattr(self.outputs,attribute), pd.DataFrame):
result_df = getattr(self.outputs, attribute)
keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df],keys=[key],names=[name])
ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'combined_outputs'), append_results=True)
for attribute in dir(self.demand.outputs):
if isinstance(getattr(self.demand.outputs,attribute), pd.DataFrame):
result_df = self.demand.outputs.return_cleaned_output(attribute)
keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df],keys=[key],names=[name])
ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'demand_outputs'), append_results=True)
for attribute in dir(self.supply.outputs):
if isinstance(getattr(self.supply.outputs,attribute), pd.DataFrame):
result_df = self.supply.outputs.return_cleaned_output(attribute)
keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df],keys=[key],names=[name])
ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'supply_outputs'), append_results=True)
def calculate_combined_cost_results(self):
#calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
if self.supply.export_costs is not None:
setattr(self.outputs,'export_costs',self.supply.export_costs)
self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
del self.outputs.export_costs
util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
self.export_costs_df.columns = [cost_unit.upper()]
else:
self.export_costs_df = None
#calculate and format emobodied supply costs
self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
self.embodied_energy_costs_df.columns = [cost_unit.upper()]
# del self.demand.outputs.demand_embodied_energy_costs
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
#calculte and format direct demand emissions
self.demand_costs_df= self.demand.outputs.return_cleaned_output('levelized_costs')
# del self.demand.outputs.levelized_costs
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
# levels_to_keep = cfg.output_levels
# levels_to_keep = [x.upper() for x in levels_to_keep]
# levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
# util.replace_index_name(self.outputs.costs, self.geography.upper() +'_EARNED', self.geography.upper() +'_SUPPLY')
# util.replace_index_name(self.outputs.costs, self.geography.upper() +'_CONSUMED', self.geography.upper())
self.outputs.costs[self.outputs.costs<0]=0
self.outputs.costs= self.outputs.costs[self.outputs.costs[cost_unit.upper()]!=0]
# self.outputs.costs.sort(inplace=True)
def calculate_combined_emissions_results(self):
#calculate and format export emissions
if self.supply.export_emissions is not None:
setattr(self.outputs,'export_emissions',self.supply.export_emissions)
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(self.outputs.export_emissions,self.geography +'_supply')
self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
del self.outputs.export_emissions
util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
else:
self.export_emissions_df = None
#calculate and format emobodied supply emissions
self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
# del self.demand.outputs.demand_embodied_emissions
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])
#calculte and format direct demand emissions
self.direct_emissions_df= self.demand.outputs.return_cleaned_output('demand_direct_emissions')
# del self.demand.outputs.demand_direct_emissions
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.direct_emissions_df = pd.concat([self.direct_emissions_df],keys=[key],names=[name])
if 'supply_geography' in cfg.output_combined_levels:
keys = self.direct_emissions_df.index.get_level_values(self.geography.upper()).values
names = self.geography.upper() +'_SUPPLY'
self.direct_emissions_df[names] = keys
self.direct_emissions_df.set_index(names,append=True,inplace=True)
# levels_to_keep = cfg.output_levels
# levels_to_keep = [x.upper() for x in levels_to_keep]
# levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['EMISSIONS TYPE']
self.outputs.emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
# util.replace_index_name(self.outputs.emissions, "ENERGY","FINAL_ENERGY")
util.replace_index_name(self.outputs.emissions, self.geography.upper() +'_EMITTED', self.geography.upper() +'_SUPPLY')
util.replace_index_name(self.outputs.emissions, self.geography.upper() +'_CONSUMED', self.geography.upper())
self.outputs.emissions= self.outputs.emissions[self.outputs.emissions['VALUE']!=0]
emissions_unit = cfg.cfgfile.get('case','mass_unit')
self.outputs.emissions.columns = [emissions_unit.upper()]
# self.outputs.emissions.sort(inplace=True)
def calculate_combined_energy_results(self):
self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
self.final_energy = self.demand.outputs.return_cleaned_output('energy')
self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
self.embodied_energy['ENERGY ACCOUNTING'] = 'EMBODIED'
self.final_energy['ENERGY ACCOUNTING'] = 'FINAL'
self.embodied_energy.set_index('ENERGY ACCOUNTING',append=True,inplace=True)
self.final_energy.set_index('ENERGY ACCOUNTING',append=True,inplace=True)
# self.outputs.energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
self.final_energy[name] = "N/A"
self.final_energy.set_index(name,append=True,inplace=True)
self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
self.outputs.energy = pd.concat([self.embodied_energy,self.final_energy])
self.outputs.energy= self.outputs.energy[self.outputs.energy['VALUE']!=0]
energy_unit = cfg.cfgfile.get('case','energy_unit')
self.outputs.energy.columns = [energy_unit.upper()]
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
Includes the primary geography of the energy system (i.e. country name) as well as the author.
"""
def __init__(self,
db_path,
cfgfile_path,
custom_pint_definitions_path=None,
name=None,
author=None):
self.model_config(db_path, cfgfile_path, custom_pint_definitions_path)
self.name = cfg.cfgfile.get('case',
'scenario') if name is None else name
self.author = cfg.cfgfile.get('case',
'author') if author is None else author
self.demand = Demand()
self.supply = Supply()
def model_config(self, db_path, cfgfile_path,
custom_pint_definitions_path):
cfg.init_cfgfile(cfgfile_path)
cfg.init_db(db_path)
cfg.init_pint(custom_pint_definitions_path)
cfg.init_geo()
cfg.init_shapes()
cfg.init_outputs_id_map()
def configure_energy_system(self):
print 'configuring energy system'
self.configure_demand()
self.configure_supply()
cfg.init_outputs_id_map()
def populate_energy_system(self):
self.populate_demand_system()
self.populate_supply_system()
def populate_measures(self):
self.populate_demand_measures()
self.populate_supply_measures()
def calculate(self):
self.calculate_demand_only()
self.pass_results_to_supply()
self.calculate_supply()
def configure_demand(self):
"""Read in and initialize data"""
# Drivers must come first
self.demand.add_drivers()
# Sectors requires drivers be read in
self.demand.add_sectors()
for sector in self.demand.sectors.values():
# print 'configuring the %s sector' %sector.name
sector.add_subsectors()
def configure_supply(self):
self.supply.add_nodes()
def populate_demand_system(self):
print 'remapping drivers'
self.demand.remap_drivers()
print 'populating energy system data'
for sector in self.demand.sectors.values():
print ' ' + sector.name + ' sector'
for subsector in sector.subsectors.values():
print ' ' + subsector.name
subsector.add_energy_system_data()
self.demand.precursor_dict()
def populate_supply_system(self):
self.supply.add_energy_system_data()
def populate_demand_measures(self):
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
subsector.add_measures()
def populate_supply_measures(self):
self.supply.add_measures()
def calculate_demand_only(self):
self.demand.manage_calculations()
def calculate_supply(self):
self.supply.calculate()
def pass_results_to_supply(self):
for sector in self.demand.sectors.values():
sector.aggregate_subsector_energy_for_supply_side()
self.demand.aggregate_sector_energy_for_supply_side()
self.supply.demand_df = self.demand.energy_demand
def pass_results_to_demand(self):
self.demand.aggregate_results()
self.demand.link_to_supply(self.supply.emissions_demand_link,
self.supply.energy_demand_link,
self.supply.cost_demand_link)
def export_results(self, specified_directory=None):
if specified_directory is None:
specified_directory = os.path.join(os.getcwd())
else:
specified_directory = os.path.join(specified_directory)
attributes = dir(self.demand.outputs)
for att in attributes:
if isinstance(getattr(self.demand.outputs, att),
pd.core.frame.DataFrame):
output = self.demand.outputs.return_cleaned_output(att)
ExportMethods.writedataframe(att, output, specified_directory)
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
Includes the primary geography of the energy system (i.e. country name) as well as the author.
"""
def __init__(self,
cfgfile_path,
custom_pint_definitions_path=None,
name=None,
author=None):
self.cfgfile_path = cfgfile_path
self.custom_pint_definitions_path = custom_pint_definitions_path
self.model_config(cfgfile_path, custom_pint_definitions_path)
self.name = cfg.cfgfile.get('case',
'scenario') if name is None else name
self.author = cfg.cfgfile.get('case',
'author') if author is None else author
self.scenario_dict = dict(
zip(
util.sql_read_table('Scenarios',
'id',
return_iterable=True,
is_active=True),
util.sql_read_table('Scenarios',
'name',
return_iterable=True,
is_active=True)))
self.outputs = Output()
self.geography = cfg.cfgfile.get('case', 'primary_geography')
def model_config(self, cfgfile_path, custom_pint_definitions_path):
cfg.init_cfgfile(cfgfile_path)
cfg.init_db()
cfg.path = custom_pint_definitions_path
cfg.init_pint(custom_pint_definitions_path)
cfg.init_geo()
cfg.init_date_lookup()
if shape.shapes.rerun:
shape.shapes.create_empty_shapes()
# shape.shapes.activate_shape(cfg.electricity_energy_type_shape_id)
cfg.init_outputs_id_map()
def configure_energy_system(self):
print 'configuring energy system'
self.demand = Demand(self.cfgfile_path,
self.custom_pint_definitions_path)
self.supply = Supply(os.path.join(os.getcwd(),
'outputs'), self.cfgfile_path,
self.custom_pint_definitions_path)
self.configure_demand()
self.configure_supply()
def populate_energy_system(self):
self.populate_demand_system()
self.populate_supply_system()
def populate_shapes(self):
print 'processing shapes'
if shape.shapes.rerun:
shape.shapes.initiate_active_shapes()
shape.shapes.process_active_shapes()
def populate_measures(self, scenario_id):
self.scenario_id = scenario_id
self.scenario = self.scenario_dict[self.scenario_id]
self.demand_case_id = util.sql_read_table('Scenarios',
'demand_case',
id=self.scenario_id)
self.populate_demand_measures()
self.supply_case_id = util.sql_read_table('Scenarios',
'supply_case',
id=self.scenario_id)
self.populate_supply_measures()
def calculate(self):
self.calculate_demand_only()
self.pass_results_to_supply()
self.calculate_supply()
def configure_demand(self):
"""Read in and initialize data"""
# Drivers must come first
self.demand.add_drivers()
# Sectors requires drivers be read in
self.demand.add_sectors()
for sector in self.demand.sectors.values():
# print 'configuring the %s sector' %sector.name
sector.add_subsectors()
def configure_supply(self):
self.supply.add_node_list()
def populate_demand_system(self):
print 'remapping drivers'
self.demand.remap_drivers()
print 'populating energy system data'
for sector in self.demand.sectors.values():
print ' ' + sector.name + ' sector'
# print 'reading energy system data for the %s sector' %sector.name
for subsector in sector.subsectors.values():
print ' ' + subsector.name
subsector.add_energy_system_data()
sector.precursor_dict()
def populate_supply_system(self):
self.supply.add_nodes()
def populate_demand_measures(self):
for sector in self.demand.sectors.values():
# print 'reading %s measures' %sector.name
for subsector in sector.subsectors.values():
subsector.add_measures(self.demand_case_id)
def populate_supply_measures(self):
self.supply.add_measures(self.supply_case_id)
def calculate_demand_only(self):
self.demand.calculate_demand()
print "aggregating demand results"
self.demand.aggregate_results()
def calculate_supply(self):
self.supply.initial_calculate()
self.supply.calculate_loop()
self.supply.final_calculate()
def pass_results_to_supply(self):
for sector in self.demand.sectors.values():
sector.aggregate_subsector_energy_for_supply_side()
self.demand.aggregate_sector_energy_for_supply_side()
self.supply.demand_object = self.demand
def pass_results_to_demand(self):
print "calculating link to supply"
self.demand.link_to_supply(self.supply.emissions_demand_link,
self.supply.demand_emissions_rates,
self.supply.energy_demand_link,
self.supply.cost_demand_link)
def calculate_combined_results(self):
print "calculating combined emissions results"
self.calculate_combined_emissions_results()
print "calculating combined cost results"
self.calculate_combined_cost_results()
print "calculating combined energy results"
self.calculate_combined_energy_results()
def export_results(self):
for attribute in dir(self.outputs):
if isinstance(getattr(self.outputs, attribute), pd.DataFrame):
result_df = getattr(self.outputs, attribute)
keys = [
self.scenario.upper(),
str(datetime.now().replace(second=0, microsecond=0))
]
names = ['SCENARIO', 'TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df],
keys=[key],
names=[name])
ExportMethods.writeobj(attribute,
result_df,
os.path.join(os.getcwd(),
'combined_outputs'),
append_results=True)
for attribute in dir(self.demand.outputs):
if isinstance(getattr(self.demand.outputs, attribute),
pd.DataFrame):
result_df = self.demand.outputs.return_cleaned_output(
attribute)
keys = [
self.scenario.upper(),
str(datetime.now().replace(second=0, microsecond=0))
]
names = ['SCENARIO', 'TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df],
keys=[key],
names=[name])
ExportMethods.writeobj(attribute,
result_df,
os.path.join(os.getcwd(),
'demand_outputs'),
append_results=True)
for attribute in dir(self.supply.outputs):
if isinstance(getattr(self.supply.outputs, attribute),
pd.DataFrame):
result_df = self.supply.outputs.return_cleaned_output(
attribute)
keys = [
self.scenario.upper(),
str(datetime.now().replace(second=0, microsecond=0))
]
names = ['SCENARIO', 'TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df],
keys=[key],
names=[name])
ExportMethods.writeobj(attribute,
result_df,
os.path.join(os.getcwd(),
'supply_outputs'),
append_results=True)
def calculate_combined_cost_results(self):
#calculate and format export costs
cost_unit = cfg.cfgfile.get(
'case', 'currency_year_id') + " " + cfg.cfgfile.get(
'case', 'currency_name')
if self.supply.export_costs is not None:
setattr(self.outputs, 'export_costs', self.supply.export_costs)
self.export_costs_df = self.outputs.return_cleaned_output(
'export_costs')
del self.outputs.export_costs
util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY',
'SUPPLY_NODE_EXPORT')
keys = ["EXPORT", "SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.export_costs_df = pd.concat([self.export_costs_df],
keys=[key],
names=[name])
self.export_costs_df.columns = [cost_unit.upper()]
else:
self.export_costs_df = None
#calculate and format emobodied supply costs
self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output(
'demand_embodied_energy_costs')
self.embodied_energy_costs_df.columns = [cost_unit.upper()]
# del self.demand.outputs.demand_embodied_energy_costs
keys = ["DOMESTIC", "SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.embodied_energy_costs_df = pd.concat(
[self.embodied_energy_costs_df], keys=[key], names=[name])
#calculte and format direct demand emissions
self.demand_costs_df = self.demand.outputs.return_cleaned_output(
'levelized_costs')
# del self.demand.outputs.levelized_costs
keys = ["DOMESTIC", "DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.demand_costs_df = pd.concat([self.demand_costs_df],
keys=[key],
names=[name])
# levels_to_keep = cfg.output_levels
# levels_to_keep = [x.upper() for x in levels_to_keep]
# levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.costs = util.df_list_concatenate([
self.export_costs_df, self.embodied_energy_costs_df,
self.demand_costs_df
],
keys=keys,
new_names=names)
# util.replace_index_name(self.outputs.costs, self.geography.upper() +'_EARNED', self.geography.upper() +'_SUPPLY')
# util.replace_index_name(self.outputs.costs, self.geography.upper() +'_CONSUMED', self.geography.upper())
self.outputs.costs[self.outputs.costs < 0] = 0
self.outputs.costs = self.outputs.costs[
self.outputs.costs[cost_unit.upper()] != 0]
# self.outputs.costs.sort(inplace=True)
def calculate_combined_emissions_results(self):
#calculate and format export emissions
if self.supply.export_emissions is not None:
setattr(self.outputs, 'export_emissions',
self.supply.export_emissions)
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(self.outputs.export_emissions,
self.geography + '_supply')
self.export_emissions_df = self.outputs.return_cleaned_output(
'export_emissions')
del self.outputs.export_emissions
util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY',
'SUPPLY_NODE_EXPORT')
keys = ["EXPORT", "SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.export_emissions_df = pd.concat(
[self.export_emissions_df], keys=[key], names=[name])
else:
self.export_emissions_df = None
#calculate and format emobodied supply emissions
self.embodied_emissions_df = self.demand.outputs.return_cleaned_output(
'demand_embodied_emissions')
# del self.demand.outputs.demand_embodied_emissions
keys = ["DOMESTIC", "SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.embodied_emissions_df = pd.concat(
[self.embodied_emissions_df], keys=[key], names=[name])
#calculte and format direct demand emissions
self.direct_emissions_df = self.demand.outputs.return_cleaned_output(
'demand_direct_emissions')
# del self.demand.outputs.demand_direct_emissions
keys = ["DOMESTIC", "DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.direct_emissions_df = pd.concat([self.direct_emissions_df],
keys=[key],
names=[name])
if 'supply_geography' in cfg.output_combined_levels:
keys = self.direct_emissions_df.index.get_level_values(
self.geography.upper()).values
names = self.geography.upper() + '_SUPPLY'
self.direct_emissions_df[names] = keys
self.direct_emissions_df.set_index(names,
append=True,
inplace=True)
# levels_to_keep = cfg.output_levels
# levels_to_keep = [x.upper() for x in levels_to_keep]
# levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['EMISSIONS TYPE']
self.outputs.emissions = util.df_list_concatenate([
self.export_emissions_df, self.embodied_emissions_df,
self.direct_emissions_df
],
keys=keys,
new_names=names)
# util.replace_index_name(self.outputs.emissions, "ENERGY","FINAL_ENERGY")
util.replace_index_name(self.outputs.emissions,
self.geography.upper() + '_EMITTED',
self.geography.upper() + '_SUPPLY')
util.replace_index_name(self.outputs.emissions,
self.geography.upper() + '_CONSUMED',
self.geography.upper())
self.outputs.emissions = self.outputs.emissions[
self.outputs.emissions['VALUE'] != 0]
emissions_unit = cfg.cfgfile.get('case', 'mass_unit')
self.outputs.emissions.columns = [emissions_unit.upper()]
# self.outputs.emissions.sort(inplace=True)
def calculate_combined_energy_results(self):
self.embodied_energy = self.demand.outputs.return_cleaned_output(
'demand_embodied_energy')
self.embodied_energy = self.embodied_energy[
self.embodied_energy['VALUE'] != 0]
self.final_energy = self.demand.outputs.return_cleaned_output('energy')
self.final_energy = self.final_energy[
self.final_energy.index.get_level_values('YEAR') >= int(
cfg.cfgfile.get('case', 'current_year'))]
self.embodied_energy['ENERGY ACCOUNTING'] = 'EMBODIED'
self.final_energy['ENERGY ACCOUNTING'] = 'FINAL'
self.embodied_energy.set_index('ENERGY ACCOUNTING',
append=True,
inplace=True)
self.final_energy.set_index('ENERGY ACCOUNTING',
append=True,
inplace=True)
# self.outputs.energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
for name in [
x for x in self.embodied_energy.index.names
if x not in self.final_energy.index.names
]:
self.final_energy[name] = "N/A"
self.final_energy.set_index(name, append=True, inplace=True)
self.final_energy = self.final_energy.groupby(
level=self.embodied_energy.index.names).sum()
self.final_energy = self.final_energy.reorder_levels(
self.embodied_energy.index.names)
self.outputs.energy = pd.concat(
[self.embodied_energy, self.final_energy])
self.outputs.energy = self.outputs.energy[
self.outputs.energy['VALUE'] != 0]
energy_unit = cfg.cfgfile.get('case', 'energy_unit')
self.outputs.energy.columns = [energy_unit.upper()]
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
"""
def __init__(self, scenario_id, api_run=False):
self.scenario_id = scenario_id
self.scenario = Scenario(self.scenario_id)
self.api_run = api_run
self.outputs = Output()
self.demand = Demand(self.scenario)
self.supply = None
self.demand_solved, self.supply_solved = False, False
def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
try:
if solve_demand and not (load_demand or load_supply):
self.calculate_demand(save_models)
if not append_results:
self.remove_old_results()
# it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
self.export_result_to_csv('demand_outputs')
if solve_supply and not load_supply:
if load_demand:
# if we are loading the demand, we are changing the supply measures and want to reload our scenarios
self.scenario = Scenario(self.scenario_id)
self.supply = Supply(self.scenario, demand_object=self.demand)
self.calculate_supply(save_models)
if load_demand and solve_supply:
# we do this now because we delayed before
self.export_result_to_csv('demand_outputs')
if self.supply_solved and export_results:
self.supply.calculate_supply_outputs()
self.pass_supply_results_back_to_demand()
self.calculate_combined_results()
self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
if self.api_run:
self.export_results_to_db()
else:
self.export_result_to_csv('supply_outputs')
self.export_result_to_csv('combined_outputs')
self.export_io()
except:
# pickle the model in the event that it crashes
if save_models:
Output.pickle(self, file_name=str(scenario_id) + cfg.model_error_append_name, path=cfg.workingdir)
raise
def calculate_demand(self, save_models):
self.demand.setup_and_solve()
self.demand_solved = True
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_d_payback()
self.calculate_d_payback_energy()
if save_models:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)
def calculate_supply(self, save_models):
if not self.demand_solved:
raise ValueError('demand must be solved first before supply')
logging.info('Configuring energy system supply')
self.supply.add_nodes()
self.supply.add_measures()
self.supply.initial_calculate()
self.supply.calculated_years = []
self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
self.supply.final_calculate()
self.supply_solved = True
if save_models:
Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
# we don't need the demand side object any more, so we can remove it to save drive space
if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))
def pass_supply_results_back_to_demand(self):
logging.info("Calculating link to supply")
self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
if cfg.output_tco == 'true':
if hasattr(self,'d_energy_tco'):
self.demand.link_to_supply_tco(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
if cfg.output_payback == 'true':
if hasattr(self,'demand.d_all_energy_demand_payback'):
self.demand.link_to_supply_payback(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link)
else:
print "demand side has not been run with tco outputs set to 'true'"
def calculate_combined_results(self):
logging.info("Calculating combined emissions results")
self.calculate_combined_emissions_results()
logging.info("Calculating combined cost results")
self.calculate_combined_cost_results()
logging.info("Calculating combined energy results")
self.calculate_combined_energy_results()
if cfg.output_tco == 'true':
if self.demand.d_energy_tco is not None:
self.calculate_tco()
if cfg.output_payback == 'true':
if self.demand.d_all_energy_demand_payback is not None:
self.calculate_payback()
def remove_old_results(self):
folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
for folder_name in folder_names:
folder = os.path.join(cfg.workingdir, folder_name)
if os.path.isdir(folder):
shutil.rmtree(folder)
def export_result_to_csv(self, result_name):
if result_name=='combined_outputs':
res_obj = self.outputs
elif result_name=='demand_outputs':
res_obj = self.demand.outputs
elif result_name=='supply_outputs':
res_obj = self.supply.outputs
else:
raise ValueError('result_name not recognized')
for attribute in dir(res_obj):
if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
continue
result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df], keys=[key], names=[name])
if attribute in ('hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
# Special case for hourly dispatch results where we want to write them outside of supply_outputs
Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
else:
Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))
def export_results_to_db(self):
scenario_run_id = util.active_scenario_run_id(self.scenario_id)
# Levelized costs
costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 1, costs)
#Energy
energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
# Energy demand by sector
util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
# Transportation Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
# Productive Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))
#Emissions
emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
# Annual emissions by sector
util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
# Transportation Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
# Productive Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))
# Domestic emissions per capita
annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
population_driver.index.name = 'YEAR'
factor = 1E6
df = util.DfOper.divi((annual_emissions, population_driver)) * factor
df.columns = ['TONNE PER CAPITA']
util.write_output_to_db(scenario_run_id, 4, df)
# Electricity supply
electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
.xs('EMBODIED', level='ENERGY ACCOUNTING')\
.groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])
def calculate_combined_cost_results(self):
#calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
if self.supply.export_costs is not None:
setattr(self.outputs,'export_costs',self.supply.export_costs)
self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
del self.outputs.export_costs
util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
self.export_costs_df.columns = [cost_unit.upper()]
else:
self.export_costs_df = None
#calculate and format emobodied supply costs
self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
self.embodied_energy_costs_df.columns = [cost_unit.upper()]
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
#calculte and format direct demand costs
self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
if self.demand_costs_df is not None:
levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
self.demand_costs_df = self.demand_costs_df.groupby(level=levels_to_keep).sum()
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
self.outputs.c_costs[self.outputs.c_costs<0]=0
self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
def calculate_tco(self):
# self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
# del self.demand.outputs.demand_embodied_emissions
#calculte and format direct demand emissions
# self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
## del self.demand.outputs.demand_direct_emissions
# emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
# #calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = self.demand.d_levelized_costs_tco
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
service_demand_df = self.demand.d_service_demand_tco
service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
keys=keys,names=names)
self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
self.outputs.c_tco[self.outputs.c_tco<0]=0
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
self.outputs.c_tco.columns = [cost_unit.upper()]
self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
def calculate_payback(self):
# self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
# del self.demand.outputs.demand_embodied_emissions
#calculte and format direct demand emissions
# self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
## del self.demand.outputs.demand_direct_emissions
# emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
# #calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
supply_side_df = supply_side_df.sort_index()
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
sales_df = sales_df.reindex(supply_side_df.index).sort_index()
keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]
self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
self.outputs.c_payback.columns = [cost_unit.upper()]
self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1
self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
def calculate_d_payback(self):
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_annual_costs_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]
self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
self.demand.outputs.d_payback.columns = [cost_unit.upper()]
self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1
self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
def calculate_d_payback_energy(self):
initial_vintage = min(cfg.supply_years)
demand_side_df = self.demand.d_all_energy_demand_payback
demand_side_df.columns = ['value']
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
sales_df = copy.deepcopy(self.demand.outputs.d_sales)
util.replace_index_name(sales_df,'vintage','year')
sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]
sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
# sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
# sales_df = sales_df.reindex(demand_side_df.index).sort_index()
self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
for sector in self.demand.sectors.values():
for subsector in sector.subsectors.values():
if hasattr(subsector,'stock') and subsector.sub_type!='link':
indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')
def calculate_combined_emissions_results(self):
#calculate and format export emissions
if self.supply.export_emissions is not None:
setattr(self.outputs,'export_emissions',self.supply.export_emissions)
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(self.outputs.export_emissions, cfg.primary_geography +'_supply')
self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
del self.outputs.export_emissions
util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
else:
self.export_emissions_df = None
#calculate and format emobodied supply emissions
self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
# del self.demand.outputs.demand_embodied_emissions
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])
#calculte and format direct demand emissions
self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
# del self.demand.outputs.demand_direct_emissions
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key, name in zip(keys, names):
self.direct_emissions_df = pd.concat([self.direct_emissions_df], keys=[key], names=[name])
if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
names = cfg.primary_geography.upper() +'_SUPPLY'
self.direct_emissions_df[names] = keys
self.direct_emissions_df.set_index(names,append=True,inplace=True)
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['EMISSIONS TYPE']
self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
emissions_unit = cfg.cfgfile.get('case','mass_unit')
self.outputs.c_emissions.columns = [emissions_unit.upper()]
def calculate_combined_energy_results(self):
energy_unit = cfg.calculation_energy_unit
if self.supply.export_costs is not None:
setattr(self.outputs,'export_energy',self.supply.export_energy)
self.export_energy = self.outputs.return_cleaned_output('export_energy')
del self.outputs.export_energy
util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","EMBODIED"]
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
else:
self.export_energy = None
self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
keys = ['DOMESTIC','EMBODIED']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
keys = ['DOMESTIC','FINAL']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
# self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
self.final_energy[name] = "N/A"
self.final_energy.set_index(name,append=True,inplace=True)
if self.export_energy is not None:
for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
self.export_energy[name] = "N/A"
self.export_energy.set_index(name,append=True,inplace=True)
self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
self.outputs.c_energy.columns = [energy_unit.upper()]
def export_io(self):
io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
df_list = []
for year in io_table_years:
sector_df_list = []
keys = self.supply.demand_sectors
name = ['sector']
for sector in self.supply.demand_sectors:
sector_df_list.append(self.supply.io_dict[year][sector])
year_df = pd.concat(sector_df_list, keys=keys,names=name)
year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
df_list.append(year_df)
keys = io_table_years
name = ['year']
df = pd.concat(df_list,keys=keys,names=name)
for row_sector in self.supply.demand_sectors:
for col_sector in self.supply.demand_sectors:
if row_sector != col_sector:
df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
self.supply.outputs.io = df
result_df = self.supply.outputs.return_cleaned_output('io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
# self.export_stacked_io()
def export_stacked_io(self):
df = copy.deepcopy(self.supply.outputs.io)
df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
df = df.stack(level=df.columns.names).to_frame()
df.columns = ['value']
self.supply.outputs.stacked_io = df
result_df = self.supply.outputs.return_cleaned_output('stacked_io')
keys = [self.scenario.name.upper(), cfg.timestamp]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
class PathwaysModel(object):
"""
Highest level classification of the definition of an energy system.
"""
def __init__(self, scenario_id, api_run=False):
self.scenario_id = scenario_id
self.api_run = api_run
self.scenario = cfg.scenario_dict[self.scenario_id]
self.demand_case_id = util.sql_read_table('Scenarios', 'demand_case', id=self.scenario_id)
self.supply_case_id = util.sql_read_table('Scenarios', 'supply_case', id=self.scenario_id)
self.outputs = Output()
self.demand = Demand()
self.supply = None
self.demand_solved, self.supply_solved = False, False
def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
try:
if solve_demand and not (load_demand or load_supply):
self.calculate_demand(save_models)
if not append_results:
self.remove_old_results()
if hasattr(self, 'demand_solved') and self.demand_solved and export_results and not self.api_run:
self.export_result_to_csv('demand_outputs')
if solve_supply and not load_supply:
self.supply = Supply(self.scenario, demand_object=self.demand)
self.calculate_supply(save_models)
if hasattr(self, 'supply_solved') and self.supply_solved and export_results:
self.supply.calculate_supply_outputs()
self.pass_supply_results_back_to_demand()
self.calculate_combined_results()
if self.api_run:
self.export_results_to_db()
else:
self.export_result_to_csv('supply_outputs')
self.export_result_to_csv('combined_outputs')
self.export_io()
except:
# pickle the model in the event that it crashes
with open(os.path.join(cfg.workingdir, str(scenario_id) + '_model_error.p'), 'wb') as outfile:
pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)
raise
def calculate_demand(self, save_models):
logging.info('Configuring energy system demand')
self.demand.add_subsectors()
self.demand.add_measures(self.demand_case_id)
self.demand.calculate_demand()
self.demand_solved = True
if save_models:
with open(os.path.join(cfg.workingdir, str(self.scenario_id) + '_model.p'), 'wb') as outfile:
pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)
def calculate_supply(self, save_models):
if not self.demand_solved:
raise ValueError('demand must be solved first before supply')
logging.info('Configuring energy system supply')
self.supply.add_nodes()
self.supply.add_measures(self.supply_case_id)
self.supply.initial_calculate()
self.supply.calculated_years = []
self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
self.supply.final_calculate()
self.supply.concatenate_annual_costs()
self.supply.calculate_capacity_utilization()
self.supply_solved = True
if save_models:
with open(os.path.join(cfg.workingdir, str(self.scenario_id) + '_full_model_run.p'), 'wb') as outfile:
pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)
def pass_supply_results_back_to_demand(self):
logging.info("Calculating link to supply")
self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
def calculate_combined_results(self):
logging.info("Calculating combined emissions results")
self.calculate_combined_emissions_results()
logging.info("Calculating combined cost results")
self.calculate_combined_cost_results()
logging.info("Calculating combined energy results")
self.calculate_combined_energy_results()
def remove_old_results(self):
folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
for folder_name in folder_names:
folder = os.path.join(cfg.workingdir, folder_name)
if os.path.isdir(folder):
shutil.rmtree(folder)
def export_result_to_csv(self, result_name):
if result_name=='combined_outputs':
res_obj = self.outputs
elif result_name=='demand_outputs':
res_obj = self.demand.outputs
elif result_name=='supply_outputs':
res_obj = self.supply.outputs
else:
raise ValueError('result_name not recognized')
for attribute in dir(res_obj):
if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
continue
result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
keys = [self.scenario.upper(),str(datetime.now().replace(second=0, microsecond=0))]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys, names):
result_df = pd.concat([result_df], keys=[key], names=[name])
Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))
def export_results_to_db(self):
scenario_run_id = util.active_scenario_run_id(self.scenario_id)
# Levelized costs
costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 1, costs)
#Energy
energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
# Energy demand by sector
util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
# Transportation Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
# Productive Energy by Fuel Type
util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))
#Emissions
emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
.groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
# Annual emissions by sector
util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
# Residential Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
# Commercial Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
# Transportation Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
# Productive Emissions by Fuel Type
util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))
# Domestic emissions per capita
annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
population_driver.index.name = 'YEAR'
factor = 1E6
df = util.DfOper.divi((annual_emissions, population_driver)) * factor
df.columns = ['TONNE PER CAPITA']
util.write_output_to_db(scenario_run_id, 4, df)
# Electricity supply
electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
.xs('EMBODIED', level='ENERGY ACCOUNTING')\
.groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])
def calculate_combined_cost_results(self):
#calculate and format export costs
cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
if self.supply.export_costs is not None:
setattr(self.outputs,'export_costs',self.supply.export_costs)
self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
del self.outputs.export_costs
util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
self.export_costs_df.columns = [cost_unit.upper()]
else:
self.export_costs_df = None
#calculate and format emobodied supply costs
self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
self.embodied_energy_costs_df.columns = [cost_unit.upper()]
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
#calculte and format direct demand costs
self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
self.demand_costs_df= self.demand_costs_df.groupby(level=levels_to_keep).sum()
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['COST TYPE']
self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
self.outputs.c_costs[self.outputs.c_costs<0]=0
self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
def calculate_combined_emissions_results(self):
#calculate and format export emissions
if self.supply.export_emissions is not None:
setattr(self.outputs,'export_emissions',self.supply.export_emissions)
if 'supply_geography' not in cfg.output_combined_levels:
util.remove_df_levels(self.outputs.export_emissions,cfg.primary_geography +'_supply')
self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
del self.outputs.export_emissions
util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
else:
self.export_emissions_df = None
#calculate and format emobodied supply emissions
self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
# del self.demand.outputs.demand_embodied_emissions
keys = ["DOMESTIC","SUPPLY"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
for key,name in zip(keys,names):
self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])
#calculte and format direct demand emissions
self.direct_emissions_df= self.demand.outputs.return_cleaned_output('demand_direct_emissions')
# del self.demand.outputs.demand_direct_emissions
keys = ["DOMESTIC","DEMAND"]
names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND",cfg.primary_geography.upper() +'_EMITTED']
for key,name in zip(keys,names):
self.direct_emissions_df = pd.concat([self.direct_emissions_df],keys=[key],names=[name])
if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
names = cfg.primary_geography.upper() +'_SUPPLY'
self.direct_emissions_df[names] = keys
self.direct_emissions_df.set_index(names,append=True,inplace=True)
# levels_to_keep = cfg.output_levels
# levels_to_keep = [x.upper() for x in levels_to_keep]
# levels_to_keep += names + [cfg.primary_geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
names = ['EMISSIONS TYPE']
self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
# util.replace_index_name(self.outputs.c_emissions, "ENERGY","FINAL_ENERGY")
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
emissions_unit = cfg.cfgfile.get('case','mass_unit')
self.outputs.c_emissions.columns = [emissions_unit.upper()]
def calculate_combined_energy_results(self):
energy_unit = cfg.calculation_energy_unit
if self.supply.export_costs is not None:
setattr(self.outputs,'export_energy',self.supply.export_energy)
self.export_energy = self.outputs.return_cleaned_output('export_energy')
del self.outputs.export_energy
util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
keys = ["EXPORT","EMBODIED"]
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
else:
self.export_energy = None
self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
keys = ['DOMESTIC','EMBODIED']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
keys = ['DOMESTIC','FINAL']
names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
for key,name in zip(keys,names):
self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
# self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
self.final_energy[name] = "N/A"
self.final_energy.set_index(name,append=True,inplace=True)
for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
self.export_energy[name] = "N/A"
self.export_energy.set_index(name,append=True,inplace=True)
self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
self.outputs.c_energy.columns = [energy_unit.upper()]
def export_io(self):
io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
df_list = []
for year in io_table_years:
sector_df_list = []
keys = self.supply.demand_sectors
name = ['sector']
for sector in self.supply.demand_sectors:
sector_df_list.append(self.supply.io_dict[year][sector])
year_df = pd.concat(sector_df_list, keys=keys,names=name)
year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
df_list.append(year_df)
keys = io_table_years
name = ['year']
df = pd.concat(df_list,keys=keys,names=name)
for row_sector in self.supply.demand_sectors:
for col_sector in self.supply.demand_sectors:
if row_sector != col_sector:
df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
self.supply.outputs.io = df
result_df = self.supply.outputs.return_cleaned_output('io')
keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
names = ['SCENARIO','TIMESTAMP']
for key, name in zip(keys,names):
result_df = pd.concat([result_df], keys=[key],names=[name])
result_df.to_csv(os.path.join(cfg.workingdir,'supply_outputs', 's_io.csv'), header=True, mode='ab')