Exemple #1
0
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    """
    def __init__(self, scenario_id, api_run=False):
        self.scenario_id = scenario_id
        self.scenario = Scenario(self.scenario_id)
        self.api_run = api_run
        self.outputs = Output()
        self.demand = Demand(self.scenario)
        self.supply = None
        self.demand_solved, self.supply_solved = False, False

    def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results, rio_scenario):
        #try:
        self.scenario_id = scenario_id
        self.scenario = Scenario(self.scenario_id)
        self.rio_scenario = rio_scenario
        if solve_demand and not (load_demand or load_supply):
            self.calculate_demand(save_models)

        if not append_results:
            self.remove_old_results()

        # it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
        if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
            # self.demand.create_electricity_reconciliation()
            # self.demand.output_subsector_electricity_profiles()
            self.export_result_to_csv('demand_outputs')

        if solve_supply:
            if load_demand:
                # if we are loading the demand, we are changing the supply measures and want to reload our scenarios
                self.scenario = Scenario(self.scenario_id)
            self.supply = Supply(self.scenario, demand_object=self.demand,rio_scenario=rio_scenario)
            self.calculate_supply(save_models)

        if load_demand and solve_supply and export_results:
            # we do this now because we delayed before
            self.export_result_to_csv('demand_outputs')

        if self.supply_solved and export_results and (load_supply or solve_supply):
            self.supply.calculate_supply_outputs()
            self.pass_supply_results_back_to_demand()
            self.calculate_combined_results()
            self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
            self.export_result_to_csv('supply_outputs')
            self.export_result_to_csv('combined_outputs')
            self.export_io()
        #except:
            # pickle the model in the event that it crashes
            #if save_models:
             #   if cfg.rio_supply_run:
              #      Output.pickle(self, file_name=self.rio_scenario + cfg.model_error_append_name, path=cfg.workingdir)
               # else:
                #    Output.pickle(self, file_name=str(self.scenario_id) + cfg.model_error_append_name,
                 #                 path=cfg.workingdir)


    def calculate_demand(self, save_models):
        self.demand.setup_and_solve()
        self.demand_solved = True
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_d_payback()
                self.calculate_d_payback_energy()
        if save_models:
            if cfg.rio_supply_run:
                Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)
            else:
                Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)

    def calculate_supply(self, save_models):
        if not self.demand_solved:
            raise ValueError('demand must be solved first before supply')
        logging.info('Configuring energy system supply')
        self.supply.add_nodes()
        self.supply.add_measures()
        self.supply.initial_calculate()
        self.supply.calculated_years = []
        self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
        self.supply.final_calculate()
        self.supply_solved = True
        if save_models:
            if cfg.rio_supply_run:
                Output.pickle(self, file_name=self.rio_scenario + cfg.full_model_append_name, path=cfg.workingdir)
            else:
                Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
            # we don't need the demand side object any more, so we can remove it to save drive space
            # if not cfg.rio_supply_run:
            #     if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
            #         os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))

    def pass_supply_results_back_to_demand(self):
        # we need to geomap to the combined output geography
        emissions_demand_link = GeoMapper.geo_map(self.supply.emissions_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
        demand_emissions_rates = GeoMapper.geo_map(self.supply.demand_emissions_rates, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
        energy_demand_link = GeoMapper.geo_map(self.supply.energy_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')
        cost_demand_link = GeoMapper.geo_map(self.supply.cost_demand_link, GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'intensity')

        logging.info("Calculating link to supply")
        self.demand.link_to_supply(emissions_demand_link, demand_emissions_rates, energy_demand_link, cost_demand_link)
        if cfg.output_tco == 'true':
            if hasattr(self,'d_energy_tco'):
                self.demand.link_to_supply_tco(emissions_demand_link, demand_emissions_rates, cost_demand_link)
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
        if cfg.output_payback == 'true':
            if hasattr(self,'demand.d_all_energy_demand_payback'):
                self.demand.link_to_supply_payback(emissions_demand_link, demand_emissions_rates, cost_demand_link)
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
    
    def calculate_combined_results(self):
        logging.info("Calculating combined emissions results")
        self.calculate_combined_emissions_results()
        logging.info("Calculating combined cost results")
        self.calculate_combined_cost_results()
        logging.info("Calculating combined energy results")
        self.calculate_combined_energy_results()
        if cfg.output_tco == 'true':
            if self.demand.d_energy_tco is not None:
                self.calculate_tco()
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_payback()

    def remove_old_results(self):
        folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
        for folder_name in folder_names:
            folder = os.path.join(cfg.workingdir, folder_name)
            if os.path.isdir(folder):
                shutil.rmtree(folder)

    def export_result_to_csv(self, result_name):
        if result_name=='combined_outputs':
            res_obj = self.outputs
        elif result_name=='demand_outputs':
            res_obj = self.demand.outputs
        elif result_name=='supply_outputs':
            res_obj = self.supply.outputs
        else:
            raise ValueError('result_name not recognized')

        def clean_and_write(result_df, attribute):
            """

            :param result_df: pandas dataframe
            :param attribute: string
            """
            if cfg.rio_supply_run and self.supply is not None:
                keys = [self.supply.rio_scenario.upper(), cfg.timestamp]
            else:
                keys = [self.scenario.name.upper(), cfg.timestamp]
            names = ['SCENARIO', 'TIMESTAMP']
            for key, name in zip(keys, names):
                result_df = pd.concat([result_df], keys=[key], names=[name])
                result_df = result_df.fillna(0)
            if attribute in (
            'hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
                # Special case for hourly dispatch results where we want to write them outside of supply_outputs
                Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
            else:
                Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, result_name))

        for attribute in dir(res_obj):
            if isinstance(getattr(res_obj, attribute), list):
                for df in getattr(res_obj, attribute):
                    result_df = getattr(res_obj, 'clean_df')(df)
                    clean_and_write(result_df,attribute)
            elif isinstance(getattr(res_obj, attribute), pd.DataFrame):
                result_df = getattr(res_obj, 'clean_df')(getattr(res_obj, attribute))
                clean_and_write(result_df, attribute)
            else:
                continue

    def calculate_tco(self):
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = self.demand.d_levelized_costs_tco
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        service_demand_df = self.demand.d_service_demand_tco
        service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
                                        util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
                                        keys=keys,names=names) 
        self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
        self.outputs.c_tco[self.outputs.c_tco<0]=0        
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
                    indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
                    self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
        self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
        self.outputs.c_tco.columns = [cost_unit.upper()]
        self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
        self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
        
    def calculate_payback(self):
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
        supply_side_df = supply_side_df.sort_index()
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
        sales_df = sales_df.reindex(supply_side_df.index).sort_index()
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
        self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]        
        self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
                    self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
        self.outputs.c_payback.columns = [cost_unit.upper()]
        self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1    
        self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
        self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
        self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
        self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
        
    def calculate_d_payback(self):
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]        
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
                    self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
        self.demand.outputs.d_payback.columns = [cost_unit.upper()]
        self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
        self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
   
    def calculate_d_payback_energy(self):
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_all_energy_demand_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
#        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
#        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]        
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
                    self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
        self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
        self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
        self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')

    def calc_and_format_export_costs(self):
        #calculate and format export costs
        if self.supply.export_costs is None:
            return None
        export_costs = GeoMapper.geo_map(self.supply.export_costs.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
        export_costs = Output.clean_df(export_costs)
        util.replace_index_name(export_costs, 'FINAL_ENERGY', 'SUPPLY_NODE_EXPORT')
        export_costs = util.add_to_df_index(export_costs, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["EXPORT", "SUPPLY"])
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        export_costs.columns = [cost_unit.upper()]
        return export_costs

    def calc_and_format_embodied_costs(self):
        #calculate and format embodied supply costs
        embodied_costs_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_energy_costs]
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        for embodied_costs in embodied_costs_list: embodied_costs.columns = [cost_unit.upper()]
        embodied_costs_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC","SUPPLY"]) for x in embodied_costs_list]
        return embodied_costs_list

    def calc_and_format_direct_demand_costs(self):
        #calculte and format direct demand costs
        if self.demand.outputs.d_levelized_costs is None:
            return None
        direct_costs = GeoMapper.geo_map(self.demand.outputs.d_levelized_costs.copy(), GeoMapper.demand_primary_geography, GeoMapper.combined_outputs_geography, 'total')
        direct_costs = direct_costs[direct_costs.index.get_level_values('year').isin(cfg.combined_years_subset)]
        levels_to_keep = [x for x in cfg.output_combined_levels if x in direct_costs.index.names]
        direct_costs = direct_costs.groupby(level=levels_to_keep).sum()
        direct_costs = Output.clean_df(direct_costs)
        direct_costs = util.add_to_df_index(direct_costs, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC","DEMAND"])
        return direct_costs

    def calculate_combined_cost_results(self):
        cost_unit = cfg.getParam('currency_year') + " " + cfg.getParam('currency_name')
        export_costs = self.calc_and_format_export_costs()
        embodied_costs_list = self.calc_and_format_embodied_costs()
        direct_costs = self.calc_and_format_direct_demand_costs()
        export_costs = util.add_and_set_index(export_costs,['COST_TYPE'],['EXPORTED'])
        embodied_costs_list = [util.add_and_set_index(x,['COST_TYPE'],['SUPPLY-SIDE']) for x in embodied_costs_list]
        direct_costs = util.add_and_set_index(direct_costs,['COST_TYPE'],['DEMAND-SIDE'])
        if export_costs is not None:
            for name in [x for x in embodied_costs_list[0].index.names if x not in export_costs.index.names]:
                export_costs[name] = "N/A"
                export_costs.set_index(name,append=True,inplace=True)
            export_costs = export_costs.groupby(level=embodied_costs_list[0].index.names).sum()
        if direct_costs is not None:
            for name in [x for x in embodied_costs_list[0].index.names if x not in direct_costs.index.names]:
                direct_costs[name] = "N/A"
                direct_costs.set_index(name, append=True, inplace=True)
            direct_costs = direct_costs.groupby(level=embodied_costs_list[0].index.names).sum()
        self.outputs.c_costs = embodied_costs_list + [direct_costs] + [export_costs]
        self.outputs.c_costs= [x[x.values!=0] for x in self.outputs.c_costs]
        for x in self.outputs.c_costs: x.index = x.index.reorder_levels(embodied_costs_list[0].index.names)


    def calc_and_format_export_emissions(self):
        #calculate and format export emissions
        if self.supply.export_emissions is None:
            return None
        export_emissions = GeoMapper.geo_map(self.supply.export_emissions.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
        if 'supply_geography' not in cfg.output_combined_levels:
            util.remove_df_levels(export_emissions, GeoMapper.supply_primary_geography +'_supply')
        export_emissions = Output.clean_df(export_emissions)
        util.replace_index_name(export_emissions, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
        index_names = export_emissions.index.names
        export_emissions = export_emissions.reset_index()
        export_emissions['FINAL_ENERGY'] = 'export ' + export_emissions['FINAL_ENERGY']
        export_emissions = export_emissions.set_index(index_names).sort_index()
        export_emissions = util.add_to_df_index(export_emissions, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["EXPORT", "SUPPLY"])
        return export_emissions

    def calc_and_format_embodied_supply_emissions(self):
        # calculate and format embodied supply emissions
        embodied_emissions_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_emissions]
        embodied_emissions_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC", "SUPPLY"]) for x in embodied_emissions_list]
        return embodied_emissions_list

    def calc_and_format_direct_demand_emissions(self):
        #calculte and format direct demand emissions
        direct_emissions_list = [Output.clean_df(x) for x in self.demand.outputs.demand_direct_emissions]
        direct_emissions_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "SUPPLY/DEMAND"], keys=["DOMESTIC", "DEMAND"]) for x in direct_emissions_list]
        if GeoMapper.combined_outputs_geography + '_supply' in cfg.output_combined_levels:
             keys = direct_emissions_list[0].index.get_level_values(GeoMapper.combined_outputs_geography.upper()).values
             names = GeoMapper.combined_outputs_geography.upper() + '_SUPPLY'
             for x in direct_emissions_list:
                x[names] = keys

             direct_emissions_list = [x.set_index(names, append=True, inplace=True) for x in direct_emissions_list]
        return direct_emissions_list

    def calculate_combined_emissions_results(self):
        export_emissions = self.calc_and_format_export_emissions()
        embodied_emissions_list = self.calc_and_format_embodied_supply_emissions()
        direct_emissions_list = self.calc_and_format_direct_demand_emissions()
        export_emissions = util.add_and_set_index(export_emissions,['EMISSIONS_TYPE'],['EXPORTED'])
        embodied_emissions_list = [util.add_and_set_index(x, ['EMISSIONS_TYPE'], ['SUPPLY_SIDE']) for x in embodied_emissions_list]
        direct_emissions_list = [util.add_and_set_index(x,['EMISSIONS_TYPE'],['DEMAND_SIDE']) for x in direct_emissions_list]
        if export_emissions is not None:
            for name in [x for x in embodied_emissions_list[0].index.names if x not in export_emissions.index.names]:
                export_emissions[name] = "N/A"
                export_emissions.set_index(name,append=True,inplace=True)
            export_emissions = export_emissions.groupby(level=embodied_emissions_list[0].index.names).sum()
        if direct_emissions_list is not None:
            for df in direct_emissions_list:
                for name in [x for x in embodied_emissions_list[0].index.names if x not in df.index.names]:
                    df[name] = "N/A"
                    df.set_index(name,append=True,inplace=True)
        self.outputs.c_emissions = [export_emissions] + embodied_emissions_list + direct_emissions_list
        self.outputs.c_emissions = [util.replace_index_name(x, GeoMapper.combined_outputs_geography.upper() +'-EMITTED', GeoMapper.combined_outputs_geography.upper() +'_SUPPLY',inplace=True) for x in self.outputs.c_emissions]
        self.outputs.c_emissions = [util.replace_index_name(x, GeoMapper.combined_outputs_geography.upper() +'-CONSUMED', GeoMapper.combined_outputs_geography.upper(),inplace=True) for x in self.outputs.c_emissions]
        self.outputs.c_emissions = [x[x['VALUE']!=0] for x in  self.outputs.c_emissions]
        emissions_unit = cfg.getParam('mass_unit')
        for x in self.outputs.c_emissions:
            x.columns = [emissions_unit.upper()]
        for x in self.outputs.c_emissions: x.index = x.index.reorder_levels([l for l in embodied_emissions_list[0].index.names if l in x.index.names])
    def calc_and_format_export_energy(self):
        if self.supply.export_energy is None:
            return None
        export_energy = GeoMapper.geo_map(self.supply.export_energy.copy(), GeoMapper.supply_primary_geography, GeoMapper.combined_outputs_geography, 'total')
        export_energy = Output.clean_df(export_energy)
        util.replace_index_name(export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
        export_energy = util.add_to_df_index(export_energy, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=["EXPORT", "EMBODIED"])
        for x in cfg.output_combined_levels:
            if x not in export_energy.index.names:
                export_energy = util.add_and_set_index(export_energy,[x],["N/A"])
        return export_energy

    def calc_and_format_embodied_supply_energy(self):
        embodied_energy_list = [Output.clean_df(x) for x in self.demand.outputs.demand_embodied_energy]
        embodied_energy_list = [x[x['VALUE']!=0] for x in embodied_energy_list]
        embodied_energy_list = [util.add_to_df_index(x, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=['DOMESTIC','EMBODIED']) for x in embodied_energy_list]
        return embodied_energy_list

    def calc_and_format_direct_demand_energy(self):
        demand_energy = GeoMapper.geo_map(self.demand.outputs.d_energy.copy(), GeoMapper.demand_primary_geography, GeoMapper.combined_outputs_geography, 'total')
        demand_energy = Output.clean_df(demand_energy)
        demand_energy = demand_energy[demand_energy.index.get_level_values('YEAR')>=cfg.getParamAsInt('current_year')]
        demand_energy = util.add_to_df_index(demand_energy, names=['EXPORT/DOMESTIC', "ENERGY ACCOUNTING"], keys=['DOMESTIC','FINAL'])
        return demand_energy

    def calculate_combined_energy_results(self):
        export_energy = self.calc_and_format_export_energy()
        embodied_energy_list = self.calc_and_format_embodied_supply_energy()
        demand_energy = self.calc_and_format_direct_demand_energy()
        # reorder levels so dfs match
        for name in [x for x in embodied_energy_list[0].index.names if x not in demand_energy.index.names]:
            demand_energy[name] = "N/A"
            demand_energy.set_index(name, append=True, inplace=True)
        demand_energy = demand_energy.groupby(level=embodied_energy_list[0].index.names).sum()
        if export_energy is not None:
            for name in [x for x in embodied_energy_list[0].index.names if x not in export_energy.index.names]:
                export_energy[name] = "N/A"
                export_energy.set_index(name,append=True,inplace=True)
            export_energy = export_energy.groupby(level=embodied_energy_list[0].index.names).sum()

        self.outputs.c_energy = embodied_energy_list + [demand_energy] + [export_energy]
        self.outputs.c_energy = [x[x['VALUE']!=0] for x in self.outputs.c_energy]
        energy_unit = cfg.calculation_energy_unit
        for x in self.outputs.c_energy: x.columns = [energy_unit.upper()]
        for x in self.outputs.c_energy: x.index = x.index.reorder_levels(embodied_energy_list[0].index.names)

    def export_io(self):
        io_table_write_step = cfg.getParamAsInt('io_table_write_step', 'output_detail')
        io_table_years = sorted([min(self.supply.years)] + range(max(self.supply.years), min(self.supply.years), -io_table_write_step))
        df_list = []
        for year in io_table_years:
            keys = self.supply.demand_sectors
            year_df = pd.concat([self.supply.io_dict[year][sector] for sector in keys], keys=keys,names=['sector'])
            year_df = pd.concat([year_df]*len(keys), keys=keys, names=['sector'], axis=1)
            df_list.append(year_df)
        keys = io_table_years
        name = ['year']
        df = pd.concat(df_list,keys=keys,names=name)
        for row_sector in self.supply.demand_sectors:
            for col_sector in self.supply.demand_sectors:
                if row_sector != col_sector:
                    df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
        self.supply.outputs.io = df
        result_df = self.supply.outputs.return_cleaned_output('io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
#        self.export_stacked_io()

    def export_stacked_io(self):
        df = copy.deepcopy(self.supply.outputs.io)
        df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
        df = df.stack(level=df.columns.names).to_frame()
        df.columns = ['value']
        self.supply.outputs.stacked_io = df
        result_df = self.supply.outputs.return_cleaned_output('stacked_io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    """
    def __init__(self, scenario_id, api_run=False):
        self.scenario_id = scenario_id
        self.scenario = Scenario(self.scenario_id)
        self.api_run = api_run
        self.outputs = Output()
        self.demand = Demand(self.scenario)
        self.supply = None
        self.demand_solved, self.supply_solved = False, False

    def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
        try:
            if solve_demand and not (load_demand or load_supply):
                self.calculate_demand(save_models)
            
            if not append_results:
                self.remove_old_results()

            # it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
            if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
                self.export_result_to_csv('demand_outputs')

            if solve_supply and not load_supply:
                if load_demand:
                    # if we are loading the demand, we are changing the supply measures and want to reload our scenarios
                    self.scenario = Scenario(self.scenario_id)
                self.supply = Supply(self.scenario, demand_object=self.demand)
                self.calculate_supply(save_models)

            if load_demand and solve_supply:
                # we do this now because we delayed before
                self.export_result_to_csv('demand_outputs')

            if self.supply_solved and export_results and load_supply or solve_supply:
                self.supply.calculate_supply_outputs()
                self.pass_supply_results_back_to_demand()
                self.calculate_combined_results()
                self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
                if self.api_run:
                    self.export_results_to_db()
                else:
                    self.export_result_to_csv('supply_outputs')
                    self.export_result_to_csv('combined_outputs')
                    self.export_io()
        except:
            # pickle the model in the event that it crashes
            if save_models:
                Output.pickle(self, file_name=str(scenario_id) + cfg.model_error_append_name, path=cfg.workingdir)
            raise

    def calculate_demand(self, save_models):
        self.demand.setup_and_solve()
        self.demand_solved = True
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_d_payback()
                self.calculate_d_payback_energy()
        if save_models:
            Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)

    def calculate_supply(self, save_models):
        if not self.demand_solved:
            raise ValueError('demand must be solved first before supply')
        logging.info('Configuring energy system supply')
        self.supply.add_nodes()
        self.supply.add_measures()
        self.supply.initial_calculate()
        self.supply.calculated_years = []
        self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
        self.supply.final_calculate()
        self.supply_solved = True
        if save_models:
            Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
            # we don't need the demand side object any more, so we can remove it to save drive space
            if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
                os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))

    def pass_supply_results_back_to_demand(self):
        logging.info("Calculating link to supply")
        self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
        if cfg.output_tco == 'true':
            if hasattr(self,'d_energy_tco'):
                self.demand.link_to_supply_tco(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link) 
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
        if cfg.output_payback == 'true':
            if hasattr(self,'demand.d_all_energy_demand_payback'):
                self.demand.link_to_supply_payback(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link) 
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
    
    def calculate_combined_results(self):
        logging.info("Calculating combined emissions results")
        self.calculate_combined_emissions_results()
        logging.info("Calculating combined cost results")
        self.calculate_combined_cost_results()
        logging.info("Calculating combined energy results")
        self.calculate_combined_energy_results()
        if cfg.output_tco == 'true':
            if self.demand.d_energy_tco is not None:
                self.calculate_tco()
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_payback()

    def remove_old_results(self):
        folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
        for folder_name in folder_names:
            folder = os.path.join(cfg.workingdir, folder_name)
            if os.path.isdir(folder):
                shutil.rmtree(folder)

    def export_result_to_csv(self, result_name):
        if result_name=='combined_outputs':
            res_obj = self.outputs
        elif result_name=='demand_outputs':
            res_obj = self.demand.outputs
        elif result_name=='supply_outputs':
            res_obj = self.supply.outputs
        else:
            raise ValueError('result_name not recognized')

        for attribute in dir(res_obj):
            if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
                continue

            result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
            keys = [self.scenario.name.upper(), cfg.timestamp]
            names = ['SCENARIO','TIMESTAMP']
            for key, name in zip(keys, names):
                result_df = pd.concat([result_df], keys=[key], names=[name])

            if attribute in ('hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
                # Special case for hourly dispatch results where we want to write them outside of supply_outputs
                Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
            else:
                Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))

    def export_results_to_db(self):
        scenario_run_id = util.active_scenario_run_id(self.scenario_id)
        # Levelized costs
        costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 1, costs)

        #Energy
        energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        # Energy demand by sector
        util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))

        #Emissions
        emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
        # Annual emissions by sector
        util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))

        # Domestic emissions per capita
        annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
        population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
        population_driver.index.name = 'YEAR'
        factor = 1E6
        df = util.DfOper.divi((annual_emissions, population_driver)) * factor
        df.columns = ['TONNE PER CAPITA']
        util.write_output_to_db(scenario_run_id, 4, df)

        # Electricity supply
        electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
        df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
            .xs('EMBODIED', level='ENERGY ACCOUNTING')\
            .groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])

    def calculate_combined_cost_results(self):
        #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        if self.supply.export_costs is not None:
            setattr(self.outputs,'export_costs',self.supply.export_costs)
            self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
            del self.outputs.export_costs
            util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
            self.export_costs_df.columns = [cost_unit.upper()]  
        else:
            self.export_costs_df = None
        #calculate and format emobodied supply costs
        self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
        self.embodied_energy_costs_df.columns = [cost_unit.upper()]
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
        #calculte and format direct demand costs
        self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
        if self.demand_costs_df is not None:
            levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
            levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
            self.demand_costs_df = self.demand_costs_df.groupby(level=levels_to_keep).sum()
            keys = ["DOMESTIC","DEMAND"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
        self.outputs.c_costs[self.outputs.c_costs<0]=0
        self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
        
    def calculate_tco(self):
#        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
#        del self.demand.outputs.demand_embodied_emissions
        #calculte and format direct demand emissions        
#        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
##        del self.demand.outputs.demand_direct_emissions
#        emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
#         #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = self.demand.d_levelized_costs_tco
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        service_demand_df = self.demand.d_service_demand_tco
        service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
                                        util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
                                        keys=keys,names=names) 
        self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
        self.outputs.c_tco[self.outputs.c_tco<0]=0        
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
                    indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
                    self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
        self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
        self.outputs.c_tco.columns = [cost_unit.upper()]
        self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
        self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
        
        
        
    def calculate_payback(self):
#        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
#        del self.demand.outputs.demand_embodied_emissions
        #calculte and format direct demand emissions        
#        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
##        del self.demand.outputs.demand_direct_emissions
#        emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
#         #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
        supply_side_df = supply_side_df.sort_index()
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
        sales_df = sales_df.reindex(supply_side_df.index).sort_index()
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
        self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]        
        self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
                    self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
        self.outputs.c_payback.columns = [cost_unit.upper()]
        self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1    
        self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
        self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
        self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
        self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
        
        
    def calculate_d_payback(self):
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]        
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
                    self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
        self.demand.outputs.d_payback.columns = [cost_unit.upper()]
        self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
        self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
   
    def calculate_d_payback_energy(self):
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_all_energy_demand_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
#        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
#        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]        
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
                    self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
        self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
        self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
        self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')
            
        
    def calculate_combined_emissions_results(self):
        #calculate and format export emissions
        if self.supply.export_emissions is not None:
            setattr(self.outputs,'export_emissions',self.supply.export_emissions)
            if 'supply_geography' not in cfg.output_combined_levels:
                util.remove_df_levels(self.outputs.export_emissions, cfg.primary_geography +'_supply')
            self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
            del self.outputs.export_emissions
            util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
        else:
            self.export_emissions_df = None
       #calculate and format emobodied supply emissions
        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
#        del self.demand.outputs.demand_embodied_emissions
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])       
        #calculte and format direct demand emissions        
        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
#        del self.demand.outputs.demand_direct_emissions
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.direct_emissions_df = pd.concat([self.direct_emissions_df], keys=[key], names=[name])
        if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
             keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
             names = cfg.primary_geography.upper() +'_SUPPLY'
             self.direct_emissions_df[names] = keys
             self.direct_emissions_df.set_index(names,append=True,inplace=True)
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['EMISSIONS TYPE']
        self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
        self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
        emissions_unit = cfg.cfgfile.get('case','mass_unit')
        self.outputs.c_emissions.columns = [emissions_unit.upper()]
            
    def calculate_combined_energy_results(self):
         energy_unit = cfg.calculation_energy_unit
         if self.supply.export_costs is not None:
            setattr(self.outputs,'export_energy',self.supply.export_energy)
            self.export_energy = self.outputs.return_cleaned_output('export_energy')
            del self.outputs.export_energy
            util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","EMBODIED"]
            names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
            for key,name in zip(keys,names):
                self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
         else:
            self.export_energy = None
         self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
         self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
         keys = ['DOMESTIC','EMBODIED']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
         self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
         self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
         keys = ['DOMESTIC','FINAL']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
    #         self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
         for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
             self.final_energy[name] = "N/A"
             self.final_energy.set_index(name,append=True,inplace=True)
         if self.export_energy is not None:
             for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
                 self.export_energy[name] = "N/A"
                 self.export_energy.set_index(name,append=True,inplace=True)
             self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
             self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
         self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
         self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
         self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
         self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
         self.outputs.c_energy.columns = [energy_unit.upper()]

    def export_io(self):
        io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
        io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
        df_list = []
        for year in io_table_years:
            sector_df_list = []
            keys = self.supply.demand_sectors
            name = ['sector']
            for sector in self.supply.demand_sectors:
                sector_df_list.append(self.supply.io_dict[year][sector])
            year_df = pd.concat(sector_df_list, keys=keys,names=name)
            year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
            df_list.append(year_df)
        keys = io_table_years
        name = ['year']
        df = pd.concat(df_list,keys=keys,names=name)
        for row_sector in self.supply.demand_sectors:
            for col_sector in self.supply.demand_sectors:
                if row_sector != col_sector:
                    df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
        self.supply.outputs.io = df
        result_df = self.supply.outputs.return_cleaned_output('io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
#        self.export_stacked_io()

    def export_stacked_io(self):
        df = copy.deepcopy(self.supply.outputs.io)
        df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
        df = df.stack(level=df.columns.names).to_frame()
        df.columns = ['value']
        self.supply.outputs.stacked_io = df
        result_df = self.supply.outputs.return_cleaned_output('stacked_io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    Includes the primary geography of the energy system (i.e. country name) as well as the author.
    """

    def __init__(self, db_path, cfgfile_path, custom_pint_definitions_path=None, name=None, author=None):
        self.model_config(db_path, cfgfile_path, custom_pint_definitions_path)
        self.name = cfg.cfgfile.get("case", "scenario") if name is None else name
        self.author = cfg.cfgfile.get("case", "author") if author is None else author
        self.demand = Demand()
        self.supply = Supply()

    def model_config(self, db_path, cfgfile_path, custom_pint_definitions_path):
        cfg.init_cfgfile(cfgfile_path)
        cfg.init_db(db_path)
        cfg.init_pint(custom_pint_definitions_path)
        cfg.init_geo()
        cfg.init_shapes()
        cfg.init_outputs_id_map()

    def configure_energy_system(self):
        print "configuring energy system"
        self.configure_demand()
        self.configure_supply()
        cfg.init_outputs_id_map()

    def populate_energy_system(self):
        self.populate_demand_system()
        self.populate_supply_system()

    def populate_measures(self):
        self.populate_demand_measures()
        self.populate_supply_measures()

    def calculate(self):
        self.calculate_demand_only()
        self.pass_results_to_supply()
        self.calculate_supply()

    def configure_demand(self):
        """Read in and initialize data"""
        # Drivers must come first
        self.demand.add_drivers()

        # Sectors requires drivers be read in
        self.demand.add_sectors()
        for sector in self.demand.sectors.values():
            # print 'configuring the %s sector'  %sector.name
            sector.add_subsectors()

    def configure_supply(self):
        self.supply.add_nodes()

    def populate_demand_system(self):
        print "remapping drivers"
        self.demand.remap_drivers()
        print "populating energy system data"
        for sector in self.demand.sectors.values():
            print "  " + sector.name + " sector"
            for subsector in sector.subsectors.values():
                print "    " + subsector.name
                subsector.add_energy_system_data()
        self.demand.precursor_dict()

    def populate_supply_system(self):
        self.supply.add_energy_system_data()

    def populate_demand_measures(self):
        for sector in self.demand.sectors.values():
            for subsector in sector.subsectors.values():
                subsector.add_measures()

    def populate_supply_measures(self):
        self.supply.add_measures()

    def calculate_demand_only(self):
        self.demand.manage_calculations()

    def calculate_supply(self):
        self.supply.calculate()

    def pass_results_to_supply(self):
        for sector in self.demand.sectors.values():
            sector.aggregate_subsector_energy_for_supply_side()
        self.demand.aggregate_sector_energy_for_supply_side()
        self.supply.demand_df = self.demand.energy_demand

    def pass_results_to_demand(self):
        self.demand.aggregate_results()
        self.demand.link_to_supply(
            self.supply.emissions_demand_link, self.supply.energy_demand_link, self.supply.cost_demand_link
        )

    def export_results(self, specified_directory=None):
        if specified_directory is None:
            specified_directory = os.path.join(os.getcwd())
        else:
            specified_directory = os.path.join(specified_directory)
        attributes = dir(self.demand.outputs)
        for att in attributes:
            if isinstance(getattr(self.demand.outputs, att), pd.core.frame.DataFrame):
                output = self.demand.outputs.return_cleaned_output(att)
                ExportMethods.writedataframe(att, output, specified_directory)
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    Includes the primary geography of the energy system (i.e. country name) as well as the author.
    """
    def __init__(self, cfgfile_path, custom_pint_definitions_path=None, name=None, author=None):
        self.cfgfile_path = cfgfile_path
        self.custom_pint_definitions_path = custom_pint_definitions_path
        self.model_config(cfgfile_path, custom_pint_definitions_path)
        self.name = cfg.cfgfile.get('case', 'scenario') if name is None else name
        self.author = cfg.cfgfile.get('case', 'author') if author is None else author      
        self.scenario_dict = dict(zip(util.sql_read_table('Scenarios','id', return_iterable=True, is_active=True),
                                  util.sql_read_table('Scenarios','name', return_iterable=True, is_active=True)))
        self.outputs = Output()
        self.geography = cfg.cfgfile.get('case', 'primary_geography')
        

    def model_config(self, cfgfile_path, custom_pint_definitions_path):
        cfg.init_cfgfile(cfgfile_path)
        cfg.init_db()
        cfg.path = custom_pint_definitions_path
        cfg.init_pint(custom_pint_definitions_path)
        cfg.init_geo()
        cfg.init_date_lookup()
        if shape.shapes.rerun:
            shape.shapes.create_empty_shapes()
#            shape.shapes.activate_shape(cfg.electricity_energy_type_shape_id)
        cfg.init_outputs_id_map()

    def configure_energy_system(self):
        print 'configuring energy system'
        self.demand = Demand(self.cfgfile_path, self.custom_pint_definitions_path)
        self.supply = Supply(os.path.join(os.getcwd(),'outputs'),self.cfgfile_path, self.custom_pint_definitions_path)
        self.configure_demand()
        self.configure_supply()

    def populate_energy_system(self):
        self.populate_demand_system()
        self.populate_supply_system()
    
    def populate_shapes(self):
        print 'processing shapes'
        if shape.shapes.rerun:
            shape.shapes.initiate_active_shapes()
            shape.shapes.process_active_shapes()

    def populate_measures(self, scenario_id):
        self.scenario_id = scenario_id
        self.scenario = self.scenario_dict[self.scenario_id]
        self.demand_case_id = util.sql_read_table('Scenarios','demand_case',id=self.scenario_id)
        self.populate_demand_measures()
        self.supply_case_id = util.sql_read_table('Scenarios','supply_case',id=self.scenario_id)
        self.populate_supply_measures()

    def calculate(self):
        self.calculate_demand_only()
        self.pass_results_to_supply()
        self.calculate_supply()

    def configure_demand(self):
        """Read in and initialize data"""
        # Drivers must come first
        self.demand.add_drivers()

        # Sectors requires drivers be read in
        self.demand.add_sectors()
        for sector in self.demand.sectors.values():
            # print 'configuring the %s sector'  %sector.name
            sector.add_subsectors()
            
    def configure_supply(self):
        self.supply.add_node_list()

    def populate_demand_system(self):
        print 'remapping drivers'
        self.demand.remap_drivers()
        print 'populating energy system data'
        for sector in self.demand.sectors.values():
            print '  '+sector.name+' sector'
#            print 'reading energy system data for the %s sector' %sector.name
            for subsector in sector.subsectors.values():
                print '    '+subsector.name
                subsector.add_energy_system_data()
            sector.precursor_dict()

    def populate_supply_system(self):
        self.supply.add_nodes()

    def populate_demand_measures(self):
        for sector in self.demand.sectors.values():
            #            print 'reading %s measures' %sector.name
            for subsector in sector.subsectors.values():
                subsector.add_measures(self.demand_case_id)
        
    def populate_supply_measures(self):
        self.supply.add_measures(self.supply_case_id)

    def calculate_demand_only(self):
        self.demand.calculate_demand()
        print "aggregating demand results"
        self.demand.aggregate_results()
    
    def calculate_supply(self):
        self.supply.initial_calculate()     
        self.supply.calculate_loop()
        self.supply.final_calculate()
             
    def pass_results_to_supply(self):
        for sector in self.demand.sectors.values():
             sector.aggregate_subsector_energy_for_supply_side()
        self.demand.aggregate_sector_energy_for_supply_side()
        self.supply.demand_object = self.demand
        
    def pass_results_to_demand(self):
        print "calculating link to supply"
        self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
    
    def calculate_combined_results(self):
        print "calculating combined emissions results"
        self.calculate_combined_emissions_results()
        print "calculating combined cost results"
        self.calculate_combined_cost_results()
        print "calculating combined energy results"
        self.calculate_combined_energy_results()
    
    def export_results(self):
        for attribute in dir(self.outputs):
            if isinstance(getattr(self.outputs,attribute), pd.DataFrame):
                result_df = getattr(self.outputs, attribute)
                keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
                names = ['SCENARIO','TIMESTAMP']
                for key, name in zip(keys,names):
                    result_df = pd.concat([result_df],keys=[key],names=[name])
                ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'combined_outputs'), append_results=True)
        for attribute in dir(self.demand.outputs):
            if isinstance(getattr(self.demand.outputs,attribute), pd.DataFrame):
                result_df = self.demand.outputs.return_cleaned_output(attribute)
                keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
                names = ['SCENARIO','TIMESTAMP']
                for key, name in zip(keys,names):
                    result_df = pd.concat([result_df],keys=[key],names=[name])
                ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'demand_outputs'), append_results=True)
        for attribute in dir(self.supply.outputs):
            if isinstance(getattr(self.supply.outputs,attribute), pd.DataFrame):
                result_df = self.supply.outputs.return_cleaned_output(attribute)
                keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
                names = ['SCENARIO','TIMESTAMP']
                for key, name in zip(keys,names):
                    result_df = pd.concat([result_df],keys=[key],names=[name])
                ExportMethods.writeobj(attribute,result_df, os.path.join(os.getcwd(),'supply_outputs'), append_results=True)
        
    def calculate_combined_cost_results(self):
        #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        if self.supply.export_costs is not None:
            setattr(self.outputs,'export_costs',self.supply.export_costs)
            self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
            del self.outputs.export_costs
            util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
            self.export_costs_df.columns = [cost_unit.upper()]  
        else:
            self.export_costs_df = None
        #calculate and format emobodied supply costs
        self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
        self.embodied_energy_costs_df.columns = [cost_unit.upper()]  
#        del self.demand.outputs.demand_embodied_energy_costs
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])       
        #calculte and format direct demand emissions        
        self.demand_costs_df= self.demand.outputs.return_cleaned_output('levelized_costs')  
#        del self.demand.outputs.levelized_costs
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
            self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])      
#        levels_to_keep = cfg.output_levels      
#        levels_to_keep = [x.upper() for x in levels_to_keep]
#        levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
#        util.replace_index_name(self.outputs.costs, self.geography.upper() +'_EARNED', self.geography.upper() +'_SUPPLY')
#        util.replace_index_name(self.outputs.costs, self.geography.upper() +'_CONSUMED', self.geography.upper())
        self.outputs.costs[self.outputs.costs<0]=0
        self.outputs.costs= self.outputs.costs[self.outputs.costs[cost_unit.upper()]!=0]
#        self.outputs.costs.sort(inplace=True)       

    
        
    def calculate_combined_emissions_results(self):
        #calculate and format export emissions
        if self.supply.export_emissions is not None:
            setattr(self.outputs,'export_emissions',self.supply.export_emissions)
            if 'supply_geography' not in cfg.output_combined_levels:
                util.remove_df_levels(self.outputs.export_emissions,self.geography +'_supply')
            self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
            del self.outputs.export_emissions
            util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
        else:
            self.export_emissions_df = None
       #calculate and format emobodied supply emissions
        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
#        del self.demand.outputs.demand_embodied_emissions
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])       
        #calculte and format direct demand emissions        
        self.direct_emissions_df= self.demand.outputs.return_cleaned_output('demand_direct_emissions')  
#        del self.demand.outputs.demand_direct_emissions
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
            self.direct_emissions_df = pd.concat([self.direct_emissions_df],keys=[key],names=[name])   
        if 'supply_geography' in cfg.output_combined_levels:
            keys = self.direct_emissions_df.index.get_level_values(self.geography.upper()).values
            names = self.geography.upper() +'_SUPPLY'
            self.direct_emissions_df[names] = keys
            self.direct_emissions_df.set_index(names,append=True,inplace=True)
#        levels_to_keep = cfg.output_levels      
#        levels_to_keep = [x.upper() for x in levels_to_keep]
#        levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['EMISSIONS TYPE']
        self.outputs.emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
#        util.replace_index_name(self.outputs.emissions, "ENERGY","FINAL_ENERGY")
        util.replace_index_name(self.outputs.emissions, self.geography.upper() +'_EMITTED', self.geography.upper() +'_SUPPLY')
        util.replace_index_name(self.outputs.emissions, self.geography.upper() +'_CONSUMED', self.geography.upper())
        self.outputs.emissions= self.outputs.emissions[self.outputs.emissions['VALUE']!=0]
        emissions_unit = cfg.cfgfile.get('case','mass_unit')
        self.outputs.emissions.columns = [emissions_unit.upper()]
        
#        self.outputs.emissions.sort(inplace=True)        
            
    def calculate_combined_energy_results(self):
         self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
         self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
         self.final_energy = self.demand.outputs.return_cleaned_output('energy')
         self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]  
         self.embodied_energy['ENERGY ACCOUNTING'] = 'EMBODIED'
         self.final_energy['ENERGY ACCOUNTING'] = 'FINAL'
         self.embodied_energy.set_index('ENERGY ACCOUNTING',append=True,inplace=True)
         self.final_energy.set_index('ENERGY ACCOUNTING',append=True,inplace=True)
    #         self.outputs.energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
         for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
             self.final_energy[name] = "N/A"
             self.final_energy.set_index(name,append=True,inplace=True)
         self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
         self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
         self.outputs.energy = pd.concat([self.embodied_energy,self.final_energy])
         self.outputs.energy= self.outputs.energy[self.outputs.energy['VALUE']!=0]
         energy_unit = cfg.cfgfile.get('case','energy_unit')
         self.outputs.energy.columns = [energy_unit.upper()]
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    Includes the primary geography of the energy system (i.e. country name) as well as the author.
    """
    def __init__(self,
                 db_path,
                 cfgfile_path,
                 custom_pint_definitions_path=None,
                 name=None,
                 author=None):
        self.model_config(db_path, cfgfile_path, custom_pint_definitions_path)
        self.name = cfg.cfgfile.get('case',
                                    'scenario') if name is None else name
        self.author = cfg.cfgfile.get('case',
                                      'author') if author is None else author
        self.demand = Demand()
        self.supply = Supply()

    def model_config(self, db_path, cfgfile_path,
                     custom_pint_definitions_path):
        cfg.init_cfgfile(cfgfile_path)
        cfg.init_db(db_path)
        cfg.init_pint(custom_pint_definitions_path)
        cfg.init_geo()
        cfg.init_shapes()
        cfg.init_outputs_id_map()

    def configure_energy_system(self):
        print 'configuring energy system'
        self.configure_demand()
        self.configure_supply()
        cfg.init_outputs_id_map()

    def populate_energy_system(self):
        self.populate_demand_system()
        self.populate_supply_system()

    def populate_measures(self):
        self.populate_demand_measures()
        self.populate_supply_measures()

    def calculate(self):
        self.calculate_demand_only()
        self.pass_results_to_supply()
        self.calculate_supply()

    def configure_demand(self):
        """Read in and initialize data"""
        # Drivers must come first
        self.demand.add_drivers()

        # Sectors requires drivers be read in
        self.demand.add_sectors()
        for sector in self.demand.sectors.values():
            # print 'configuring the %s sector'  %sector.name
            sector.add_subsectors()

    def configure_supply(self):
        self.supply.add_nodes()

    def populate_demand_system(self):
        print 'remapping drivers'
        self.demand.remap_drivers()
        print 'populating energy system data'
        for sector in self.demand.sectors.values():
            print '  ' + sector.name + ' sector'
            for subsector in sector.subsectors.values():
                print '    ' + subsector.name
                subsector.add_energy_system_data()
        self.demand.precursor_dict()

    def populate_supply_system(self):
        self.supply.add_energy_system_data()

    def populate_demand_measures(self):
        for sector in self.demand.sectors.values():
            for subsector in sector.subsectors.values():
                subsector.add_measures()

    def populate_supply_measures(self):
        self.supply.add_measures()

    def calculate_demand_only(self):
        self.demand.manage_calculations()

    def calculate_supply(self):
        self.supply.calculate()

    def pass_results_to_supply(self):
        for sector in self.demand.sectors.values():
            sector.aggregate_subsector_energy_for_supply_side()
        self.demand.aggregate_sector_energy_for_supply_side()
        self.supply.demand_df = self.demand.energy_demand

    def pass_results_to_demand(self):
        self.demand.aggregate_results()
        self.demand.link_to_supply(self.supply.emissions_demand_link,
                                   self.supply.energy_demand_link,
                                   self.supply.cost_demand_link)

    def export_results(self, specified_directory=None):
        if specified_directory is None:
            specified_directory = os.path.join(os.getcwd())
        else:
            specified_directory = os.path.join(specified_directory)
        attributes = dir(self.demand.outputs)
        for att in attributes:
            if isinstance(getattr(self.demand.outputs, att),
                          pd.core.frame.DataFrame):
                output = self.demand.outputs.return_cleaned_output(att)
                ExportMethods.writedataframe(att, output, specified_directory)
Exemple #6
0
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    Includes the primary geography of the energy system (i.e. country name) as well as the author.
    """
    def __init__(self,
                 cfgfile_path,
                 custom_pint_definitions_path=None,
                 name=None,
                 author=None):
        self.cfgfile_path = cfgfile_path
        self.custom_pint_definitions_path = custom_pint_definitions_path
        self.model_config(cfgfile_path, custom_pint_definitions_path)
        self.name = cfg.cfgfile.get('case',
                                    'scenario') if name is None else name
        self.author = cfg.cfgfile.get('case',
                                      'author') if author is None else author
        self.scenario_dict = dict(
            zip(
                util.sql_read_table('Scenarios',
                                    'id',
                                    return_iterable=True,
                                    is_active=True),
                util.sql_read_table('Scenarios',
                                    'name',
                                    return_iterable=True,
                                    is_active=True)))
        self.outputs = Output()
        self.geography = cfg.cfgfile.get('case', 'primary_geography')

    def model_config(self, cfgfile_path, custom_pint_definitions_path):
        cfg.init_cfgfile(cfgfile_path)
        cfg.init_db()
        cfg.path = custom_pint_definitions_path
        cfg.init_pint(custom_pint_definitions_path)
        cfg.init_geo()
        cfg.init_date_lookup()
        if shape.shapes.rerun:
            shape.shapes.create_empty_shapes()
#            shape.shapes.activate_shape(cfg.electricity_energy_type_shape_id)
        cfg.init_outputs_id_map()

    def configure_energy_system(self):
        print 'configuring energy system'
        self.demand = Demand(self.cfgfile_path,
                             self.custom_pint_definitions_path)
        self.supply = Supply(os.path.join(os.getcwd(),
                                          'outputs'), self.cfgfile_path,
                             self.custom_pint_definitions_path)
        self.configure_demand()
        self.configure_supply()

    def populate_energy_system(self):
        self.populate_demand_system()
        self.populate_supply_system()

    def populate_shapes(self):
        print 'processing shapes'
        if shape.shapes.rerun:
            shape.shapes.initiate_active_shapes()
            shape.shapes.process_active_shapes()

    def populate_measures(self, scenario_id):
        self.scenario_id = scenario_id
        self.scenario = self.scenario_dict[self.scenario_id]
        self.demand_case_id = util.sql_read_table('Scenarios',
                                                  'demand_case',
                                                  id=self.scenario_id)
        self.populate_demand_measures()
        self.supply_case_id = util.sql_read_table('Scenarios',
                                                  'supply_case',
                                                  id=self.scenario_id)
        self.populate_supply_measures()

    def calculate(self):
        self.calculate_demand_only()
        self.pass_results_to_supply()
        self.calculate_supply()

    def configure_demand(self):
        """Read in and initialize data"""
        # Drivers must come first
        self.demand.add_drivers()

        # Sectors requires drivers be read in
        self.demand.add_sectors()
        for sector in self.demand.sectors.values():
            # print 'configuring the %s sector'  %sector.name
            sector.add_subsectors()

    def configure_supply(self):
        self.supply.add_node_list()

    def populate_demand_system(self):
        print 'remapping drivers'
        self.demand.remap_drivers()
        print 'populating energy system data'
        for sector in self.demand.sectors.values():
            print '  ' + sector.name + ' sector'
            #            print 'reading energy system data for the %s sector' %sector.name
            for subsector in sector.subsectors.values():
                print '    ' + subsector.name
                subsector.add_energy_system_data()
            sector.precursor_dict()

    def populate_supply_system(self):
        self.supply.add_nodes()

    def populate_demand_measures(self):
        for sector in self.demand.sectors.values():
            #            print 'reading %s measures' %sector.name
            for subsector in sector.subsectors.values():
                subsector.add_measures(self.demand_case_id)

    def populate_supply_measures(self):
        self.supply.add_measures(self.supply_case_id)

    def calculate_demand_only(self):
        self.demand.calculate_demand()
        print "aggregating demand results"
        self.demand.aggregate_results()

    def calculate_supply(self):
        self.supply.initial_calculate()
        self.supply.calculate_loop()
        self.supply.final_calculate()

    def pass_results_to_supply(self):
        for sector in self.demand.sectors.values():
            sector.aggregate_subsector_energy_for_supply_side()
        self.demand.aggregate_sector_energy_for_supply_side()
        self.supply.demand_object = self.demand

    def pass_results_to_demand(self):
        print "calculating link to supply"
        self.demand.link_to_supply(self.supply.emissions_demand_link,
                                   self.supply.demand_emissions_rates,
                                   self.supply.energy_demand_link,
                                   self.supply.cost_demand_link)

    def calculate_combined_results(self):
        print "calculating combined emissions results"
        self.calculate_combined_emissions_results()
        print "calculating combined cost results"
        self.calculate_combined_cost_results()
        print "calculating combined energy results"
        self.calculate_combined_energy_results()

    def export_results(self):
        for attribute in dir(self.outputs):
            if isinstance(getattr(self.outputs, attribute), pd.DataFrame):
                result_df = getattr(self.outputs, attribute)
                keys = [
                    self.scenario.upper(),
                    str(datetime.now().replace(second=0, microsecond=0))
                ]
                names = ['SCENARIO', 'TIMESTAMP']
                for key, name in zip(keys, names):
                    result_df = pd.concat([result_df],
                                          keys=[key],
                                          names=[name])
                ExportMethods.writeobj(attribute,
                                       result_df,
                                       os.path.join(os.getcwd(),
                                                    'combined_outputs'),
                                       append_results=True)
        for attribute in dir(self.demand.outputs):
            if isinstance(getattr(self.demand.outputs, attribute),
                          pd.DataFrame):
                result_df = self.demand.outputs.return_cleaned_output(
                    attribute)
                keys = [
                    self.scenario.upper(),
                    str(datetime.now().replace(second=0, microsecond=0))
                ]
                names = ['SCENARIO', 'TIMESTAMP']
                for key, name in zip(keys, names):
                    result_df = pd.concat([result_df],
                                          keys=[key],
                                          names=[name])
                ExportMethods.writeobj(attribute,
                                       result_df,
                                       os.path.join(os.getcwd(),
                                                    'demand_outputs'),
                                       append_results=True)
        for attribute in dir(self.supply.outputs):
            if isinstance(getattr(self.supply.outputs, attribute),
                          pd.DataFrame):
                result_df = self.supply.outputs.return_cleaned_output(
                    attribute)
                keys = [
                    self.scenario.upper(),
                    str(datetime.now().replace(second=0, microsecond=0))
                ]
                names = ['SCENARIO', 'TIMESTAMP']
                for key, name in zip(keys, names):
                    result_df = pd.concat([result_df],
                                          keys=[key],
                                          names=[name])
                ExportMethods.writeobj(attribute,
                                       result_df,
                                       os.path.join(os.getcwd(),
                                                    'supply_outputs'),
                                       append_results=True)

    def calculate_combined_cost_results(self):
        #calculate and format export costs
        cost_unit = cfg.cfgfile.get(
            'case', 'currency_year_id') + " " + cfg.cfgfile.get(
                'case', 'currency_name')
        if self.supply.export_costs is not None:
            setattr(self.outputs, 'export_costs', self.supply.export_costs)
            self.export_costs_df = self.outputs.return_cleaned_output(
                'export_costs')
            del self.outputs.export_costs
            util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY',
                                    'SUPPLY_NODE_EXPORT')
            keys = ["EXPORT", "SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key, name in zip(keys, names):
                self.export_costs_df = pd.concat([self.export_costs_df],
                                                 keys=[key],
                                                 names=[name])
            self.export_costs_df.columns = [cost_unit.upper()]
        else:
            self.export_costs_df = None
        #calculate and format emobodied supply costs
        self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output(
            'demand_embodied_energy_costs')
        self.embodied_energy_costs_df.columns = [cost_unit.upper()]
        #        del self.demand.outputs.demand_embodied_energy_costs
        keys = ["DOMESTIC", "SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.embodied_energy_costs_df = pd.concat(
                [self.embodied_energy_costs_df], keys=[key], names=[name])
        #calculte and format direct demand emissions
        self.demand_costs_df = self.demand.outputs.return_cleaned_output(
            'levelized_costs')
        #        del self.demand.outputs.levelized_costs
        keys = ["DOMESTIC", "DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.demand_costs_df = pd.concat([self.demand_costs_df],
                                             keys=[key],
                                             names=[name])
#        levels_to_keep = cfg.output_levels
#        levels_to_keep = [x.upper() for x in levels_to_keep]
#        levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.costs = util.df_list_concatenate([
            self.export_costs_df, self.embodied_energy_costs_df,
            self.demand_costs_df
        ],
                                                      keys=keys,
                                                      new_names=names)
        #        util.replace_index_name(self.outputs.costs, self.geography.upper() +'_EARNED', self.geography.upper() +'_SUPPLY')
        #        util.replace_index_name(self.outputs.costs, self.geography.upper() +'_CONSUMED', self.geography.upper())
        self.outputs.costs[self.outputs.costs < 0] = 0
        self.outputs.costs = self.outputs.costs[
            self.outputs.costs[cost_unit.upper()] != 0]
#        self.outputs.costs.sort(inplace=True)

    def calculate_combined_emissions_results(self):
        #calculate and format export emissions
        if self.supply.export_emissions is not None:
            setattr(self.outputs, 'export_emissions',
                    self.supply.export_emissions)
            if 'supply_geography' not in cfg.output_combined_levels:
                util.remove_df_levels(self.outputs.export_emissions,
                                      self.geography + '_supply')
            self.export_emissions_df = self.outputs.return_cleaned_output(
                'export_emissions')
            del self.outputs.export_emissions
            util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY',
                                    'SUPPLY_NODE_EXPORT')
            keys = ["EXPORT", "SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key, name in zip(keys, names):
                self.export_emissions_df = pd.concat(
                    [self.export_emissions_df], keys=[key], names=[name])
        else:
            self.export_emissions_df = None

    #calculate and format emobodied supply emissions
        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output(
            'demand_embodied_emissions')
        #        del self.demand.outputs.demand_embodied_emissions
        keys = ["DOMESTIC", "SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.embodied_emissions_df = pd.concat(
                [self.embodied_emissions_df], keys=[key], names=[name])
        #calculte and format direct demand emissions
        self.direct_emissions_df = self.demand.outputs.return_cleaned_output(
            'demand_direct_emissions')
        #        del self.demand.outputs.demand_direct_emissions
        keys = ["DOMESTIC", "DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.direct_emissions_df = pd.concat([self.direct_emissions_df],
                                                 keys=[key],
                                                 names=[name])
        if 'supply_geography' in cfg.output_combined_levels:
            keys = self.direct_emissions_df.index.get_level_values(
                self.geography.upper()).values
            names = self.geography.upper() + '_SUPPLY'
            self.direct_emissions_df[names] = keys
            self.direct_emissions_df.set_index(names,
                                               append=True,
                                               inplace=True)
#        levels_to_keep = cfg.output_levels
#        levels_to_keep = [x.upper() for x in levels_to_keep]
#        levels_to_keep += names + [self.geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['EMISSIONS TYPE']
        self.outputs.emissions = util.df_list_concatenate([
            self.export_emissions_df, self.embodied_emissions_df,
            self.direct_emissions_df
        ],
                                                          keys=keys,
                                                          new_names=names)
        #        util.replace_index_name(self.outputs.emissions, "ENERGY","FINAL_ENERGY")
        util.replace_index_name(self.outputs.emissions,
                                self.geography.upper() + '_EMITTED',
                                self.geography.upper() + '_SUPPLY')
        util.replace_index_name(self.outputs.emissions,
                                self.geography.upper() + '_CONSUMED',
                                self.geography.upper())
        self.outputs.emissions = self.outputs.emissions[
            self.outputs.emissions['VALUE'] != 0]
        emissions_unit = cfg.cfgfile.get('case', 'mass_unit')
        self.outputs.emissions.columns = [emissions_unit.upper()]

#        self.outputs.emissions.sort(inplace=True)

    def calculate_combined_energy_results(self):
        self.embodied_energy = self.demand.outputs.return_cleaned_output(
            'demand_embodied_energy')
        self.embodied_energy = self.embodied_energy[
            self.embodied_energy['VALUE'] != 0]
        self.final_energy = self.demand.outputs.return_cleaned_output('energy')
        self.final_energy = self.final_energy[
            self.final_energy.index.get_level_values('YEAR') >= int(
                cfg.cfgfile.get('case', 'current_year'))]
        self.embodied_energy['ENERGY ACCOUNTING'] = 'EMBODIED'
        self.final_energy['ENERGY ACCOUNTING'] = 'FINAL'
        self.embodied_energy.set_index('ENERGY ACCOUNTING',
                                       append=True,
                                       inplace=True)
        self.final_energy.set_index('ENERGY ACCOUNTING',
                                    append=True,
                                    inplace=True)
        #         self.outputs.energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
        for name in [
                x for x in self.embodied_energy.index.names
                if x not in self.final_energy.index.names
        ]:
            self.final_energy[name] = "N/A"
            self.final_energy.set_index(name, append=True, inplace=True)
        self.final_energy = self.final_energy.groupby(
            level=self.embodied_energy.index.names).sum()
        self.final_energy = self.final_energy.reorder_levels(
            self.embodied_energy.index.names)
        self.outputs.energy = pd.concat(
            [self.embodied_energy, self.final_energy])
        self.outputs.energy = self.outputs.energy[
            self.outputs.energy['VALUE'] != 0]
        energy_unit = cfg.cfgfile.get('case', 'energy_unit')
        self.outputs.energy.columns = [energy_unit.upper()]
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    """
    def __init__(self, scenario_id, api_run=False):
        self.scenario_id = scenario_id
        self.scenario = Scenario(self.scenario_id)
        self.api_run = api_run
        self.outputs = Output()
        self.demand = Demand(self.scenario)
        self.supply = None
        self.demand_solved, self.supply_solved = False, False

    def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
        try:
            if solve_demand and not (load_demand or load_supply):
                self.calculate_demand(save_models)
            
            if not append_results:
                self.remove_old_results()

            # it is nice if when loading a demand side object to rerun supply, it doesn't re-output these results every time
            if self.demand_solved and export_results and not self.api_run and not (load_demand and solve_supply):
                self.export_result_to_csv('demand_outputs')

            if solve_supply and not load_supply:
                if load_demand:
                    # if we are loading the demand, we are changing the supply measures and want to reload our scenarios
                    self.scenario = Scenario(self.scenario_id)
                self.supply = Supply(self.scenario, demand_object=self.demand)
                self.calculate_supply(save_models)

            if load_demand and solve_supply:
                # we do this now because we delayed before
                self.export_result_to_csv('demand_outputs')

            if self.supply_solved and export_results:
                self.supply.calculate_supply_outputs()
                self.pass_supply_results_back_to_demand()
                self.calculate_combined_results()
                self.outputs.electricity_reconciliation = self.demand.electricity_reconciliation # we want to write these to outputs
                if self.api_run:
                    self.export_results_to_db()
                else:
                    self.export_result_to_csv('supply_outputs')
                    self.export_result_to_csv('combined_outputs')
                    self.export_io()
        except:
            # pickle the model in the event that it crashes
            if save_models:
                Output.pickle(self, file_name=str(scenario_id) + cfg.model_error_append_name, path=cfg.workingdir)
            raise

    def calculate_demand(self, save_models):
        self.demand.setup_and_solve()
        self.demand_solved = True
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_d_payback()
                self.calculate_d_payback_energy()
        if save_models:
            Output.pickle(self, file_name=str(self.scenario_id) + cfg.demand_model_append_name, path=cfg.workingdir)

    def calculate_supply(self, save_models):
        if not self.demand_solved:
            raise ValueError('demand must be solved first before supply')
        logging.info('Configuring energy system supply')
        self.supply.add_nodes()
        self.supply.add_measures()
        self.supply.initial_calculate()
        self.supply.calculated_years = []
        self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
        self.supply.final_calculate()
        self.supply_solved = True
        if save_models:
            Output.pickle(self, file_name=str(self.scenario_id) + cfg.full_model_append_name, path=cfg.workingdir)
            # we don't need the demand side object any more, so we can remove it to save drive space
            if os.path.isfile(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name)):
                os.remove(os.path.join(cfg.workingdir, str(self.scenario_id) + cfg.demand_model_append_name))

    def pass_supply_results_back_to_demand(self):
        logging.info("Calculating link to supply")
        self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
        if cfg.output_tco == 'true':
            if hasattr(self,'d_energy_tco'):
                self.demand.link_to_supply_tco(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link) 
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
        if cfg.output_payback == 'true':
            if hasattr(self,'demand.d_all_energy_demand_payback'):
                self.demand.link_to_supply_payback(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.cost_demand_link) 
            else:
               print  "demand side has not been run with tco outputs set to 'true'"
    
    def calculate_combined_results(self):
        logging.info("Calculating combined emissions results")
        self.calculate_combined_emissions_results()
        logging.info("Calculating combined cost results")
        self.calculate_combined_cost_results()
        logging.info("Calculating combined energy results")
        self.calculate_combined_energy_results()
        if cfg.output_tco == 'true':
            if self.demand.d_energy_tco is not None:
                self.calculate_tco()
        if cfg.output_payback == 'true':
            if self.demand.d_all_energy_demand_payback is not None:
                self.calculate_payback()

    def remove_old_results(self):
        folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
        for folder_name in folder_names:
            folder = os.path.join(cfg.workingdir, folder_name)
            if os.path.isdir(folder):
                shutil.rmtree(folder)

    def export_result_to_csv(self, result_name):
        if result_name=='combined_outputs':
            res_obj = self.outputs
        elif result_name=='demand_outputs':
            res_obj = self.demand.outputs
        elif result_name=='supply_outputs':
            res_obj = self.supply.outputs
        else:
            raise ValueError('result_name not recognized')

        for attribute in dir(res_obj):
            if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
                continue

            result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
            keys = [self.scenario.name.upper(), cfg.timestamp]
            names = ['SCENARIO','TIMESTAMP']
            for key, name in zip(keys, names):
                result_df = pd.concat([result_df], keys=[key], names=[name])

            if attribute in ('hourly_dispatch_results', 'electricity_reconciliation', 'hourly_marginal_cost', 'hourly_production_cost'):
                # Special case for hourly dispatch results where we want to write them outside of supply_outputs
                Output.write(result_df, attribute + '.csv', os.path.join(cfg.workingdir, 'dispatch_outputs'))
            else:
                Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))

    def export_results_to_db(self):
        scenario_run_id = util.active_scenario_run_id(self.scenario_id)
        # Levelized costs
        costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 1, costs)

        #Energy
        energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        # Energy demand by sector
        util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))

        #Emissions
        emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
        # Annual emissions by sector
        util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))

        # Domestic emissions per capita
        annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
        population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
        population_driver.index.name = 'YEAR'
        factor = 1E6
        df = util.DfOper.divi((annual_emissions, population_driver)) * factor
        df.columns = ['TONNE PER CAPITA']
        util.write_output_to_db(scenario_run_id, 4, df)

        # Electricity supply
        electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
        df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
            .xs('EMBODIED', level='ENERGY ACCOUNTING')\
            .groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])

    def calculate_combined_cost_results(self):
        #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        if self.supply.export_costs is not None:
            setattr(self.outputs,'export_costs',self.supply.export_costs)
            self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
            del self.outputs.export_costs
            util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
            self.export_costs_df.columns = [cost_unit.upper()]  
        else:
            self.export_costs_df = None
        #calculate and format emobodied supply costs
        self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
        self.embodied_energy_costs_df.columns = [cost_unit.upper()]
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])
        #calculte and format direct demand costs
        self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
        if self.demand_costs_df is not None:
            levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
            levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
            self.demand_costs_df = self.demand_costs_df.groupby(level=levels_to_keep).sum()
            keys = ["DOMESTIC","DEMAND"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
        self.outputs.c_costs[self.outputs.c_costs<0]=0
        self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
        
    def calculate_tco(self):
#        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
#        del self.demand.outputs.demand_embodied_emissions
        #calculte and format direct demand emissions        
#        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
##        del self.demand.outputs.demand_direct_emissions
#        emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
#         #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_tco
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = self.demand.d_levelized_costs_tco
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        service_demand_df = self.demand.d_service_demand_tco
        service_demand_df = service_demand_df[service_demand_df.index.get_level_values('vintage')>=initial_vintage]
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_tco = pd.concat([util.DfOper.divi([supply_side_df,util.remove_df_levels(service_demand_df,'unit')]),
                                        util.DfOper.divi([demand_side_df,util.remove_df_levels(service_demand_df,'unit')])],
                                        keys=keys,names=names) 
        self.outputs.c_tco = self.outputs.c_tco.replace([np.inf,np.nan],0)
        self.outputs.c_tco[self.outputs.c_tco<0]=0        
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'service_demand') and hasattr(subsector,'stock'):
                    indexer = util.level_specific_indexer(self.outputs.c_tco,'subsector',subsector.id)
                    self.outputs.c_tco.loc[indexer,'unit'] = subsector.service_demand.unit.upper()
        self.outputs.c_tco = self.outputs.c_tco.set_index('unit',append=True)
        self.outputs.c_tco.columns = [cost_unit.upper()]
        self.outputs.c_tco= self.outputs.c_tco[self.outputs.c_tco[cost_unit.upper()]!=0]
        self.outputs.c_tco = self.outputs.return_cleaned_output('c_tco')
        
        
        
    def calculate_payback(self):
#        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions_tco')
#        del self.demand.outputs.demand_embodied_emissions
        #calculte and format direct demand emissions        
#        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
##        del self.demand.outputs.demand_direct_emissions
#        emissions = util.DfOper.add([self.embodied_emissions_df,self.direct_emissions_df])
#         #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        supply_side_df = self.demand.outputs.demand_embodied_energy_costs_payback
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('vintage')>=initial_vintage]
        supply_side_df = supply_side_df[supply_side_df.index.get_level_values('year')>=initial_vintage]
        supply_side_df = supply_side_df.sort_index()
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        demand_side_df = demand_side_df.reindex(supply_side_df.index).sort_index()
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(supply_side_df.index.names)
        sales_df = sales_df.reindex(supply_side_df.index).sort_index()
        keys = ['SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_payback = pd.concat([util.DfOper.divi([supply_side_df, sales_df]), util.DfOper.divi([demand_side_df, sales_df])],keys=keys,names=names)
        self.outputs.c_payback = self.outputs.c_payback[np.isfinite(self.outputs.c_payback.values)]        
        self.outputs.c_payback = self.outputs.c_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.outputs.c_payback,'subsector',subsector.id)
                    self.outputs.c_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.outputs.c_payback = self.outputs.c_payback.set_index('unit', append=True)
        self.outputs.c_payback.columns = [cost_unit.upper()]
        self.outputs.c_payback['lifetime_year'] = self.outputs.c_payback.index.get_level_values('year')-self.outputs.c_payback.index.get_level_values('vintage')+1    
        self.outputs.c_payback = self.outputs.c_payback.set_index('lifetime_year',append=True)
        self.outputs.c_payback = util.remove_df_levels(self.outputs.c_payback,'year')
        self.outputs.c_payback = self.outputs.c_payback.groupby(level = [x for x in self.outputs.c_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.outputs.c_payback = self.outputs.c_payback[self.outputs.c_payback[cost_unit.upper()]!=0]
        self.outputs.c_payback = self.outputs.return_cleaned_output('c_payback')
        
        
    def calculate_d_payback(self):
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_annual_costs_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[np.isfinite(self.demand.outputs.d_payback.values)]        
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback,'subsector',subsector.id)
                    self.demand.outputs.d_payback.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('unit', append=True)
        self.demand.outputs.d_payback.columns = [cost_unit.upper()]
        self.demand.outputs.d_payback['lifetime_year'] = self.demand.outputs.d_payback.index.get_level_values('year')-self.demand.outputs.d_payback.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback = util.remove_df_levels(self.demand.outputs.d_payback,'year')
        self.demand.outputs.d_payback = self.demand.outputs.d_payback.groupby(level = [x for x in self.demand.outputs.d_payback.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback = self.demand.outputs.d_payback[self.demand.outputs.d_payback[cost_unit.upper()]!=0]
        self.demand.outputs.d_payback = self.demand.outputs.return_cleaned_output('d_payback')
   
    def calculate_d_payback_energy(self):
        initial_vintage = min(cfg.supply_years)
        demand_side_df = self.demand.d_all_energy_demand_payback
        demand_side_df.columns = ['value']
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('vintage')>=initial_vintage]
        demand_side_df = demand_side_df[demand_side_df.index.get_level_values('year')>=initial_vintage]
        sales_df = copy.deepcopy(self.demand.outputs.d_sales)
        util.replace_index_name(sales_df,'vintage','year')
        sales_df = sales_df[sales_df.index.get_level_values('vintage')>=initial_vintage]     
        sales_df = util.add_and_set_index(sales_df,'year',cfg.supply_years)
#        sales_df.index = sales_df.index.reorder_levels(demand_side_df.index.names)
#        sales_df = sales_df.reindex(demand_side_df.index).sort_index()
        self.demand.outputs.d_payback_energy = util.DfOper.divi([demand_side_df, sales_df])
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[np.isfinite(self.demand.outputs.d_payback_energy.values)]        
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.replace([np.inf,np.nan],0)
        for sector in self.demand.sectors.values():
          for subsector in sector.subsectors.values():
                if hasattr(subsector,'stock') and subsector.sub_type!='link':
                    indexer = util.level_specific_indexer(self.demand.outputs.d_payback_energy,'subsector',subsector.id)
                    self.demand.outputs.d_payback_energy.loc[indexer,'unit'] = subsector.stock.unit.upper()
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('unit', append=True)
        self.demand.outputs.d_payback_energy.columns = [cfg.calculation_energy_unit.upper()]
        self.demand.outputs.d_payback_energy['lifetime_year'] = self.demand.outputs.d_payback_energy.index.get_level_values('year')-self.demand.outputs.d_payback_energy.index.get_level_values('vintage')+1    
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.set_index('lifetime_year',append=True)
        self.demand.outputs.d_payback_energy = util.remove_df_levels(self.demand.outputs.d_payback_energy,'year')
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy.groupby(level = [x for x in self.demand.outputs.d_payback_energy.index.names if x !='lifetime_year']).transform(lambda x: x.cumsum())
        self.demand.outputs.d_payback_energy = self.demand.outputs.d_payback_energy[self.demand.outputs.d_payback_energy[cfg.calculation_energy_unit.upper()]!=0]
        self.demand.outputs.d_payback_energy = self.demand.outputs.return_cleaned_output('d_payback_energy')
            
        
    def calculate_combined_emissions_results(self):
        #calculate and format export emissions
        if self.supply.export_emissions is not None:
            setattr(self.outputs,'export_emissions',self.supply.export_emissions)
            if 'supply_geography' not in cfg.output_combined_levels:
                util.remove_df_levels(self.outputs.export_emissions, cfg.primary_geography +'_supply')
            self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
            del self.outputs.export_emissions
            util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
        else:
            self.export_emissions_df = None
       #calculate and format emobodied supply emissions
        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
#        del self.demand.outputs.demand_embodied_emissions
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])       
        #calculte and format direct demand emissions        
        self.direct_emissions_df = self.demand.outputs.return_cleaned_output('demand_direct_emissions')
#        del self.demand.outputs.demand_direct_emissions
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key, name in zip(keys, names):
            self.direct_emissions_df = pd.concat([self.direct_emissions_df], keys=[key], names=[name])
        if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
             keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
             names = cfg.primary_geography.upper() +'_SUPPLY'
             self.direct_emissions_df[names] = keys
             self.direct_emissions_df.set_index(names,append=True,inplace=True)
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['EMISSIONS TYPE']
        self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
        self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
        emissions_unit = cfg.cfgfile.get('case','mass_unit')
        self.outputs.c_emissions.columns = [emissions_unit.upper()]
            
    def calculate_combined_energy_results(self):
         energy_unit = cfg.calculation_energy_unit
         if self.supply.export_costs is not None:
            setattr(self.outputs,'export_energy',self.supply.export_energy)
            self.export_energy = self.outputs.return_cleaned_output('export_energy')
            del self.outputs.export_energy
            util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","EMBODIED"]
            names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
            for key,name in zip(keys,names):
                self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
         else:
            self.export_energy = None
         self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
         self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
         keys = ['DOMESTIC','EMBODIED']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
         self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
         self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
         keys = ['DOMESTIC','FINAL']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
    #         self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
         for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
             self.final_energy[name] = "N/A"
             self.final_energy.set_index(name,append=True,inplace=True)
         if self.export_energy is not None:
             for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
                 self.export_energy[name] = "N/A"
                 self.export_energy.set_index(name,append=True,inplace=True)
             self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
             self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
         self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
         self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
         self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
         self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
         self.outputs.c_energy.columns = [energy_unit.upper()]

    def export_io(self):
        io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
        io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
        df_list = []
        for year in io_table_years:
            sector_df_list = []
            keys = self.supply.demand_sectors
            name = ['sector']
            for sector in self.supply.demand_sectors:
                sector_df_list.append(self.supply.io_dict[year][sector])
            year_df = pd.concat(sector_df_list, keys=keys,names=name)
            year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
            df_list.append(year_df)
        keys = io_table_years
        name = ['year']
        df = pd.concat(df_list,keys=keys,names=name)
        for row_sector in self.supply.demand_sectors:
            for col_sector in self.supply.demand_sectors:
                if row_sector != col_sector:
                    df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
        self.supply.outputs.io = df
        result_df = self.supply.outputs.return_cleaned_output('io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
#        self.export_stacked_io()

    def export_stacked_io(self):
        df = copy.deepcopy(self.supply.outputs.io)
        df.index.names = [x + '_input'if x!= 'year' else x for x in df.index.names ]
        df = df.stack(level=df.columns.names).to_frame()
        df.columns = ['value']
        self.supply.outputs.stacked_io = df
        result_df = self.supply.outputs.return_cleaned_output('stacked_io')
        keys = [self.scenario.name.upper(), cfg.timestamp]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        Output.write(result_df, 's_stacked_io.csv', os.path.join(cfg.workingdir, 'supply_outputs'))
Exemple #8
0
class PathwaysModel(object):
    """
    Highest level classification of the definition of an energy system.
    """
    def __init__(self, scenario_id, api_run=False):
        self.scenario_id = scenario_id
        self.api_run = api_run
        self.scenario = cfg.scenario_dict[self.scenario_id]
        self.demand_case_id = util.sql_read_table('Scenarios', 'demand_case', id=self.scenario_id)
        self.supply_case_id = util.sql_read_table('Scenarios', 'supply_case', id=self.scenario_id)
        self.outputs = Output()
        self.demand = Demand()
        self.supply = None
        self.demand_solved, self.supply_solved = False, False

    def run(self, scenario_id, solve_demand, solve_supply, load_demand, load_supply, export_results, save_models, append_results):
        try:
            if solve_demand and not (load_demand or load_supply):
                self.calculate_demand(save_models)
            
            if not append_results:
                self.remove_old_results()

            if hasattr(self, 'demand_solved') and self.demand_solved and export_results and not self.api_run:
                self.export_result_to_csv('demand_outputs')

            if solve_supply and not load_supply:
                self.supply = Supply(self.scenario, demand_object=self.demand)
                self.calculate_supply(save_models)

            if hasattr(self, 'supply_solved') and self.supply_solved and export_results:
                self.supply.calculate_supply_outputs()
                self.pass_supply_results_back_to_demand()
                self.calculate_combined_results()
                if self.api_run:
                    self.export_results_to_db()
                else:
                    self.export_result_to_csv('supply_outputs')
                    self.export_result_to_csv('combined_outputs')
                    self.export_io()
        except:
            # pickle the model in the event that it crashes
            with open(os.path.join(cfg.workingdir, str(scenario_id) + '_model_error.p'), 'wb') as outfile:
                pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)
            raise

    def calculate_demand(self, save_models):
        logging.info('Configuring energy system demand')
        self.demand.add_subsectors()
        self.demand.add_measures(self.demand_case_id)
        self.demand.calculate_demand()
        self.demand_solved = True
        if save_models:
            with open(os.path.join(cfg.workingdir, str(self.scenario_id) + '_model.p'), 'wb') as outfile:
                pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)

    def calculate_supply(self, save_models):
        if not self.demand_solved:
            raise ValueError('demand must be solved first before supply')
        logging.info('Configuring energy system supply')
        self.supply.add_nodes()
        self.supply.add_measures(self.supply_case_id)
        self.supply.initial_calculate()
        self.supply.calculated_years = []
        self.supply.calculate_loop(self.supply.years, self.supply.calculated_years)
        self.supply.final_calculate()
        self.supply.concatenate_annual_costs()
        self.supply.calculate_capacity_utilization()
        self.supply_solved = True
        if save_models:
            with open(os.path.join(cfg.workingdir, str(self.scenario_id) + '_full_model_run.p'), 'wb') as outfile:
                pickle.dump(self, outfile, pickle.HIGHEST_PROTOCOL)

    def pass_supply_results_back_to_demand(self):
        logging.info("Calculating link to supply")
        self.demand.link_to_supply(self.supply.emissions_demand_link, self.supply.demand_emissions_rates, self.supply.energy_demand_link, self.supply.cost_demand_link)
    
    def calculate_combined_results(self):
        logging.info("Calculating combined emissions results")
        self.calculate_combined_emissions_results()
        logging.info("Calculating combined cost results")
        self.calculate_combined_cost_results()
        logging.info("Calculating combined energy results")
        self.calculate_combined_energy_results()

    def remove_old_results(self):
        folder_names = ['combined_outputs', 'demand_outputs', 'supply_outputs', 'dispatch_outputs']
        for folder_name in folder_names:
            folder = os.path.join(cfg.workingdir, folder_name)
            if os.path.isdir(folder):
                shutil.rmtree(folder)

    def export_result_to_csv(self, result_name):
        if result_name=='combined_outputs':
            res_obj = self.outputs
        elif result_name=='demand_outputs':
            res_obj = self.demand.outputs
        elif result_name=='supply_outputs':
            res_obj = self.supply.outputs
        else:
            raise ValueError('result_name not recognized')

        for attribute in dir(res_obj):
            if not isinstance(getattr(res_obj, attribute), pd.DataFrame):
                continue

            result_df = getattr(res_obj, 'return_cleaned_output')(attribute)
            keys = [self.scenario.upper(),str(datetime.now().replace(second=0, microsecond=0))]
            names = ['SCENARIO','TIMESTAMP']
            for key, name in zip(keys, names):
                result_df = pd.concat([result_df], keys=[key], names=[name])
            
            Output.write(result_df, attribute+'.csv', os.path.join(cfg.workingdir, result_name))

    def export_results_to_db(self):
        scenario_run_id = util.active_scenario_run_id(self.scenario_id)
        # Levelized costs
        costs = self.outputs.c_costs.groupby(level=['SUPPLY/DEMAND', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 1, costs)

        #Energy
        energy = self.outputs.c_energy.xs('FINAL', level='ENERGY ACCOUNTING')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        # Energy demand by sector
        util.write_output_to_db(scenario_run_id, 2, energy.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 6, energy.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 8, energy.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 10, energy.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Energy by Fuel Type
        util.write_output_to_db(scenario_run_id, 12, energy.xs('PRODUCTIVE', level='SECTOR'))

        #Emissions
        emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC')\
            .groupby(level=['SECTOR', 'FINAL_ENERGY', 'YEAR']).sum()
        emissions = util.DfOper.mult((emissions, 1-(emissions.abs()<1E-10).groupby(level='FINAL_ENERGY').all())) # get rid of noise
        # Annual emissions by sector
        util.write_output_to_db(scenario_run_id, 3, emissions.groupby(level=['SECTOR', 'YEAR']).sum())
        # Residential Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 7, emissions.xs('RESIDENTIAL', level='SECTOR'))
        # Commercial Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 9, emissions.xs('COMMERCIAL', level='SECTOR'))
        # Transportation Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 11, emissions.xs('TRANSPORTATION', level='SECTOR'))
        # Productive Emissions by Fuel Type
        util.write_output_to_db(scenario_run_id, 13, emissions.xs('PRODUCTIVE', level='SECTOR'))

        # Domestic emissions per capita
        annual_emissions = self.outputs.c_emissions.xs('DOMESTIC', level='EXPORT/DOMESTIC').groupby(level=['YEAR']).sum()
        population_driver = self.demand.drivers[2].values.groupby(level='year').sum().loc[annual_emissions.index]
        population_driver.index.name = 'YEAR'
        factor = 1E6
        df = util.DfOper.divi((annual_emissions, population_driver)) * factor
        df.columns = ['TONNE PER CAPITA']
        util.write_output_to_db(scenario_run_id, 4, df)

        # Electricity supply
        electricity_node_names = [self.supply.nodes[nodeid].name.upper() for nodeid in util.flatten_list(self.supply.injection_nodes.values())]
        df = self.outputs.c_energy.xs('ELECTRICITY', level='FINAL_ENERGY')\
            .xs('EMBODIED', level='ENERGY ACCOUNTING')\
            .groupby(level=['SUPPLY_NODE', 'YEAR']).sum()
        util.write_output_to_db(scenario_run_id, 5, df.loc[electricity_node_names])

    def calculate_combined_cost_results(self):
        #calculate and format export costs
        cost_unit = cfg.cfgfile.get('case','currency_year_id') + " " + cfg.cfgfile.get('case','currency_name')
        if self.supply.export_costs is not None:
            setattr(self.outputs,'export_costs',self.supply.export_costs)
            self.export_costs_df = self.outputs.return_cleaned_output('export_costs')
            del self.outputs.export_costs
            util.replace_index_name(self.export_costs_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_costs_df = pd.concat([self.export_costs_df],keys=[key],names=[name])
            self.export_costs_df.columns = [cost_unit.upper()]  
        else:
            self.export_costs_df = None
        #calculate and format emobodied supply costs
        self.embodied_energy_costs_df = self.demand.outputs.return_cleaned_output('demand_embodied_energy_costs')
        self.embodied_energy_costs_df.columns = [cost_unit.upper()]
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_energy_costs_df = pd.concat([self.embodied_energy_costs_df],keys=[key],names=[name])       
        #calculte and format direct demand costs
        self.demand_costs_df = self.demand.outputs.return_cleaned_output('d_levelized_costs')
        levels_to_keep = [x.upper() for x in cfg.output_combined_levels]
        levels_to_keep = [x for x in levels_to_keep if x in self.demand_costs_df.index.names]
        self.demand_costs_df= self.demand_costs_df.groupby(level=levels_to_keep).sum()
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
            self.demand_costs_df = pd.concat([self.demand_costs_df],keys=[key],names=[name])
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['COST TYPE']
        self.outputs.c_costs = util.df_list_concatenate([self.export_costs_df, self.embodied_energy_costs_df, self.demand_costs_df],keys=keys,new_names=names)
        self.outputs.c_costs[self.outputs.c_costs<0]=0
        self.outputs.c_costs= self.outputs.c_costs[self.outputs.c_costs[cost_unit.upper()]!=0]
        
    def calculate_combined_emissions_results(self):
        #calculate and format export emissions
        if self.supply.export_emissions is not None:
            setattr(self.outputs,'export_emissions',self.supply.export_emissions)
            if 'supply_geography' not in cfg.output_combined_levels:
                util.remove_df_levels(self.outputs.export_emissions,cfg.primary_geography +'_supply')
            self.export_emissions_df = self.outputs.return_cleaned_output('export_emissions')
            del self.outputs.export_emissions
            util.replace_index_name(self.export_emissions_df, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","SUPPLY"]
            names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
            for key,name in zip(keys,names):
                self.export_emissions_df = pd.concat([self.export_emissions_df],keys=[key],names=[name])
        else:
            self.export_emissions_df = None
       #calculate and format emobodied supply emissions
        self.embodied_emissions_df = self.demand.outputs.return_cleaned_output('demand_embodied_emissions')
#        del self.demand.outputs.demand_embodied_emissions
        keys = ["DOMESTIC","SUPPLY"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND"]
        for key,name in zip(keys,names):
           self.embodied_emissions_df = pd.concat([self.embodied_emissions_df],keys=[key],names=[name])       
        #calculte and format direct demand emissions        
        self.direct_emissions_df= self.demand.outputs.return_cleaned_output('demand_direct_emissions')  
#        del self.demand.outputs.demand_direct_emissions
        keys = ["DOMESTIC","DEMAND"]
        names = ['EXPORT/DOMESTIC', "SUPPLY/DEMAND",cfg.primary_geography.upper() +'_EMITTED']
        for key,name in zip(keys,names):
            self.direct_emissions_df = pd.concat([self.direct_emissions_df],keys=[key],names=[name])   
        if cfg.primary_geography+'_supply' in cfg.output_combined_levels:
            keys = self.direct_emissions_df.index.get_level_values(cfg.primary_geography.upper()).values
            names = cfg.primary_geography.upper() +'_SUPPLY'
            self.direct_emissions_df[names] = keys
            self.direct_emissions_df.set_index(names,append=True,inplace=True)
#        levels_to_keep = cfg.output_levels      
#        levels_to_keep = [x.upper() for x in levels_to_keep]
#        levels_to_keep += names + [cfg.primary_geography.upper() +'_SUPPLY', 'SUPPLY_NODE']
        keys = ['EXPORTED', 'SUPPLY-SIDE', 'DEMAND-SIDE']
        names = ['EMISSIONS TYPE']
        self.outputs.c_emissions = util.df_list_concatenate([self.export_emissions_df, self.embodied_emissions_df, self.direct_emissions_df],keys=keys,new_names = names)
#        util.replace_index_name(self.outputs.c_emissions, "ENERGY","FINAL_ENERGY")
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-EMITTED', cfg.primary_geography.upper() +'_SUPPLY')
        util.replace_index_name(self.outputs.c_emissions, cfg.primary_geography.upper() +'-CONSUMED', cfg.primary_geography.upper())
        self.outputs.c_emissions= self.outputs.c_emissions[self.outputs.c_emissions['VALUE']!=0]
        emissions_unit = cfg.cfgfile.get('case','mass_unit')
        self.outputs.c_emissions.columns = [emissions_unit.upper()]
            
    def calculate_combined_energy_results(self):
         energy_unit = cfg.calculation_energy_unit
         if self.supply.export_costs is not None:
            setattr(self.outputs,'export_energy',self.supply.export_energy)
            self.export_energy = self.outputs.return_cleaned_output('export_energy')
            del self.outputs.export_energy
            util.replace_index_name(self.export_energy, 'FINAL_ENERGY','SUPPLY_NODE_EXPORT')
            keys = ["EXPORT","EMBODIED"]
            names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
            for key,name in zip(keys,names):
                self.export_energy = pd.concat([self.export_energy],keys=[key],names=[name])
         else:
            self.export_energy = None
         self.embodied_energy = self.demand.outputs.return_cleaned_output('demand_embodied_energy')
         self.embodied_energy = self.embodied_energy[self.embodied_energy ['VALUE']!=0]
         keys = ['DOMESTIC','EMBODIED']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.embodied_energy = pd.concat([self.embodied_energy],keys=[key],names=[name])
         self.final_energy = self.demand.outputs.return_cleaned_output('d_energy')
         self.final_energy = self.final_energy[self.final_energy.index.get_level_values('YEAR')>=int(cfg.cfgfile.get('case','current_year'))]
         keys = ['DOMESTIC','FINAL']
         names = ['EXPORT/DOMESTIC', 'ENERGY ACCOUNTING']
         for key,name in zip(keys,names):
             self.final_energy = pd.concat([self.final_energy],keys=[key],names=[name])
    #         self.outputs.c_energy = pd.concat([self.embodied_energy, self.final_energy],keys=['DROP'],names=['DROP'])
         for name in [x for x in self.embodied_energy.index.names if x not in self.final_energy.index.names]:
             self.final_energy[name] = "N/A"
             self.final_energy.set_index(name,append=True,inplace=True)
         for name in [x for x in self.embodied_energy.index.names if x not in self.export_energy.index.names]:
             self.export_energy[name] = "N/A"
             self.export_energy.set_index(name,append=True,inplace=True)
         self.final_energy = self.final_energy.groupby(level=self.embodied_energy.index.names).sum()
         self.final_energy = self.final_energy.reorder_levels(self.embodied_energy.index.names)
         self.export_energy = self.export_energy.groupby(level=self.embodied_energy.index.names).sum()
         self.export_energy = self.export_energy.reorder_levels(self.embodied_energy.index.names)
         self.outputs.c_energy = pd.concat([self.embodied_energy,self.final_energy,self.export_energy])
         self.outputs.c_energy= self.outputs.c_energy[self.outputs.c_energy['VALUE']!=0]
         self.outputs.c_energy.columns = [energy_unit.upper()]

    def export_io(self):
        io_table_write_step = int(cfg.cfgfile.get('output_detail','io_table_write_step'))
        io_table_years = sorted([min(cfg.supply_years)] + range(max(cfg.supply_years), min(cfg.supply_years), -io_table_write_step))
        df_list = []
        for year in io_table_years:
            sector_df_list = []
            keys = self.supply.demand_sectors
            name = ['sector']
            for sector in self.supply.demand_sectors:
                sector_df_list.append(self.supply.io_dict[year][sector])
            year_df = pd.concat(sector_df_list, keys=keys,names=name)
            year_df = pd.concat([year_df]*len(keys),keys=keys,names=name,axis=1)
            df_list.append(year_df)
        keys = io_table_years
        name = ['year']
        df = pd.concat(df_list,keys=keys,names=name)
        for row_sector in self.supply.demand_sectors:
            for col_sector in self.supply.demand_sectors:
                if row_sector != col_sector:
                    df.loc[util.level_specific_indexer(df,'sector',row_sector),util.level_specific_indexer(df,'sector',col_sector,axis=1)] = 0
        self.supply.outputs.io = df
        result_df = self.supply.outputs.return_cleaned_output('io')
        keys = [self.scenario.upper(),str(datetime.now().replace(second=0,microsecond=0))]
        names = ['SCENARIO','TIMESTAMP']
        for key, name in zip(keys,names):
            result_df = pd.concat([result_df], keys=[key],names=[name])
        result_df.to_csv(os.path.join(cfg.workingdir,'supply_outputs', 's_io.csv'), header=True, mode='ab')