def __init__(self, sector, directory, activity_data=None, residential_floorspace=None, nominal_energy_intensity=None, end_year=2018, projections=False): self.end_year = end_year self.directory = directory self.sector = sector self.activity_data = activity_data self.nominal_energy_intensity = nominal_energy_intensity self.residential_floorspace = residential_floorspace self.eia_data = GetEIAData(self.sector) self.projections = projections print("WEATHER FACTORS os.getcwd()", os.getcwd()) # self.lmdi_prices = pd.read_excel(os.path.join( # "..", 'Indicators_Spreadsheets_2020', # 'EnergyPrices_by_Sector_010820_DBB.xlsx' # ), sheet_name='LMDI-Prices', header=14, usecols='A:B, EY') self.lmdi_prices = pd.read_excel(os.path.join( os.getcwd(), 'Indicators_Spreadsheets_2020', 'EnergyPrices_by_Sector_010820_DBB.xlsx' ), sheet_name='LMDI-Prices', header=14, usecols='A:B, EY') self.regions_subregions = ['northeast', 'new_england', 'middle_atlantic', 'midwest', 'east_north_central', 'west_north_central', 'south', 'south_atlantic', 'east_south_central', 'west_south_central', 'west', 'mountain', 'pacific'] self.sub_regions_dict = {'northeast': ['New England', 'Middle Atlantic'], 'midwest': ['East North Central', 'West North Central'], 'south': ['South Atlantic', 'East South Central', 'West South Central'], 'west': ['Mountain', 'Pacific']}
def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.sub_categories_list = {'Manufacturing': {'Food, Beverages, & Tobacco': None, 'Textile Mills and Products': None, 'Apparel & Leather': None, 'Wood Products': None, 'Paper': None, 'Printing & Allied Support': None, 'Petroleum & Coal Products': None, 'Chemicals': None, 'Plastics & Rubber Products': None, 'Nonmetallic Mineral Products': None, 'Primary Metals': None, 'Fabricated Metal Products': None, 'Machinery': None, 'Computer & Electronic Products': None, 'Electical Equip. & Appliances': None, 'Transportation Equipment': None, 'Furniture & Related Products': None, 'Miscellaneous': None}, 'Nonmanufacturing': {'Agriculture, Forestry & Fishing': None, 'Mining': {'Petroleum and Natural Gas': None, 'Other Mining': None, 'Petroleum drilling and Mining Services': None}, 'Construction': None}} self.ind_eia = GetEIAData('industry') self.conversion_factors = self.ind_eia.conversion_factors() self.MER_Nov19_Table24 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER10_Table21d = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER11_Table21d_MER0816 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.mer_dataT0204 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.BEA_Output_data = [0] # Chain-type Quantity Indexes for Value Added by Industry from Bureau of Economic Analysis self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] super().__init__(sector='industry', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
def __init__(self, directory, output_directory, sector, config_path, categories_dict, level_of_aggregation): self.sector = sector self.eia = GetEIAData('emissions') self.config_path = config_path self.level_of_aggregation = level_of_aggregation super().__init__(sector, level_of_aggregation=self.level_of_aggregation, categories_dict=categories_dict, directory=directory, output_directory=output_directory, primary_activity=None, unit_conversion_factor=1, weather_activity=None, use_yaml_config=True, config_path=self.config_path)
def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.eia_res = GetEIAData('residential') housing_types = \ {'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None} self.sub_categories_list = \ {'National': {'Northeast': housing_types, 'Midwest': housing_types, 'South': housing_types, 'West': housing_types}} self.national_calibration = \ self.eia_res.national_calibration() self.seds_census_region = \ self.eia_res.get_seds() # energy_consumtpion_data_regional RF = ResidentialFloorspace() self.ahs_Data = RF.update_ahs_data() self.regions = ['Northeast', 'South', 'West', 'Midwest', 'National'] self.base_year = base_year self.directory = directory self.end_year = end_year self.energy_types = ['elec', 'fuels', 'deliv', 'source'] super().__init__(sector='residential', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, energy_types=self.energy_types, directory=directory, output_directory=output_directory, primary_activity='occupied_housing_units', base_year=base_year, end_year=end_year, weather_activity='floorspace_square_feet') print("self.dir()):", dir(self))
def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model=['multiplicative'], end_year=2018, base_year=1985): self.end_year = end_year self.sub_categories_list = { 'Commercial': { 'Commercial_Total': None } } #, 'Total_Commercial_LMDI_UtilAdj': None} self.eia_comm = GetEIAData('commercial') self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] super().__init__(sector='commercial', level_of_aggregation=level_of_aggregation,lmdi_models=lmdi_model, \ directory=directory, output_directory=output_directory, categories_dict=self.sub_categories_list, energy_types=self.energy_types, \ base_year=base_year)
def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.eia_res = GetEIAData('residential') self.sub_categories_list = { 'National': { 'Northeast': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'Midwest': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'South': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'West': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None } } } self.national_calibration = self.eia_res.national_calibration() self.seds_census_region = self.eia_res.get_seds( ) # energy_consumtpion_data_regional self.ahs_Data = ResidentialFloorspace.update_ahs_data() self.conversion_factors = self.eia_res.conversion_factors() self.regions = ['Northeast', 'South', 'West', 'Midwest', 'National'] self.base_year = base_year self.directory = directory self.end_year = end_year self.energy_types = ['elec', 'fuels', 'deliv', 'source'] super().__init__(sector='residential', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
def __init__(self, sector, directory, activity_data=None, residential_floorspace=None, nominal_energy_intensity=None, \ end_year=2018): self.end_year = end_year self.directory = directory self.sector = sector self.activity_data = activity_data self.nominal_energy_intensity = nominal_energy_intensity self.residential_floorspace = residential_floorspace self.eia_data = GetEIAData(self.sector) self.lmdi_prices = pd.read_excel(f'{self.directory}/EnergyPrices_by_Sector_010820_DBB.xlsx', sheet_name='LMDI-Prices', header=14, usecols='A:B, EY') self.regions_subregions = ['northeast', 'new_england', 'middle_atlantic', 'midwest', 'east_north_central', 'west_north_central', 'south', 'south_atlantic', 'east_south_central', 'west_south_central', 'west', 'mountain', 'pacific'] self.sub_regions_dict = {'northeast': ['New England', 'Middle Atlantic'], 'midwest': ['East North Central', 'West North Central'], 'south': ['South Atlantic', 'East South Central', 'West South Central'], 'west': ['Mountain', 'Pacific']}
def residential_projections(self): """Gather Residential projections data activity: - Occupied Housing Units - Floorspace, Square Feet energy: Energy Consumption Trillion Btu """ # Residential : Key Indicators : Average House Square Footage, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_NA_NA_NA_USA_SQFT.A' # Residential : Key Indicators : Households : Mobile Homes, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_MBH_NA_NA_USA_MILL.A' # Residential : Key Indicators : Households : Multifamily, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_MFR_NA_NA_USA_MILL.A' # Residential : Key Indicators : Households : Single-Family, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_SFR_NA_NA_USA_MILL.A' # Residential : Key Indicators : Households : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_TEN_NA_NA_USA_MILL.A' self.sub_regions_dict = {'Northeast': {'New England': 'NENGL', 'Middle Atlantic': 'MATL'}, 'Midwest': {'East North Central': 'ENC', 'West North Central': 'WNC'}, 'South': {'South Atlantic': 'SATL', 'East South Central': 'ESC', 'West South Central': 'WSC'}, 'West': {'Mountain': 'MTN', 'Pacific': 'PCF'}} residential_eia = GetEIAData('residential') energy_use_residential_electricity_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_ELC_NA_NA_QBTU.A', id_type='series') energy_use_residential_total_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_TOT_NA_NA_QBTU.A', id_type='series') energy_use_residential_delivered_energy_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_DELE_NA_NA_QBTU.A', id_type='series') res_housholds= residential_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_HHD_RESD_TEN_NA_NA_USA_MILL.A', id_type='series', new_name='Residential') res_avg_size = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_HHD_RESD_NA_NA_NA_USA_SQFT.A', id_type='series', new_name='Residential') residential_floorspace = res_avg_size.multiply(res_housholds.values) activity_data = {'households': res_housholds, 'average_size': res_avg_size, 'total_floorspace': residential_floorspace} energy_data = {'elec': residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Residential'), 'deliv': residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Residential')} data_dict = {'energy': energy_data, 'activity': activity_data} return data_dict
def commercial_projections(self): """activity: floorspace energy: consumption trillion Btu """ # Commercial : Total Floorspace : New Additions, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_NADN_USA_BLNSQFT.A' # Commercial : Total Floorspace : Surviving, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_SURV_USA_BLNSQFT.A' # Commercial : Total Floorspace : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_TOT_USA_BLNSQFT.A' commercial_categories = {'Commercial_Total': None, 'Total_Commercial_LMDI_UtilAdj': None} commercial_eia = GetEIAData('commercial') energy_use_commercial_electricity_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_ELC_NA_NA_QBTU.A', id_type='series') energy_use_commercial_total_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_TOT_NA_NA_QBTU.A', id_type='series') energy_use_commercial_delivered_energy_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_DELE_NA_NA_QBTU.A', id_type='series') commercial_total_floorspace = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_TOT_USA_BLNSQFT.A', id_type='series') data_dict = {'Commercial_Total': {'energy': {'elec': energy_use_commercial_electricity_us, 'deliv': energy_use_commercial_delivered_energy_us}, 'activity': commercial_total_floorspace}, 'Total_Commercial_LMDI_UtilAdj': None} return data_dict
def transportation_projections(self): """Gather Transportation projections data activity: - Passenger-miles [P-M] (Passenger) - Ton-miles [T-M] (Freight) energy: Energy Consumption Trillion Btu """ # Commercial Carriers? --> domestic air carriers: AEO.2020.AEO2019REF.CNSM_NA_TRN_AIR_DAC_NA_NA_TRLBTU.A # international air carriers: AEO.2020.AEO2019REF.CNSM_NA_TRN_AIR_IAC_NA_NA_TRLBTU.A # Transportation Energy Use : Highway : Commercial Light Trucks, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_CML_NA_NA_TRLBTU.A # Transportation Energy Use : Highway : Freight Trucks : Large, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_LGT26KLBS_NA_TRLBTU.A # Transportation Energy Use : Highway : Freight Trucks : Light Medium, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_LITEMED_NA_TRLBTU.A # Transportation Energy Use : Highway : Freight Trucks : Medium, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_MD10T26KLB_NA_TRLBTU.A # Transportation Energy Use : Highway : Freight Trucks, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_NA_NA_TRLBTU.A # Transportation Energy Use : Non-Highway : Rail : Freight, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_RAIL_FGT_NA_NA_TRLBTU.A transportation_eia = GetEIAData('transportation') transportation_categories = {'All_Passenger': {'Highway': {'Passenger Cars and Trucks': {'Light Trucks – LWB Vehicles': {'Light Trucks': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_LDV_LTRT_NA_TRLBTU.A', id_type='series', new_name='Light Trucks')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_CML_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Light Trucks')}, 'Light-Duty Vehicles': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_LDV_LTRT_NA_TRLBTU.A', id_type='series', new_name='Light-Duty Vehicles')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_LDV_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Light-Duty Vehicles')}}}, 'Buses': {'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020._TRV_TRN_NA_BST_NA_NA_BPM.A', id_type='series', new_name='Buses'), 'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_BUS_NA_NA_TRLBTU.A', id_type='series', new_name='Buses')}}}, 'Rail': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_RAIL_PSG_PSG_NA_TRLBTU.A', id_type='series', new_name='Rail')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020._TRV_TRN_NA_RLP_NA_NA_BPM.A', id_type='series', new_name='Rail')}, 'Air': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_IAC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_AIR_NA_NA_BLNSEATMLS.A', id_type='series', new_name='Air')}}, 'All_Freight': {'Highway': {'Freight-Trucks': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_FGHT_USE_NA_NA_QBTU.A', id_type='series', new_name='Freight-Trucks')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_FGHT_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Freight-Trucks')}}, 'Rail': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_NA_TRN_RAIL_FGT_NA_NA_TRLBTU.A', id_type='series', new_name='Rail')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_RAIL_NA_NA_BLNTNMLS.A', id_type='series', new_name='Rail')}, 'Air': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_FTC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')}, 'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_FTC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')}}} energy_use_transportation_electricity_us = transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Transportation') energy_use_transportation_total_us = transportation_eia.eia_api(id_='YOUR_API_KEY_HERE&series_id=AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_TOT_NA_NA_QBTU.A', id_type='series', new_name='Transportation') energy_use_transportation_delivered_energy_us = transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Transportation') data_dict = transportation_categories return data_dict
def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model=['multiplicative'], base_year=1985): self.sub_categories_list = { 'Elec Generation Total': { 'Elec Power Sector': { 'Electricity Only': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Nuclear': None, 'Hydro Electric': None, 'Renewable': { 'Wood': None, 'Waste': None, 'Geothermal': None, 'Solar': None, 'Wind': None } }, 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Renewable': { 'Wood': None, 'Waste': None } } }, 'Commercial Sector': None, 'Industrial Sector': None }, 'All CHP': { 'Elec Power Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } }, 'Commercial Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Hydroelectric': None, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } }, 'Industrial Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Hydroelectric': None, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } } } } self.elec_power_eia = GetEIAData(sector='electricity') # self.Table21f = pd.read_excel('https://www.eia.gov/totalenergy/data/browser/xls.php?tbl=T02.06') #self.elec_power_eia.eia_api(id_='711254') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711254' # self.Table82a = self.elec_power_eia.eia_api(id_='3') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=3' # self.Table82b = self.elec_power_eia.eia_api(id_='21') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21' # self.Table82c = self.elec_power_eia.eia_api(id_='1736765') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1736765' ? # self.Table82d = self.elec_power_eia.eia_api(id_='711282') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711282' # # self.Table82d_2012_and_later = self.elec_power_eia.eia_api(id_='1017') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1017' # self.Table83d_03 = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ? # self.Table84b = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ? # self.Table85c = self.elec_power_eia.eia_api(id_='379') # ? # self.Table86b = self.elec_power_eia.eia_api(id_='463') # ? # self.MER_T72b_1013_AnnualData = self.elec_power_eia.eia_api(id_='21') #'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21' # self.MER_T72c_1013_AnnualData = self.elec_power_eia.eia_api(id_='2') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=2' self.energy_types = ['primary'] self.Table84c = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03C' ) # Consumption of combustible fuels for electricity generation: Commercial and industrial sectors (selected fuels) # elec_power_eia.eia_api(id_='456') self.Table85c = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03B' ) # Consumption of combustible fuels for electricity generation: Electric power sector self.Table82d = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.02C') super().__init__(sector='electric', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
class ElectricityIndicators(CalculateLMDI): def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model=['multiplicative'], base_year=1985): self.sub_categories_list = { 'Elec Generation Total': { 'Elec Power Sector': { 'Electricity Only': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Nuclear': None, 'Hydro Electric': None, 'Renewable': { 'Wood': None, 'Waste': None, 'Geothermal': None, 'Solar': None, 'Wind': None } }, 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Renewable': { 'Wood': None, 'Waste': None } } }, 'Commercial Sector': None, 'Industrial Sector': None }, 'All CHP': { 'Elec Power Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } }, 'Commercial Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Hydroelectric': None, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } }, 'Industrial Sector': { 'Combined Heat & Power': { 'Fossil Fuels': { 'Coal': None, 'Petroleum': None, 'Natural Gas': None, 'Other Gasses': None }, 'Hydroelectric': None, 'Renewable': { 'Wood': None, 'Waste': None }, 'Other': None } } } } self.elec_power_eia = GetEIAData(sector='electricity') # self.Table21f = pd.read_excel('https://www.eia.gov/totalenergy/data/browser/xls.php?tbl=T02.06') #self.elec_power_eia.eia_api(id_='711254') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711254' # self.Table82a = self.elec_power_eia.eia_api(id_='3') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=3' # self.Table82b = self.elec_power_eia.eia_api(id_='21') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21' # self.Table82c = self.elec_power_eia.eia_api(id_='1736765') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1736765' ? # self.Table82d = self.elec_power_eia.eia_api(id_='711282') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711282' # # self.Table82d_2012_and_later = self.elec_power_eia.eia_api(id_='1017') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1017' # self.Table83d_03 = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ? # self.Table84b = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ? # self.Table85c = self.elec_power_eia.eia_api(id_='379') # ? # self.Table86b = self.elec_power_eia.eia_api(id_='463') # ? # self.MER_T72b_1013_AnnualData = self.elec_power_eia.eia_api(id_='21') #'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21' # self.MER_T72c_1013_AnnualData = self.elec_power_eia.eia_api(id_='2') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=2' self.energy_types = ['primary'] self.Table84c = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03C' ) # Consumption of combustible fuels for electricity generation: Commercial and industrial sectors (selected fuels) # elec_power_eia.eia_api(id_='456') self.Table85c = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03B' ) # Consumption of combustible fuels for electricity generation: Electric power sector self.Table82d = pd.read_csv( 'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.02C') super().__init__(sector='electric', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year) @staticmethod def get_eia_aer(): """Prior to 2012, the data for the indicators were taken directly from tables published (and downloaded in Excel format) from EIA's Annual Energy Review. """ pass @staticmethod def get_reconciles(): """ The EIA Annual Energy Review data (pre 2012) for energy consumption to produce electricity were generally supplied in physical units only (e.g., mcf of natural gas, tons of coal, etc.) The values needed to be converted to Btu, and still be consistent with aggregate energy consumption for this sector as published by EIA. For each major fossil fuel, a separate worksheet was developed; these worksheets are identified with the suffix “reconcile.” Thus, the worksheet “NatGas Reconcile” seeks to produce an estimate of the Btu consumption of natural gas used to generate electricity. Similar worksheets were developed for coal, petroleum, and other fuels. /// Does this need to be done or can download Btu directly instead of converting physical units to Btu --> no """ pass def process_utility_level_data(self): """The indicators for electricity are derived entirely from data collected by EIA. Since 2012, the indicators are based entirely upon te EIA-923 survey """ eia_923_schedules = pd.read_excel('./') net_generation = pd.pivot_table(eia_923_schedules, columns='EIA Sector Number', index='AERFuel Type Code', aggfunc='sum') # page A-71, # 'Net Generation' lower right-hand quadrant? net_generation['Grand_Total'] = net_generation.sum( axis=1, skipna=True ) # Should have 18 rows labeled by type of fuel and seven columns # plus one for 'Grand Total'. Note: rows is not an arg of pd.pivot_table elec_btu_consumption = pd.pivot_table(eia_923_schedules, columns='EIA Sector Number', index='AERFuel Type Code', aggfunc='sum') # page A-71, # 'Elec Fuel ConsumptionMMBTU' lower right-hand quadrant? elec_btu_consumption['Grand_Total'] = elec_btu_consumption.sum( axis=1, skipna=True ) # Should have 18 rows labeled by type of fuel and seven columns # plus one for 'Grand Total' previous_years_net_gen = pd.read_excel('./') previous_yeas_elec_btu_consumption = pd.read_excel('./') master_net_gen = previous_years_net_gen.concat(net_generation) maseter_elec_btu_consumption = previous_yeas_elec_btu_consumption.concat( elec_btu_consumption) # Aggregate data?? page A-72 fpr net generation and elec btu consumption return None @staticmethod def reconcile(total, elec_gen, elec_only_plants, chp_elec, assumed_conv_factor, chp_heat): """total: df, Btu elec_gen: df, Btu elec_only_plants: df, Short tons chp_elec: df, Short tons assumed_conv_factor: float, MMBtu/Ton chp_heat """ difference = total.subtract(elec_gen) # Btu implied_conversion_factor = total.divide(elec_only_plants).multiply( 1000) # MMBtu/Ton elec_only_billionbtu = elec_only_plants.multiply( assumed_conv_factor).multiply(1000) # Billion Btu chp_elec_billionbtu = chp_elec.multiply(assumed_conv_factor).multiply( 0.001) # Billion Btu chp_heat_billionbtu = chp_heat.multiply(assumed_conv_factor).multiply( 0.001) # Billion Btu total_fuel = elec_only_billionbtu.add(chp_elec_billionbtu).add( chp_heat_billionbtu) # Cross Check total_short_tons = elec_only_plants + chp_elec + chp_heat # Short Tons implied_conversion_factor_cross = total.divide( total_short_tons).multiply(1000) # MMBtu/Ton implied_conversion_factor_revised = elec_gen.divide( chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton chp_plants_fuel = implied_conversion_factor_revised.multiply( chp_elec).multiply(0.000001) # Trillion Btu elec_only_fuel = elec_gen.multiply(.001).subtract( chp_plants_fuel) # Trillion Btu resulting_total = chp_plants_fuel.add(elec_only_fuel) return chp_plants_fuel, elec_only_fuel def coal_reconcile(self): energy_consumption_coal = self.elec_power_eia.eia_api( id_='TOTAL.CLEIBUS.A', id_type='series') # Table21f11 column b consumption_for_electricity_generation_coal = self.elec_power_eia.eia_api( id_='TOTAL.CLEIBUS.A', id_type='series') # Table84b11 column b consumption_combustible_fuels_electricity_generation_coal = self.elec_power_eia.eia_api( id_='TOTAL.CLL1PUS.A', id_type='series' ) # Table85c11 column B SHOULD BE separated Elec-only/CHP consumption_combustible_fuels_useful_thermal_output_coal = self.elec_power_eia.eia_api( id_='TOTAL.CLEIPUS.A', id_type='series') # Table86b11 column B assumed_conversion_factor = 20.9 total = energy_consumption_coal elec_gen = consumption_for_electricity_generation_coal elec_only_plants = consumption_combustible_fuels_electricity_generation_coal # should be separate part? chp_elec = consumption_combustible_fuels_electricity_generation_coal # should be separate part? assumed_conv_factor = assumed_conversion_factor chp_heat = consumption_combustible_fuels_useful_thermal_output_coal # eia-923 pivot table difference = total.subtract(elec_gen) # Btu implied_conversion_factor = total.divide(elec_only_plants).multiply( 1000) # MMBtu/Ton elec_only_billionbtu = elec_only_plants.multiply( assumed_conv_factor).multiply(1000) # Billion Btu chp_elec_billionbtu = chp_elec.multiply(assumed_conv_factor).multiply( 0.001) # Billion Btu chp_heat_billionbtu = chp_heat.multiply(assumed_conv_factor).multiply( 0.001) # Billion Btu total_fuel = elec_only_billionbtu.add(chp_elec_billionbtu).add( chp_heat_billionbtu) # Cross Check total_short_tons = elec_only_plants + chp_elec + chp_heat # Short Tons implied_conversion_factor_cross = total.divide( total_short_tons).multiply(1000) # MMBtu/Ton implied_conversion_factor_revised = elec_gen.divide( chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton chp_plants_fuel = implied_conversion_factor_revised.multiply( chp_elec).multiply(0.000001) # Trillion Btu elec_only_fuel = elec_gen.multiply(.001).subtract( chp_plants_fuel) # Trillion Btu resulting_total = chp_plants_fuel.add(elec_only_fuel) return chp_plants_fuel, elec_only_fuel def natgas_reconcile(self): energy_consumption_natgas = self.elec_power_eia.eia_api( id_='TOTAL.NNEIBUS.A', id_type='series') # Table21f11 column d consumption_for_electricity_generation_natgas = self.elec_power_eia.eia_api( id_='TOTAL.NNEIBUS.A', id_type='series') # Table84b11 column f consumption_combustible_fuels_electricity_generation_natgas = self.elec_power_eia.eia_api( id_='TOTAL.NGL1PUS.A', id_type='series') # Table85c11 column N consumption_combustible_fuels_useful_thermal_output_natgas = self.elec_power_eia.eia_api( id_='TOTAL.NGEIPUS.A', id_type='series') # Table86b11 column M # eia-923 pivot table """total: df, Billion Btu elec_gen: df, Billion Btu elec_only_plants: df, Thou. Cu. Ft. chp_elec: df, Thou. CF assumed_conv_factor: float, MMBtu/Ton chp_heat: df, Thou. CF """ total = energy_consumption_natgas elec_gen = consumption_for_electricity_generation_natgas elec_only_plants = consumption_combustible_fuels_electricity_generation_natgas chp_elec = consumption_combustible_fuels_electricity_generation_natgas # different part assumed_conv_factor = 1.028 chp_heat = consumption_combustible_fuels_useful_thermal_output_natgas difference = total.subtract(elec_gen) # Billion Btu implied_conversion_factor = elec_gen.divide(elec_only_plants).multiply( 1000) # kBtu/CF * different from coal_reconcile elec_only_trillionbtu = elec_only_plants.multiply( assumed_conv_factor).multiply(0.001) # Trillion Btu chp_elec_trillionbtu = chp_elec.multiply(assumed_conv_factor).multiply( 0.001) # Trillion Btu chp_heat_trillionbtu = chp_heat.multiply(assumed_conv_factor).multiply( 0.001) # Trillion Btu total_fuel = elec_only_trillionbtu.add(chp_elec_trillionbtu).add( chp_heat_trillionbtu) # Cross Check total_thou_cf = elec_only_plants + chp_elec + chp_heat # Thou. CF implied_conversion_factor_cross = total.divide(total_thou_cf).multiply( 1000) # kBtu/CF implied_conversion_factor_revised = elec_gen.divide( chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton chp_plants_fuel = implied_conversion_factor_revised.multiply( chp_elec).multiply(0.000001) # Trillion Btu elec_only_fuel = elec_gen.multiply(.001).subtract( chp_plants_fuel) # Trillion Btu resulting_total = chp_plants_fuel.add(elec_only_fuel) return chp_plants_fuel, elec_only_fuel def petroleum_reconcile(self): energy_consumption_petroleum = self.elec_power_eia.eia_api( id_='TOTAL.PAEIBUS.A', id_type='series') # Table21f11 column F consumption_for_electricity_generation_petroleum = self.elec_power_eia.eia_api( id_='TOTAL.PAEIBUS.A', id_type='series') # Table84b11 column D consumption_combustible_fuels_electricity_generation_distillate_fuel_oil = self.elec_power_eia.eia_api( id_='TOTAL.DKL1PUS.A', id_type='series') # Table85c11 column D, F, H, J consumption_combustible_fuels_electricity_generation_residual_fuel_oil = self.elec_power_eia.eia_api( id_='TOTAL.RFL1PUS.A', id_type='series') consumption_combustible_fuels_electricity_generation_other_liquids = self.elec_power_eia.eia_api( id_='TOTAL.OLL1PUS.A', id_type='series') consumption_combustible_fuels_electricity_generation_petroleum_coke = self.elec_power_eia.eia_api( id_='TOTAL.PCL1MUS.A', id_type='series') consumption_combustible_fuels_electricity_generation_total_petroleum = self.elec_power_eia.eia_api( id_='TOTAL.PAL1PUS.A', id_type='series') fuels_list = [ consumption_combustible_fuels_electricity_generation_distillate_fuel_oil, consumption_combustible_fuels_electricity_generation_residual_fuel_oil, consumption_combustible_fuels_electricity_generation_other_liquids, consumption_combustible_fuels_electricity_generation_petroleum_coke ] elec_only_plants_petroleum = reduce( lambda x, y: pd.merge(x, y, on='Period'), fuels_list) consumption_combustible_fuels_useful_thermal_output_distillate_fuel_oil = self.elec_power_eia.eia_api( id_='TOTAL.DKEIPUS.A', id_type='series') # Table86b11 column D, F, G, I consumption_combustible_fuels_useful_thermal_output_residual_fuel_oil = self.elec_power_eia.eia_api( id_='TOTAL.RFEIPUS.A', id_type='series') consumption_combustible_fuels_useful_thermal_output_other_liquids = self.elec_power_eia.eia_api( id_='TOTAL.OLEIPUS.A', id_type='series') consumption_combustible_fuels_useful_thermal_output_petroleum_coke = self.elec_power_eia.eia_api( id_='TOTAL.PCEIMUS.A', id_type='series') consumption_combustible_fuels_useful_thermal_output_total_petroleum = self.elec_power_eia.eia_api( id_='TOTAL.PTEIPUS.A', id_type='series') chp_elec = consumption_combustible_fuels_electricity_generation_total_petroleum # eia-923 pivot table assumed_conversion_factors = [5.825, 6.287, 5.67, 30120] chp_plants_petroleum, elec_only_petroleum = self.reconcile( total=energy_consumption_petroleum, elec_gen=consumption_for_electricity_generation_petroleum, elec_only_plants=elec_only_plants_petroleum, chp_elec=chp_elec, assumed_conv_factor=assumed_conversion_factors, chp_heat= consumption_combustible_fuels_useful_thermal_output_total_petroleum ) return chp_plants_petroleum, elec_only_petroleum def othgas_reconcile(self): consumption_for_electricity_generation_fossil_fuels = self.elec_power_eia.eia_api( id_='TOTAL.FFEIBUS.A', id_type='series') # Table84b11 column H consumption_for_electricity_generation_oth_gas = consumption_for_electricity_generation_fossil_fuels - consumption_for_electricity_generation_petroleum - consumption_for_electricity_generation_natgas - consumption_for_electricity_generation_coal consumption_combustible_fuels_electricity_generation_oth_gas = self.elec_power_eia.eia_api( id_='TOTAL.OJL1BUS.A', id_type='series') # Table85c11 column P consumption_combustible_fuels_useful_thermal_output_othgas = self.elec_power_eia.eia_api( id_='TOTAL.OJEIBUS.A', id_type='series') # Table86b11 column O elec_gen = consumption_for_electricity_generation_oth_gas elec_only_plants = consumption_combustible_fuels_electricity_generation_oth_gas chp_elec = consumption_combustible_fuels_electricity_generation_oth_gas # ** different part of the series?' chp_heat = consumption_combustible_fuels_useful_thermal_output_othgas total_other_gas = elec_only_plants.add(chp_elec).add(chp_heat) return chp_elec # consumption_combustible_fuels_electricity_generation = # print(consumption_combustible_fuels_electricity_generation) # consumption_combustible_fuels_useful_thermal_output = # print(consumption_combustible_fuels_useful_thermal_output) def industrial_sector_chp_renew(self): wood_energy = self.Table84c[ self.Table84c['Description'] == 'Wood Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) waste_energy = self.Table84c[ self.Table84c['Description'] == 'Waste Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4C column R # TBtu wood_activity = self.elec_power_eia.eia_api( id_='TOTAL.WDI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column R waste_activity = self.elec_power_eia.eia_api( id_='TOTAL.WSI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column T data_dict = { 'Wood': { 'energy': { 'primary': wood_energy }, 'activity': wood_activity }, 'Waste': { 'energy': { 'primary': waste_energy }, 'activity': waste_activity } } return data_dict def industrial_sector_chp_fossil(self): coal_energy = self.Table84c[ self.Table84c['Description'] == 'Coal Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4c column B # TBtu petroleum_energy = self.Table84c[ self.Table84c['Description'] == 'Petroleum Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4c column D # TBtu natgas_energy = self.Table84c[ self.Table84c['Description'] == 'Natural Gas Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4c column F # TBtu othgas_energy = self.Table84c[ self.Table84c['Description'] == 'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4c column H # TBtu coal_activity = self.elec_power_eia.eia_api( id_='TOTAL.CLI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column B petroleum_activity = self.elec_power_eia.eia_api( id_='TOTAL.PAI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column D natgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.NGI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column F othgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.OJI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column H data_dict = { 'Coal': { 'energy': { 'primary': coal_energy }, 'activity': coal_activity }, 'Petroleum': { 'energy': { 'primary': petroleum_energy }, 'activity': petroleum_activity }, 'Natural Gas': { 'energy': { 'primary': natgas_energy }, 'activity': natgas_activity }, 'Other Gasses': { 'energy': { 'primary': othgas_energy }, 'activity': othgas_activity } } return data_dict def industrial_sector_total(self): """Note: Other includes batteries, chemicals, hydrogen, pitch, purchased steam, sulfur, and miscellaneous technologies """ hydroelectric_energy = None #.multiply(0.001) # Table 8.4C11 column N other_energy = self.Table84c[ self.Table84c['Description'] == 'Other Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4C11 column AB hydroelectric_activity = self.elec_power_eia.eia_api( id_='TOTAL.HVI5PUS.A', id_type='series').multiply(0.001) # Table 8.2d11 Column P other_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d11 Column AD data_dict = { 'Hydroelectric': { 'energy': { 'primary': hydroelectric_energy }, 'activity': hydroelectric_activity }, 'Other': { 'energy': { 'primary': other_energy }, 'activity': other_activity } } return data_dict def comm_sector_chp_renew(self): """As is, these are the same sources as ind sector, but should be different part of columns? """ wood_energy = self.Table84c[ self.Table84c['Description'] == 'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4C column P # TBtu waste_energy = self.Table84c[ self.Table84c['Description'] == 'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply( 0.001) # Table 8.4C column R # TBtu wood_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d column R waste_activity = self.elec_power_eia.eia_api( id_='TOTAL.WSC5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column T data_dict = { 'Wood': { 'energy': { 'primary': wood_energy }, 'activity': wood_activity }, 'Waste': { 'energy': { 'primary': waste_energy }, 'activity': waste_activity } } return data_dict def comm_sector_chp_fossil(self): coal_energy = self.Table84c[ self.Table84c['Description'] == 'Coal Consumption for Electricity Generation, Commercial Sector'].multiply( 0.001) # Table 8.4c column B # TBtu petroleum_energy = self.Table84c[ self.Table84c['Description'] == 'Petroleum Consumption for Electricity Generation, Commercial Sector'].multiply( 0.001) # Table 8.4c column D # TBtu natgas_energy = self.Table84c[ self.Table84c['Description'] == 'Natural Gas Consumption for Electricity Generation, Commercial Sector'].multiply( 0.001) # Table 8.4c column F # TBtu othgas_energy = None #.multiply(0.001) # Table 8.4c column H # TBtu coal_activity = self.elec_power_eia.eia_api( id_='TOTAL.CLC5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column B petroleum_activity = self.elec_power_eia.eia_api( id_='TOTAL.PAC5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column D natgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.NGC5PUS.A', id_type='series').multiply(0.001) # Table 8.2d column F othgas_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d column H # Industrial and Commercial have same spreadsheet sources but different parts of the columns data_dict = { 'Coal': { 'energy': { 'primary': coal_energy }, 'activity': coal_activity }, 'Petroleum': { 'energy': { 'primary': petroleum_energy }, 'activity': petroleum_activity }, 'Natural Gas': { 'energy': { 'primary': natgas_energy }, 'activity': natgas_activity }, 'Other Gasses': { 'energy': { 'primary': othgas_energy }, 'activity': othgas_activity } } return data_dict def comm_sector_total(self): """Note: Other includes batteries, chemicals, hydrogen, pitch, purchased steam, sulfur, and miscellaneous technologies """ hydroelectric_energy = None #.multiply(0.001) # Table 8.4C11 column N other_energy = None # .multiply(0.001) # Table 8.4C11 column AB hydroelectric_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d11 Column P other_activity = None # self.Table82d contains 'Electricity Net Generation Total (including from sources not shown), Commercial Sector', # could get "other" by subtracting other categories from ^ ?.multiply(0.001) # Table 8.2d11 Column AD data_dict = { 'Hydroelectric': { 'energy': { 'primary': hydroelectric_energy }, 'activity': hydroelectric_activity }, 'Other': { 'energy': { 'primary': other_energy }, 'activity': other_activity } } return data_dict def elec_power_sector_chp_renew(self): wood_energy = self.Table85c[ self.Table85c['Description'] == 'Wood Consumption for Electricity Generation, Electric Power Sector'].multiply( 0.001) # Table 8.5C column R # TBtu waste_energy = self.Table85c[ self.Table85c['Description'] == 'Waste Consumption for Electricity Generation, Electric Power Sector'].multiply( 0.001) # Table 8.5C column T # TBtu wood_activity = self.elec_power_eia.eia_api( id_='TOTAL.WDEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column R waste_activity = self.elec_power_eia.eia_api( id_='TOTAL.WSEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column T data_dict = { 'Wood': { 'energy': { 'primary': wood_energy }, 'activity': wood_activity }, 'Waste': { 'energy': { 'primary': waste_energy }, 'activity': waste_activity } } return data_dict def elec_power_sector_chp_fossil(self): coal_energy = self.coal_reconcile() petroleum_energy = self.petroleum_reconcile() natgas_energy = self.natgas_reconcile() othgas_energy = self.othgas_reconcile() coal_activity = self.elec_power_eia.eia_api( id_='TOTAL.CLEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column B petroleum_activity = self.elec_power_eia.eia_api( id_='TOTAL.PAEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column D natgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.NGEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column F othgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.OJEGPUS.A', id_type='series').multiply(0.001) # Table 8.2c column H data_dict = { 'Coal': { 'energy': { 'primary': coal_energy }, 'activity': coal_activity }, 'Petroleum': { 'energy': { 'primary': petroleum_energy }, 'activity': petroleum_activity }, 'Natural Gas': { 'energy': { 'primary': natgas_energy }, 'activity': natgas_activity }, 'Other Gasses': { 'energy': { 'primary': othgas_energy }, 'activity': othgas_activity } } return data_dict def elec_power_sector_chp_total(self): other_energy = self.Table85c[ self.Table85c['Description'] == 'Other Consumption for Electricity Generation, Electric Power Sector'].multiply( 0.001) # Table 8.5c column V other_activty = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2c columnAD data_dict = { 'Other': { 'energy': { 'primary': other_energy }, 'activity': other_activty } } return data_dict def electricity_only_renew(self): wood_activity = self.elec_power_eia.eia_api( id_='TOTAL.WDL1BUS.A', id_type='series').multiply(0.001) # Table 8.3d I, 8.5C R waste_activity = self.elec_power_eia.eia_api( id_='TOTAL.WSL1BUS.A', id_type='series').multiply(0.001) # Table 8.3d J, 8.5C T geothermal_activity = self.elec_power_eia.eia_api( id_='TOTAL.GEEGBUS.A', id_type='series').multiply(0.001) # Table 8.4 T solar_activity = self.elec_power_eia.eia_api( id_='TOTAL.SOEGBUS.A', id_type='series').multiply(0.001) # Table 8.4 V wind_activity = self.elec_power_eia.eia_api(id_='TOTAL.WYEGBUS.A', id_type='series').multiply( 0.001) # Table 8.4 X wood_energy = self.elec_power_eia.eia_api( id_='TOTAL.WDEGPUS.A', id_type='series').multiply(0.001) # Table 8.2b S, 8.2c R waste_energy = self.elec_power_eia.eia_api( id_='TOTAL.WSEGPUS.A', id_type='series').multiply(0.001) # Table 8.2b T, 8.2c T geothermal_energy = self.elec_power_eia.eia_api( id_='TOTAL.GEEGPUS.A', id_type='series').multiply(0.001) # Table 8.2b V, 8.2c V solar_energy = self.elec_power_eia.eia_api( id_='TOTAL.SOEGPUS.A', id_type='series').multiply(0.001) # Table 8.2b X, 8.2c X wind_energy = self.elec_power_eia.eia_api( id_='TOTAL.WYEGPUS.A', id_type='series').multiply(0.001) # Table 8.2b Z, 8.2c Z data_dict = { 'Wood': { 'energy': { 'primary': wood_energy }, 'activity': wood_activity }, 'Waste': { 'energy': { 'primary': waste_energy }, 'activity': waste_activity }, 'Geothermal': { 'energy': { 'primary': geothermal_energy }, 'activity': geothermal_activity }, 'Solar': { 'energy': { 'primary': solar_energy }, 'activity': solar_activity }, 'Wind': { 'energy': { 'primary': wind_energy }, 'activity': wind_activity } } return data_dict def electricity_only_fossil(self): coal_energy = self.coal_reconcile() petroleum_energy = self.petroleum_reconcile() natgas_energy = self.natgas_reconcile() othgas_energy = self.othgas_reconcile() coal_activity = self.elec_power_eia.eia_api( id_='TOTAL.CLEGPUS.A', id_type='series').multiply(0.001) # 8.2b B, 8.2c B petroleum_activity = self.elec_power_eia.eia_api( id_='TOTAL.PAEGPUS.A', id_type='series').multiply(0.001) # 8.2b D, 8.2c D natgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.NGEGPUS.A', id_type='series').multiply(0.001) # 8.2b B, 8.2c F othgas_activity = self.elec_power_eia.eia_api( id_='TOTAL.OJEGPUS.A', id_type='series').multiply(0.001) # 8.2c H data_dict = { 'Coal': { 'energy': { 'primary': coal_energy }, 'activity': coal_activity }, 'Petroleum': { 'energy': { 'primary': petroleum_energy }, 'activity': petroleum_activity }, 'Natural Gas': { 'energy': { 'primary': natgas_energy }, 'activity': natgas_activity }, 'Other Gasses': { 'energy': { 'primary': othgas_energy }, 'activity': othgas_activity } } return data_dict def electricity_only_total(self): nuclear_energy = self.elec_power_eia.eia_api( id_='TOTAL.NUEGBUS.A', id_type='series').multiply(0.001) # 8.4b L hydroelectric_energy = self.elec_power_eia.eia_api( id_='TOTAL.HVEGBUS.A', id_type='series').multiply(0.001) # 8.4b N nuclear_activity = self.elec_power_eia.eia_api( id_='TOTAL.NUEGPUS.A', id_type='series').multiply(0.001) # 8.2b L hydroelectric_activity = self.elec_power_eia.eia_api( id_='TOTAL.HVEGPUS.A', id_type='series').multiply(0.001) # 8.2b O data_dict = { 'Hydro Electric': { 'energy': { 'primary': hydroelectric_energy }, 'activity': hydroelectric_activity }, 'Nuclear': { 'energy': { 'primary': nuclear_energy }, 'activity': nuclear_activity } } return data_dict def collect_data(self): industrial_sector_chp_renew = self.industrial_sector_chp_renew() industrial_sector_chp_fossil = self.industrial_sector_chp_fossil() industrial_sector_total = self.industrial_sector_total() industrial_sector_total['Fossil Fuels'] = industrial_sector_chp_fossil industrial_sector_total['Renewable'] = industrial_sector_chp_renew comm_sector_chp_renew = self.comm_sector_chp_renew() comm_sector_chp_fossil = self.comm_sector_chp_fossil() comm_sector_total = self.comm_sector_total() comm_sector_total['Fossil Fuels'] = comm_sector_chp_fossil comm_sector_total['Renewable'] = comm_sector_chp_renew elec_power_sector_chp_renew = self.elec_power_sector_chp_renew() elec_power_sector_chp_fossil = self.elec_power_sector_chp_fossil() elec_power_sector_chp_total = self.elec_power_sector_chp_total() elec_power_sector_chp_total[ 'Fossil Fuels'] = elec_power_sector_chp_fossil elec_power_sector_chp_total['Renewable'] = elec_power_sector_chp_renew all_chp = dict() all_chp['Industrial Sector'] = industrial_sector_total all_chp['Commercial Sector'] = comm_sector_total all_chp['Elec Power Sector'] = elec_power_sector_chp_total electricity_only_renew = self.electricity_only_renew() electricity_only_fossil = self.electricity_only_fossil() electricity_only_total = self.electricity_only_total() electricity_only_total['Fossil Fuels'] = electricity_only_fossil electricity_only_total['Renewable'] = electricity_only_total chp_elec_power_sector = dict() chp_renew = self.chp_renew() chp_fossil = self.chp_fossil() chp_elec_power_sector['Fossil Fuels'] = chp_fossil chp_elec_power_sector['Renewable'] = chp_renew elec_power_sector = dict() elec_power_sector['Electricity Only'] = electricity_only_total elec_power_sector['Combined Heat & Power'] = chp_elec_power_sector elec_generation_total = dict( ) # Need to build this from commerical and industrial_totals elec_generation_total['Elec Power Sector'] = elec_power_sector elec_generation_total['Commercial Sector'] = comm_sector_total elec_generation_total['Industrial Sector'] = industrial_sector_total data_dict = { 'Elec Generation Total': elec_generation_total, 'All CHP': all_chp } return data_dict def main(self, breakout, save_breakout, calculate_lmdi): data_dict = self.collect_data() print(data_dict) results_dict, formatted_results = self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, save_breakout=save_breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict)
class CommercialIndicators(CalculateLMDI): """ Data Sources: - New construction is based on data from Dodge Data and Analytics. Dodge data on new floor space additions is available from the published versions of the Statistical Abstract of the United States (SAUS). The Most recent data is from the 2020 SAUS, Table 995 "Construction Contracts Started- Value of the Construction and Floor Space of Buildings by Class of Construction: 2014 to 2018". """ def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model=['multiplicative'], end_year=2018, base_year=1985): self.end_year = end_year self.sub_categories_list = { 'Commercial_Total': None } #, 'Total_Commercial_LMDI_UtilAdj': None} self.eia_comm = GetEIAData('commercial') self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] super().__init__(sector='commercial', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, directory=directory, output_directory=output_directory, categories_dict=self.sub_categories_list, energy_types=self.energy_types, base_year=base_year, end_year=end_year) # self.cbecs = # self.residential_housing_units = [0] # Use regional estimates of residential housing units as interpolator, extrapolator via regression model # self.mer_data23_May_2016 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251' # self.mer_data23_Jan_2017 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251' # self.mer_data23_Dec_2019 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251' # self.AER11_Table21C_Update = GetEIAData.eia_api(id_='711251') # Estimates? def collect_input_data(self, dataset_name): datasets = \ {'national_calibration': self.eia_comm.national_calibration(), 'SEDS_CensusRgn': self.eia_comm.get_seds(), 'mer_data_23': self.eia_comm.eia_api(id_='711251', id_type='category')} return datasets[dataset_name] def adjusted_supplier_data(self): """ This worksheet adjusts some of commercial energy consumption data as reported in the Annual Energy Review. These adjustments are based upon state-by-state analysis of energy consumption in the industrial and commercial sectors. For electricity, there have been a number of reclassifications by utilities since 1990 that has moved sales from the industrial sector to the commercial sector. The adjustment for electricity consumption is based upon a state-by-state examination of commercial and electricity sales from 1990 through 2011. This data is collected by EIA via Survey EIA-861. Significant discontinuities in the sales data from one year to the next were removed. In most cases, these adjustments caused industrial consumption to increase and commercial consumption to decrease. The spreadsheet with these adjustments is Sectoral_reclassification5.xls (10/25/2012). In 2009, there was a significant decline in commercial electricity sales in MA and a corresponding increase in industrial sales Assuming that industrial consumption would have fallen by 2% between 2008 and 2009, the adjustment to both the commercial (+) and industrial sectors (-) was estimated to be 7.61 TWh. . The 7.61 TWh converted to Tbtu is 26.0. This value is then added to the negative 164.0 Tbtu in 2009 and subsequent years. State Energy Data System (Jan. 2017) via National Calibration worksheet """ # 1949-1969 published_consumption_trillion_btu = \ self.eia_comm.eia_api(id_='TOTAL.ESCCBUS.A', id_type='series') # Column W (electricity retail sales to the commercial sector) # for years 1949-69 published_consumption_trillion_btu = \ published_consumption_trillion_btu.rename( columns={'Electricity Retail Sales to the Commercial Sector, Annual, Trillion Btu': 'published_consumption_trillion_btu'}) # 1970-2018 national_calibration = self.collect_input_data('national_calibration') published_consumption_trillion_btu.loc['1970':, [ 'published_consumption_trillion_btu' ]] = national_calibration.loc['1970':, [ 'Final Est. (Trillion Btu)_elec' ]].values # Column G (electricity final est) # for years 1970-2018 # 1977-1989 years = list( range( 1977, max(published_consumption_trillion_btu.index.astype(int)) + 1)) years = [str(y) for y in years] # adjustment_to_commercial_trillion_btu_early = number_for_1990 adjustment_to_commercial_trillion_btu = \ [9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340654000005, 29.77918535999970, 10.21012680399960, 1.70263235599987, -40.63866012000020, -40.63865670799990, -117.72073870000000, -117.72073528800000, -117.72073187600000, -117.72072846400000, -162.61452790400100, -136.25241618800100, -108.91594645600000, -125.97594304400000, -125.97593963200100, -163.95020989600000, -163.95020648400000, -163.95020307200000, -137.98708428968000, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100] # First value is for 1977 - 2018 adjustment_df = \ pd.DataFrame([years, adjustment_to_commercial_trillion_btu]).transpose() adjustment_df.columns = \ ['Year', 'adjustment_to_commercial_trillion_btu'] adjusted_supplier_data = \ adjustment_df.merge(published_consumption_trillion_btu, how='outer', on='Year') adjusted_supplier_data['adjustment_to_commercial_trillion_btu'] = \ adjusted_supplier_data['adjustment_to_commercial_trillion_btu'].fillna(0) adjusted_supplier_data = adjusted_supplier_data.set_index('Year') adjusted_supplier_data['adjusted_consumption_trillion_btu'] = \ adjusted_supplier_data['adjustment_to_commercial_trillion_btu'].add(adjusted_supplier_data['published_consumption_trillion_btu']) adjusted_supplier_data['adjusted_consumption_trillion_btu'] = \ adjusted_supplier_data['adjusted_consumption_trillion_btu'].astype(float) adjusted_supplier_data = \ adjusted_supplier_data.sort_index(ascending=True) return adjusted_supplier_data[['adjusted_consumption_trillion_btu']] @staticmethod def get_saus(): """Get Data from the Statistical Abstract of the United States (SAUS) """ print('os.getcwd():', os.getcwd()) try: saus_2002 = \ pd.read_csv('./EnergyIntensityIndicators/Data/SAUS2002_table995.csv').set_index('Year') except FileNotFoundError: os.chdir('..') saus_2002 = \ pd.read_csv('./EnergyIntensityIndicators/Data/SAUS2002_table995.csv').set_index('Year') saus_1994 = { 1980: 738, 1981: 787, 1982: 631, 1983: 716, 1984: 901, 1985: 1039, 1986: 960, 1987: 933, 1988: 883, 1989: 867, 1990: 694, 1991: 477, 1992: 462, 1993: 479 } saus_2001 = { 1980: 738, 1981: None, 1982: None, 1983: None, 1984: None, 1985: 1039, 1986: None, 1987: None, 1988: None, 1989: 867, 1990: 694, 1991: 476, 1992: 462, 1993: 481, 1994: 600, 1995: 700, 1996: 723, 1997: 855, 1998: 1106, 1999: 1117, 2000: 1176 } saus_merged = dict() for (year, value) in saus_2001.items(): if value == None: set_value = saus_1994[year] else: set_value = value saus_merged[year] = set_value saus_merged_df = \ pd.DataFrame.from_dict( saus_merged, orient='index', columns=['Value']) return saus_2002, saus_merged_df @staticmethod def dod_compare_old(): """ DODCompareOld Note from PNNL (David B. Belzer): "These series are of unknown origin--need to check Jackson and Johnson 197 (sic)? """ dod_old = pd.read_csv( './EnergyIntensityIndicators/Data/DODCompareOld.csv').set_index( 'Year') cols_list = ['Retail', 'Auto R', 'Office', 'Warehouse'] dod_old['Commercial'] = dod_old[cols_list].sum(axis=1) dod_old_subset = dod_old.loc[list(range(1960, 1982)), cols_list] dod_old_hotel = dod_old.loc[list(range(1980, 1990)), ['Commercial']] return dod_old, dod_old_subset, dod_old_hotel def dodge_adjustment_ratios(self, dodge_dataframe, start_year, stop_year, adjust_years, late): """(1985, 1990) or (1960, 1970) """ year_indices = self.years_to_str(start_year, stop_year) revision_factor_commercial = dodge_dataframe.loc[year_indices, ['Commercial']].sum( axis=0).values categories = ['Retail', 'Auto R', 'Office', 'Warehouse', 'Hotel'] if late: col = 'Commercial' else: col = 'Commercial, Excl Hotel' for category in categories: revision_factor_cat = dodge_dataframe.loc[ year_indices, [category]].sum(axis=0).values / revision_factor_commercial dodge_dataframe.loc[adjust_years, [category]] = dodge_dataframe.loc[ adjust_years, [col]].values * revision_factor_cat[0] return dodge_dataframe def west_inflation(self): """Jackson and Johnson Estimate of West Census Region Inflation Factor, West Region Shares Based on CBECS Note from PNNL: "Staff: Based upon CBECS, the percentage of construction in west census region was slightly greater in the 1900-1919 period than in 1920-1945. Thus, factor is set to 1.12, approximately same as 1925 value published by Jackson and Johnson" """ # hist_stat = self.hist_stat()['Commercial (million SF)'] # hist_stat column E # # west inflation column Q ornl_78 = { 1925: 1.127, 1930: 1.144, 1935: 1.12, 1940: 1.182, 1945: 1.393, 1950: 1.216, 1951: 1.237, 1952: 1.224, 1953: 1.209, 1954: 1.213, 1955: 1.229 } all_years = list(range(min(ornl_78.keys()), max(ornl_78.keys()) + 1)) increment_years = list(ornl_78.keys()) final_factors = {year: 1.12 for year in list(range(1919, 1925))} for index, y_ in enumerate(increment_years): if index > 0: year_before = increment_years[index - 1] num_years = y_ - year_before infl_factor_year_before = ornl_78[year_before] infl_factor_y_ = ornl_78[y_] increment = 1 / num_years for delta in range(num_years): value = infl_factor_year_before * (1 - increment * delta) + \ infl_factor_y_ * (increment * delta) year = year_before + delta final_factors[year] = value final_factors_df = pd.DataFrame.from_dict(final_factors, columns=['Final Factors'], orient='index') return final_factors_df @staticmethod def years_to_str(start_year, end_year): """Create list of year strings from start_year and end_year range """ list_ = list(range(start_year, end_year + 1)) return [str(l) for l in list_] def hist_stat(self): """Historical Dodge Data through 1970 Data Source: Series N 90-100 Historical Statistics of the U.S., Colonial Times to 1970 """ historical_dodge = pd.read_csv( './EnergyIntensityIndicators/Data/historical_dodge_data.csv' ).set_index('Year') pub_inst_values = historical_dodge.loc[list(range(1919, 1925)), ['Pub&Institutional']].values total_1925_6 = pd.DataFrame.sum( historical_dodge.loc[list(range(1925, 1927)), ], axis=0).drop(index='Commercial (million SF)') inst_pub_total_1925 = pd.DataFrame.sum( historical_dodge.loc[1925, ].drop('Commercial (million SF)'), axis=0) inst_pub_total_1926 = pd.DataFrame.sum( historical_dodge.loc[1926, ].drop('Commercial (million SF)'), axis=0) inst_pub_total_1925_6 = inst_pub_total_1925 + inst_pub_total_1926 shares = total_1925_6.divide(inst_pub_total_1925_6) for col in list(total_1925_6.index): values = historical_dodge.loc[list(range(1919, 1925)), ['Pub&Institutional']].multiply( shares[col]).values historical_dodge.at[list(range(1919, 1925)), col] = values historical_dodge.at[list(range(1919, 1925)), ['Pub&Institutional']] = pub_inst_values return historical_dodge def hist_stat_adj(self): """Adjust historical Dodge data to account for omission of data for the West Census Region prior to 1956 """ hist_data = self.hist_stat() west_inflation = self.west_inflation() hist_data = hist_data.merge(west_inflation, how='outer', left_index=True, right_index=True) hist_data['Final Factors'] = hist_data['Final Factors'].fillna(1) adjusted_for_west = hist_data.drop( columns=['Final Factors', 'Pub&Institutional']).multiply( hist_data['Final Factors'].values, axis=0) return adjusted_for_west.loc[list(range(1919, 1960)), :] def dodge_revised(self): """Dodge Additions, adjusted for omission of West Census Region prior to 1956 """ saus_2002, saus_merged = self.get_saus() dod_old, dod_old_subset, dod_old_hotel = self.dod_compare_old() west_inflation = self.hist_stat_adj() dodge_revised = pd.read_csv( './EnergyIntensityIndicators/Data/Dodge_Data.csv').set_index( 'Year') dodge_revised.index = dodge_revised.index.astype(str) dodge_revised = dodge_revised.reindex( dodge_revised.columns.tolist() + ['Commercial, Excl Hotel', 'Hotel'], axis=1).fillna(np.nan) years_1919_1989 = self.years_to_str(1919, 1990) years_1990_1997 = self.years_to_str(1990, 1997) dodge_revised.loc[self.years_to_str(1960, 1981), ['Retail', 'Auto R', 'Office', 'Warehouse' ]] = dod_old_subset.values dodge_revised.loc[self.years_to_str(1919, 1959), ['Commercial, Excl Hotel']] = west_inflation[ 'Commercial (million SF)'].values.reshape( 41, 1) # hist_stat_adj column Q hist_adj_cols = [ 'Education', 'Hospital', 'Public', 'Religious', 'Soc/Amuse', 'Misc' ] dodge_revised.loc[self.years_to_str(1919, 1959), hist_adj_cols] = west_inflation.drop( 'Commercial (million SF)', axis=1).values dodge_revised.loc[self.years_to_str(1990, 1998), hist_adj_cols] = saus_2002.loc[ self.years_to_str(1990, 1998), [ 'Educational', 'Health', 'Pub. Bldg', 'Religious', 'Soc/Rec', 'Misc.' ]].values dodge_revised.loc[self.years_to_str(1990, 2003), ['Soc/Misc']] = saus_2002.loc[ self.years_to_str(1990, 2001), ['Soc/Rec']].add( saus_2002.loc[self.years_to_str(1990, 2001), ['Misc.']].values) dodge_revised.loc[self.years_to_str(1999, 2001), 'Misc'] = saus_2002.loc[ self.years_to_str(1999, 2001), ['Misc.']].values.reshape(3, ) dodge_revised.loc[self.years_to_str(1961, 1989), 'Misc'] = dodge_revised.loc[ self.years_to_str(1961, 1989), 'Soc/Misc'].subtract(dodge_revised.loc[ self.years_to_str(1961, 1989), 'Soc/Amuse'].values) dodge_revised.loc[str(2000), 'Hospital'] = saus_2002.loc[str(2000), 'Health'] dodge_revised.loc[self.years_to_str(1960, 1989), ['Commercial, Excl Hotel']] = dodge_revised.loc[ self.years_to_str(1960, 1989), ['Retail', 'Auto R', 'Office', 'Warehouse']].sum( axis=1) hotel_80_89 = saus_merged.loc[list(range(1980, 1989 + 1)), ['Value']].subtract(dod_old_hotel.values) dodge_revised.loc[self.years_to_str(1980, 1989), ['Hotel']] = hotel_80_89 hotel_80_89_ratio = hotel_80_89.sum(axis=0).values / dodge_revised.loc[ self.years_to_str(1980, 1989), ['Commercial, Excl Hotel']].sum(axis=0).values dodge_revised.loc[ self.years_to_str(1919, 1979), ['Hotel']] = dodge_revised.loc[ self.years_to_str(1919, 1979), ['Commercial, Excl Hotel']].values * hotel_80_89_ratio dodge_revised.loc[years_1990_1997, ['Commercial, Incl Hotel']] = saus_2002.loc[ years_1990_1997, ['Commercial']].values dodge_revised.loc[self.years_to_str(1985, 1989), ['Commercial']] = saus_merged.loc[ list(range(1985, 1989 + 1)), ['Value']].values dodge_revised.loc[self.years_to_str(1990, 2018), ['Commercial']] = dodge_revised.loc[ self.years_to_str(1990, 2018), ['Commercial, Incl Hotel']].values dodge_revised = self.dodge_adjustment_ratios( dodge_revised, 1960, 1969, adjust_years=self.years_to_str(1919, 1959), late=False) dodge_revised = self.dodge_adjustment_ratios( dodge_revised, 1985, 1989, adjust_years=self.years_to_str(1990, 2018), late=True) dodge_revised.loc[years_1919_1989, ['Commercial, Incl Hotel']] = dodge_revised.loc[ years_1919_1989, ['Commercial, Excl Hotel']].add( dodge_revised.loc[years_1919_1989, ['Hotel']].values) dodge_revised['Total'] = dodge_revised.drop( ['Commercial, Incl Hotel', 'Commercial, Excl Hotel'], axis=1).sum(axis=1).values return dodge_revised def dodge_to_cbecs(self): """Redefine the Dodge building categories more along the lines of CBECS categories. Constant fractions of floor space are moved among categories. Returns: dodge_to_cbecs (dataframe): redefined data """ # Key Assumptions: education_floor_space_office = .10 auto_repair_retail = .80 retail_merc_service = .80 # remainder to food service and sales retail_merc_service_food_sales = .11 retail_merc_service_food_service = .90 education_assembly = .05 education_misc = .05 # (laboratories) health_transfered_to_cbecs_health = .75 # 25% to lodging (nursing homes) misc_public_assembly = .10 # (passenger terminals) dodge_revised = self.dodge_revised() # dataframe dodge_to_cbecs = pd.DataFrame(dodge_revised[[ 'Total', 'Religious', 'Warehouse' ]]).rename(columns={'Total': 'Dodge_Totals'}) dodge_to_cbecs['Office'] = dodge_revised[ 'Office'] + education_floor_space_office * dodge_revised[ 'Education'] dodge_to_cbecs['Merc/Serv'] = retail_merc_service * ( dodge_revised['Retail'] + auto_repair_retail * dodge_revised['Auto R']) dodge_to_cbecs['Food_Sales'] = retail_merc_service_food_sales * ( dodge_revised['Retail'] + auto_repair_retail * dodge_revised['Auto R']) dodge_to_cbecs['Food_Serv'] = retail_merc_service_food_service * ( dodge_revised['Retail'] + auto_repair_retail * dodge_revised['Auto R']) dodge_to_cbecs['Education'] = ( 1 - education_floor_space_office - education_assembly - education_misc) * dodge_revised['Education'] dodge_to_cbecs[ 'Health'] = health_transfered_to_cbecs_health * dodge_revised[ 'Hospital'] dodge_to_cbecs['Lodging'] = dodge_revised['Hotel'] + ( 1 - health_transfered_to_cbecs_health) * dodge_revised['Hospital'] dodge_to_cbecs['Assembly'] = dodge_revised[ 'Soc/Amuse'] + misc_public_assembly * dodge_revised[ 'Misc'] + education_assembly * dodge_revised['Education'] dodge_to_cbecs['Other'] = dodge_revised['Public'] + ( 1 - misc_public_assembly) * dodge_revised['Misc'] + ( 1 - auto_repair_retail) * dodge_revised[ 'Auto R'] + education_misc * dodge_revised['Education'] dodge_to_cbecs['Redefined_Totals'] = dodge_to_cbecs.drop( 'Dodge_Totals', axis=1).sum(axis=1).values # dodge_to_cbecs = dodge_to_cbecs.drop() # don't need totals? return dodge_to_cbecs def nems_logistic(self, dataframe, params): """ PNNL errors found: - Column S in spreadsheet has CO-StatePop2.xls are incorrectly aligned with years - Column AL does not actually scale by 1.28 as suggested in the column header """ current_year = dt.datetime.now().year link_factors = pd.read_excel( f'{self.directory}/CO-EST_statepop2.xls', sheet_name='Stock', usecols='D:E', header=1, skiprows=158).rename(columns={1789: 'Year'}) state_pop = link_factors.set_index('Year').rename( columns={' New': 'state_pop'}) state_pop = state_pop[state_pop.index.notnull()] state_pop.index = state_pop.index.astype(int) state_pop.index = state_pop.index.astype(str) dataframe = dataframe.merge(state_pop, how='outer', left_index=True, right_index=True) dataframe = dataframe[dataframe.index.notnull()] dataframe = dataframe.reindex(columns=dataframe.columns.tolist() + [ 'adjusted_state_pop', 'adjusted_state_pop_scaled_b', 'adjusted_state_pop_scaled_c', 'scaled_additions_estimate_a', 'scaled_additions_estimate_b', 'scaled_additions_estimate_c', 'removal', 'adjusted_removals', 'old_stk_retain', 'floorspace_bsf' ]) dataframe['Year_Int'] = dataframe.index.astype(int) dataframe['age'] = dataframe['Year_Int'].subtract( current_year).multiply(-1) dataframe['remaining'] = ((dataframe['age'].divide(params[1])).pow( params[0]).add(1)).pow(-1) dataframe['inflate_fac'] = dataframe['remaining'].pow(-1) link_factor = 0.1 adjusted_state_pop_1 = 40 timing_wgts_current_yr = 0.4 timing_wgts_lag_yr = 0.6 benchmark_factor = 1 dataframe.loc[str(1838), ['state_pop']] = 400 dataframe.loc[self.years_to_str(1838, 1919), ['adjusted_state_pop']] = dataframe.loc[ self.years_to_str(1838, 1919), ['state_pop']].values * link_factor dataframe.loc[self.years_to_str(1920, current_year), ['adjusted_state_pop']] = dataframe.loc[ self.years_to_str(1920, current_year), ['Redefined_Totals']].values for year in self.years_to_str(1838, current_year): adjusted_state_pop_value = dataframe.loc[ year, ['adjusted_state_pop']].values if year == '1838': vpip_estimate = adjusted_state_pop_value elif year == '1920': vpip_estimate = adjusted_state_pop_value else: adjusted_state_pop_year_before = dataframe.loc[ str(int(year) - 1), ['adjusted_state_pop']].values vpip_estimate = ( timing_wgts_current_yr * adjusted_state_pop_value + timing_wgts_lag_yr * adjusted_state_pop_year_before) * benchmark_factor dataframe.loc[year, 'VPIP-Estimate'] = vpip_estimate _variable = 1.2569 # This should be solved for x_column_value = _variable db_estimates = 1.2 db_estimates2 = [1.25 - 0.01 * d for d in list(range(1990, 2021))] post_1989_scaling_factor = db_estimates # Should choose this variable_2 = 1.533 # This should be solved for without_lags = dataframe.loc[self.years_to_str(1990, current_year), ['adjusted_state_pop']].multiply( post_1989_scaling_factor) dataframe.loc[:str(1989), ['scaled_additions_estimate_a']] = dataframe.loc[:str( 1989), ['VPIP-Estimate']].values * variable_2 dataframe.loc[self.years_to_str(1990, current_year), ['scaled_additions_estimate_a']] = dataframe.loc[ self.years_to_str(1990, current_year), ['VPIP-Estimate']].values * post_1989_scaling_factor dataframe.loc[:str(1989), ['adjusted_state_pop_scaled_b']] = dataframe.loc[ self.years_to_str(1790, 1989), ['scaled_additions_estimate_a']].values dataframe.loc[self.years_to_str(1990, 2001), ['adjusted_state_pop_scaled_b']] = dataframe.loc[ self.years_to_str(1990, 2001), ['scaled_additions_estimate_a']].values * 1.15 dataframe.loc[self.years_to_str(2002, current_year), ['adjusted_state_pop_scaled_b']] = dataframe.loc[ self.years_to_str(2002, current_year), ['scaled_additions_estimate_a']].values dataframe.loc[str(1790), ['adjusted_state_pop_scaled_c']] = 1 dataframe.loc[self.years_to_str(1791, 1989), ['adjusted_state_pop_scaled_c']] = dataframe.loc[ self.years_to_str(1791, 1989), ['scaled_additions_estimate_a']].values dataframe.loc[self.years_to_str(1990, 2001), ['adjusted_state_pop_scaled_c']] = dataframe.loc[ self.years_to_str(1990, 2001), ['scaled_additions_estimate_a']].values * 1.28 dataframe.loc[self.years_to_str(2002, current_year), ['adjusted_state_pop_scaled_c']] = dataframe.loc[ self.years_to_str(2002, current_year), ['scaled_additions_estimate_a']].values for y in self.years_to_str(1839, current_year): years_diff = current_year - 1870 start_year = int(y) - years_diff # first_index_year = int(dataframe.index[0]) # if start_year < first_index_year: # start_year = first_index_year year_index = self.years_to_str(start_year, int(y)) remaining = dataframe.loc[self.years_to_str(1870, current_year), ['remaining']].values.flatten() adjusted_state_pop_scaled_b = dataframe.loc[ year_index, ['adjusted_state_pop_scaled_b']].fillna(0).values.flatten() adjusted_state_pop_scaled_c = dataframe.loc[ year_index, ['adjusted_state_pop_scaled_c']].fillna(0).values.flatten() b_value = np.dot(adjusted_state_pop_scaled_b, remaining) c_value = np.dot(adjusted_state_pop_scaled_c, remaining) dataframe.loc[y, ['scaled_additions_estimate_b']] = b_value dataframe.loc[y, ['scaled_additions_estimate_c']] = c_value removal_chg = 1 # Not sure what this is about fractions = [0.3, 0.4, 0.4, 0.35, 0.35, 0.35, 0.35, 0.3, 0.3, 0.3] fraction_retained = [f * removal_chg for f in fractions] for i in dataframe.index: if i >= '1870': removal = dataframe.loc[ i, ['scaled_additions_estimate_c']].values - dataframe.loc[ str(int(i) - 1), ['scaled_additions_estimate_c']].values - dataframe.loc[ i, ['scaled_additions_estimate_a']].values dataframe.loc[i, ['removal']] = removal dataframe.loc[self.years_to_str(2009, 2009 + len(fractions) - 1), ['adjusted_removals']] = dataframe.loc[ self.years_to_str(2009, 2009 + len(fractions) - 1), ['removal']].values.flatten() * fraction_retained for y_ in list(range(2009, 2009 + len(fractions))): if y_ == 2009: dataframe.loc[str(y_), ['old_stk_retain']] = dataframe.loc[ str(y_), ['adjusted_removals']] else: dataframe.loc[str(y_), ['old_stk_retain']] = dataframe.loc[ str(y_ - 1), ['old_stk_retain']].values + dataframe.loc[ str(y_), ['adjusted_removals']].values dataframe['adjusted_removals'] = dataframe['adjusted_removals'].fillna( 0) dataframe.loc[ self.years_to_str(1960, current_year), ['floorspace_bsf']] = dataframe.loc[ self.years_to_str(1960, current_year), ['scaled_additions_estimate_c']].values - dataframe.loc[ self.years_to_str(1960, current_year), ['adjusted_removals']].values return dataframe[['floorspace_bsf']].dropna() def solve_logistic(self, dataframe): """Solve NES logistic parameters """ pnnl_coefficients = [3.92276415015621, 73.2238120168849] # [gamma, lifetime] # popt, pcov = curve_fit(self.nems_logistic, xdata=dataframe[], ydata=dataframe[] , p0=pnnl_coefficients) # return popt return pnnl_coefficients def activity(self): """Use logistic parameters to find predicted historical floorspace Returns: historical_floorspace_billion_sq_feet (pd.DataFrame): historical floorspace in the Commercial Sector. Years: Units: Billion Square Feet Data Source: """ dodge_to_cbecs = self.dodge_to_cbecs( ) # columns c-m starting with year 1920 (row 17) coeffs = self.solve_logistic(dodge_to_cbecs) historical_floorspace_late = self.nems_logistic( dodge_to_cbecs, coeffs) # properly formatted? historical_floorspace_early = { 1949: 27235.1487296062, 1950: 27788.6370796569, 1951: 28246.642791733, 1952: 28701.4989706012, 1953: 29253.2282427217, 1954: 29913.8330998026, 1955: 30679.7157232176, 1956: 31512.6191323126, 1957: 32345.382764321, 1958: 33206.8483392728, 1959: 34088.6640247816 } historical_floorspace_early = pd.DataFrame.from_dict( historical_floorspace_early, columns=['floorspace_bsf'], orient='index') historical_floorspace_early.index = historical_floorspace_early.index.astype( str) historical_floorspace = pd.concat( [historical_floorspace_early, historical_floorspace_late]) historical_floorspace_billion_sq_feet = historical_floorspace.multiply( 0.001) return historical_floorspace_billion_sq_feet def fuel_electricity_consumption(self): """ Trillion Btu Returns: energy_data (dict): Dictionary of dataframes with keys 'elec' and 'fuels' """ year_range = list(range(1949, 1970)) year_range = [str(y) for y in year_range] national_calibration = self.collect_input_data('national_calibration') total_primary_energy_consumption = \ self.eia_comm.eia_api(id_='TOTAL.TXCCBUS.A', id_type='series') # pre 1969: AER table 2.1c update column U total_primary_energy_consumption = \ total_primary_energy_consumption.rename( columns={'Total Primary Energy Consumed by the Commercial Sector, Annual, Trillion Btu': 'total_primary'}) # total_primary_energy_consumption = total_primary_energy_consumption[total_primary_energy_consumption.index.isin(year_range)] # total_primary_energy_consumption = total_primary_energy_consumption.multiply(0.001) fuels_dataframe = total_primary_energy_consumption.copy() replacement_data = national_calibration.loc['1970':, [ 'Final Est. (Trillion Btu)_fuels' ]] # >= 1970: National Calibration Column 0 fuels_dataframe.loc['1970':, ['total_primary']] = replacement_data.values fuels_dataframe = fuels_dataframe.rename( columns={'total_primary': 'adjusted_consumption_trillion_btu'}) fuels_dataframe['adjusted_consumption_trillion_btu'] = fuels_dataframe[ 'adjusted_consumption_trillion_btu'].astype(float) elec_dataframe = self.adjusted_supplier_data() energy_data = {'elec': elec_dataframe, 'fuels': fuels_dataframe} return energy_data def get_seds(self): """Collect SEDS data Returns: data (dict): Dictionary of dataframes with keys 'elec' and 'fuels' """ seds = self.collect_input_data('SEDS_CensusRgn') census_regions = {4: 'West', 3: 'South', 2: 'Midwest', 1: 'Northeast'} total_fuels = seds[0].rename(columns=census_regions) elec = seds[1].rename(columns=census_regions) data = {'elec': elec, 'fuels': total_fuels} return data def collect_weather(self, comm_activity): """Gather weather data for the Commercial sector Args: comm_activity ([type]): [description] Returns: weather_data (dict): [description] """ seds = self.get_seds() res = ResidentialIndicators(directory=self.directory, output_directory=self.output_directory, base_year=self.base_year) # residential_activity_data = res.get_floorspace() # residential_floorspace = residential_activity_data['floorspace_square_feet'] weather = WeatherFactors(sector='commercial', directory=self.directory, activity_data=comm_activity) # residential_floorspace=residential_floorspace) weather_factors = weather.get_weather(seds_data=seds) # weather_factors = weather.adjust_for_weather() # What should this return?? (e.g. weather factors or weather adjusted data, both?) weather_data = dict() for key, value in weather_factors.items(): value = value.drop('electricity_weather_factor', axis=1, errors='ignore') weather_data[key] = value return weather_data def collect_data(self): """Gather decomposition input data for the Commercial sector Returns: data_dict (dict): Commercial Sector data input to the LMDI model """ # Activity: Floorspace_Estimates column U, B # Energy: Elec --> Adjusted Supplier Data Column D # Fuels --> AER11 Table 2.1C_Update column U, National Calibration Column O activity_data = self.activity() print('Activity data collected without issue') energy_data = self.fuel_electricity_consumption() print('Energy data collected without issue') weather_factors = \ self.collect_weather(comm_activity=activity_data) data_dict = { 'Commercial_Total': { 'energy': energy_data, 'activity': activity_data, 'weather_factors': weather_factors } } return data_dict def main(self, breakout, calculate_lmdi): """Decompose energy use for the Commercial sector Args: breakout ([type]): [description] calculate_lmdi ([type]): [description] Returns: [type]: [description] """ data_dict = self.collect_data() results_dict, formatted_results = \ self.get_nested_lmdi(level_of_aggregation=self.level_of_aggregation, breakout=breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, lmdi_type='LMDI-I') return results_dict
def industrial_projections(self): """Gather Industrial projections data activity: - Value added --> Gross Domestic Product (Total Industrial only) - Gross Output - Value Added energy: Energy Consumption Trillion Btu """ # Industrial : Value of Shipments : Agriculture, Mining, and Construction, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_NMFG_VOS_NA_USA_BLNY09DLR.A' # Industrial : Value of Shipments : Manufacturing, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_MANF_VOS_NA_USA_BLNY09DLR.A' # Industrial : Value of Shipments : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_NA_VOS_NA_USA_BLNY09DLR.A' industrial_eia = GetEIAData('industrial') data_dict = {'Manufacturing': {'Food, Beverages, & Tobacco': {'Food Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_FDP_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Food Products'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_FDP_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Food Products')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_FDP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Food Products')}}, # 'Beverages and Tobacco Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_BVTP_NA_NA_NA_BLNY09DLR.A'}} }, # 'Textile Mills and Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_TMP_NA_NA_NA_BLNY09DLR.A'}}, 'Wood Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_WDP_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Wood Products'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_WDP_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Wood Products')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_SHP_IDAL_WDP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Wood Products')}}, 'Paper': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PPM_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Paper'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PPM_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Paper')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_PPM_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Paper')}} , # 'Printing': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_PRN_NA_NA_NA_BLNY09DLR.A'}}, # 'Petroleum & Coal Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='', id_type='series'), # 'purchased_electricity': industrial_eia.eia_api(id_='', id_type='series')}, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PCL_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, 'Chemicals': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BCH_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Chemicals'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BCH_HAP_PRC_NA_TRLBTU.A', id_type='series', new_name='Chemicals')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_CHE_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Chemicals')}}, 'Plastics & Rubber Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PLI_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Plastics & Rubber Products'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PLI_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Plastics & Rubber Products')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PRP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Plastics & Rubber Products')}}, # 'Nonmetallic Mineral Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_SCG_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, , # 'Primary Metals': {'energy': {'total_energy': {'iron_and_steel': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_ISM_TOT_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals'), # 'aluminum': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AAP_TOT_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals')}, # 'purchased_electricity': {'iron_and_steel': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_ISM_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals'), # 'aluminum': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AAP_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals')}}, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PMM_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Primary Metals')}}, # 'Fabricated Metal Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_FABM_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, , # 'Machinery': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_MCHI_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, , # 'Computer & Electronic Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_CMPEL_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, # 'Electrical Equipment': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_EEI_NA_NA_NA_BLNY09DLR.A', id_type='series')}}}, # 'Transportation Equipment': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_TEQ_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, # 'Furniture & Related Products': {'energy': {'total_energy': , 'purchased_electricity': }, # 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_FRP_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, 'Miscellaneous': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BMF_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Miscellaneous'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BMF_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Miscellaneous')}, # Energy data from "Balance of Manufacturing" 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_MISC_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Miscellaneous')}}}, 'Nonmanufacturing': {'Agriculture/Forestry/Fishing/Hunting': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AGG_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AGG_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_AFF_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting')}}, # here elec is purchased electricity, Note: try to find total elec 'Mining': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Mining'), 'purchased_electricity': {'excluding_oil_shale': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_PEO_NA_NA_TRLBTU.A', id_type='series', new_name='Mining'), 'including_oil_shale': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_PES_NA_NA_TRLBTU.A', id_type='series', new_name='Mining')}}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_MING_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Mining')}}, 'Construction': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_CNS_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Construction'), 'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_CNS_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Construction')}, 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_CNS_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Construction')}}}} energy_use_industrial_electricity_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Industry') energy_use_industrial_total_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_TOT_NA_NA_QBTU.A', id_type='series', new_name='Industry') energy_use_industrial_delivered_energy_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Industry') return data_dict
class WeatherFactors: def __init__(self, sector, directory, activity_data=None, residential_floorspace=None, nominal_energy_intensity=None, \ end_year=2018): self.end_year = end_year self.directory = directory self.sector = sector self.activity_data = activity_data self.nominal_energy_intensity = nominal_energy_intensity self.residential_floorspace = residential_floorspace self.eia_data = GetEIAData(self.sector) self.lmdi_prices = pd.read_excel(f'{self.directory}/EnergyPrices_by_Sector_010820_DBB.xlsx', sheet_name='LMDI-Prices', header=14, usecols='A:B, EY') self.regions_subregions = ['northeast', 'new_england', 'middle_atlantic', 'midwest', 'east_north_central', 'west_north_central', 'south', 'south_atlantic', 'east_south_central', 'west_south_central', 'west', 'mountain', 'pacific'] self.sub_regions_dict = {'northeast': ['New England', 'Middle Atlantic'], 'midwest': ['East North Central', 'West North Central'], 'south': ['South Atlantic', 'East South Central', 'West South Central'], 'west': ['Mountain', 'Pacific']} @staticmethod def adjust_data(subregions, hdd_by_division, hdd_activity_weights, cooling=True, cdd_by_division=None, \ cdd_activity_weights=None, use_weights_1961_90=True): """Calculate weights for adjusted weather factors prediction """ years_1961_90 = list(range(1961, 1990 + 1)) years_1981_2010 = list(range(1981, 1990 + 1)) if cooling: cdd_by_division = cdd_by_division.set_index('Year') cdd_by_division.index = cdd_by_division.index.astype(int) averages_1961_90_cooling = cdd_by_division.loc[years_1961_90, :].mean(axis=0) averages_1981_2010_cooling = cdd_by_division.loc[years_1981_2010, :].mean(axis=0) hdd_by_division = hdd_by_division.set_index('Year') hdd_by_division.index = hdd_by_division.index.astype(int) averages_1961_90_heating = hdd_by_division.loc[years_1961_90, :].mean(axis=0) averages_1981_2010_heating = hdd_by_division.loc[years_1981_2010, :].mean(axis=0) all_s_weights_heating = [] all_s_weights_cooling = [] for s in subregions: if use_weights_1961_90: subregion_weights_heating = averages_1961_90_heating.loc[s] * hdd_activity_weights[s] if cooling: subregion_weights_cooling = averages_1961_90_cooling.loc[s] * cdd_activity_weights[s] all_s_weights_cooling.append(subregion_weights_cooling) else: subregion_weights_heating = averages_1981_2010_heating.loc[s] * hdd_activity_weights[s] if cooling: subregion_weights_cooling = averages_1981_2010_cooling.loc[s] * cdd_activity_weights[s] all_s_weights_cooling.append(subregion_weights_cooling) all_s_weights_heating.append(subregion_weights_heating) weights_dict = dict() if cooling: weights_cooling = sum(all_s_weights_cooling) weights_dict['cooling'] = weights_cooling weights_heating = sum(all_s_weights_heating) weights_dict['heating'] = weights_heating return weights_dict def process_prices(self, weather_factors_df): """TODO: Are distributed lag and time cubed ever the desired variable? Does this method need to exist? """ lmdi_prices = self.lmdi_prices # distributed_lag = # time_cubed = selected_variable = [1] * len(weather_factors_df) return selected_variable @staticmethod def cbecs_1995_shares(): """Calculate fuels and elec shares for the commercial sector from CBECS 1995 data """ electricty_consumption_tbtu = {'Northeast': 436, 'Midwest': 558, 'South': 1027, 'West': 587} electricty_consumption_tbtu['Total'] = sum(electricty_consumption_tbtu.values()) electricity_df = pd.DataFrame.from_dict(electricty_consumption_tbtu, orient='index', \ columns=['electricity_consumption_tbtu']) energy_tbtu = {'Northeast': 1035, 'Midwest': 1497, 'South': 1684, 'West': 1106} energy_tbtu['Total'] = sum(energy_tbtu.values()) energy_df = pd.DataFrame.from_dict(energy_tbtu, orient='index', columns=['energy']) shares_df = energy_df.merge(electricity_df, left_index=True, right_index=True, how='outer') shares_df['elec_share'] = shares_df.electricity_consumption_tbtu.divide(shares_df.loc['Total', \ 'electricity_consumption_tbtu']) shares_df['fuel_consumption'] = shares_df.energy.subtract(shares_df.electricity_consumption_tbtu) shares_df['fuels_share'] = shares_df.fuel_consumption.divide(shares_df.loc['Total', 'fuel_consumption']) return shares_df @staticmethod def recs_1993_shares(): """Calculate fuels and elec shares for the residential sector from RECS 1993 data """ electricty_consumption_tbtu = {'Northeast': 470, 'Midwest': 740, 'South': 1510, 'West': 560} electricty_consumption_tbtu['Total'] = sum(electricty_consumption_tbtu.values()) electricity_df = pd.DataFrame.from_dict(electricty_consumption_tbtu, orient='index', \ columns=['electricity_consumption_tbtu']) energy_tbtu = {'Northeast': 2380, 'Midwest': 3130, 'South': 2950, 'West': 1550} energy_tbtu['Total'] = sum(energy_tbtu.values()) energy_df = pd.DataFrame.from_dict(energy_tbtu, orient='index', columns=['energy']) shares_df = energy_df.merge(electricity_df, left_index=True, right_index=True, how='outer') shares_df['elec_share'] = shares_df.electricity_consumption_tbtu.divide(shares_df.loc['Total', \ 'electricity_consumption_tbtu']) shares_df['fuel_consumption'] = shares_df.energy.subtract(shares_df.electricity_consumption_tbtu) shares_df['fuels_share'] = shares_df.fuel_consumption.divide(shares_df.loc['Total', 'fuel_consumption']) return shares_df def regional_shares(self, dataframe, cols): """Calulate shares of regional totals by subregion """ dataframe = dataframe.set_index('regions_subregions') weights_data = dict() for col in cols: shares_dict = dict() for r_, subregions in self.sub_regions_dict.items(): subregions = [s.lower().replace(' ', '_') for s in subregions] regions_ = subregions + [r_] region_total = dataframe.loc[r_, col] for r in regions_: share_value = dataframe.loc[r, col] / region_total shares_dict[r] = share_value weights_data[col] = shares_dict return weights_data def gather_weights_data(self): """Calculate weights to aggregate subregions into four regions """ if self.sector == 'residential': electricity_data = {'total_elec_tbtu': {'northeast': 470, 'midwest': 740, 'south': 1510, 'west': 560}, 'heating_tbtu': {'northeast': 12 * 3.412, 'midwest': 22 * 3.412, 'south': 61 * 3.412, 'west': 25 * 3.412}, 'cooling_tbtu': {'northeast': 40, 'midwest': 80, 'south': 310, 'west': 30}} fuels_data = {'all_energy_tbtu': {'northeast': 2380, 'midwest': 3130, 'south': 2950, 'west': 1550}, 'electricity_tbtu': {'northeast': 470, 'midwest': 740, 'south': 1510, 'west': 560}, 'heating_all_energy_tbtu': {'northeast': 1490, 'midwest': 1920, 'south': 1210, 'west': 700}} # Residential Heating Households Millions heating_activity = [4.1, 1, 3.1, 5.8, 3.5, 2.4, 18.8, 10.7, 3.4, 4.8, 8.3, 2, 6.3] # Residential Cooling Households Millions cooling_activity = [10.9, 2.1, 8.8, 16.4, 10.8, 5.6, 29.4, 15, 5.3, 9.2, 7.1, 2.1, 5.1] all_energy = [19.1, 4.9, 14.2, 23.2, 16.3, 6.9, 32.8, 16.8, 5.9, 10.1, 19.4, 5.3, 14.1] electricity = [1.9, 0.5, 1.4, 2.9, 1.6, 1.3, 14.6, 8.7, 2.5, 3.4, 5.6, 1.4, 4.2] elif self.sector == 'commercial': electricity_data = {'total_elec_tbtu': {'northeast': 436, 'midwest': 558, 'south': 1027, 'west': 587}, 'heating_tbtu': {'northeast': 18, 'midwest': 23, 'south': 43, 'west': 28}, 'cooling_tbtu': {'northeast': 44, 'midwest': 60, 'south': 172, 'west': 64}} fuels_data = {'all_energy_tbtu': {'northeast': 1035, 'midwest': 1497, 'south': 1684, 'west': 1106}, 'electricity_tbtu': {'northeast': 436, 'midwest': 558, 'south': 1027, 'west': 587}, 'heating_all_energy_tbtu': {'northeast': 385, 'midwest': 668, 'south': 376, 'west': 275}} # Commercial Heating Floorspace Million SF heating_activity = [657, 137, 520, 779, 345, 434, 3189, 1648, 1140, 401, 1219, 469, 750] # Commercial Cooling Floorspace Million SF cooling_activity = [5919, 1472, 4447, 10860, 7301, 3559, 13666, 6512, 3265, 3889, 7058, 2812, 4246] all_energy = [7661, 2031, 5630, 10860, 7301, 3559, 13666, 6512, 3265, 3889, 7065, 2819, 4246] electricity = [657, 137, 520, 779, 345, 434, 3189, 1648, 1140, 401, 1219, 469, 750] else: return None weights_data_ = {'regions_subregions': self.regions_subregions, 'heating_activity': heating_activity, 'cooling_activity': cooling_activity, 'all_energy': all_energy, 'electricity': electricity} weights_df = pd.DataFrame(data=weights_data_) weights_df['fuels'] = weights_df['all_energy'].subtract(weights_df['electricity']) return weights_df def heating_cooling_data(self): hdd_by_division_historical = pd.read_csv('./Data/historical_hdd_census_division.csv').set_index('Year') cdd_by_division_historical = pd.read_csv('./Data/historical_cdd_census_division.csv').set_index('Year') hdd_by_division = self.eia_data.eia_api(id_='1566347', id_type='category') hdd_to_drop = [c for c in list(hdd_by_division.columns) if 'Monthly' in c] hdd_by_division = hdd_by_division.drop(hdd_to_drop, axis=1) hdd_rename_dict = {c: c.replace(', Annual, Number', '') for c in list(hdd_by_division.columns)} hdd_by_division = hdd_by_division.rename(columns=hdd_rename_dict) hdd_by_division = pd.concat([hdd_by_division_historical, hdd_by_division], sort=True) cdd_by_division = self.eia_data.eia_api(id_='1566348', id_type='category') cdd_to_drop = [c for c in list(cdd_by_division.columns) if 'Monthly' in c] cdd_by_division = cdd_by_division.drop(cdd_to_drop, axis=1) cdd_rename_dict = {c: c.replace(', Annual, Number', '') for c in list(cdd_by_division.columns)} cdd_by_division = cdd_by_division.rename(columns=cdd_rename_dict) cdd_by_division = pd.concat([cdd_by_division_historical, cdd_by_division], sort=True) title_case_regions = [s.replace('_', ' ').title() for s in self.regions_subregions] hdd_names = [f'Heating Degree-Days, {r}' for r in title_case_regions] cdd_names = [f'Cooling Degree-Days, {r}' for r in title_case_regions] hdd_new_names_dict = {name: name_title for name, name_title in zip(hdd_names, title_case_regions)} cdd_new_names_dict = {name: name_title for name, name_title in zip(cdd_names, title_case_regions)} hdd_by_division = hdd_by_division.rename(columns=hdd_new_names_dict) cdd_by_division = cdd_by_division.rename(columns=cdd_new_names_dict) return hdd_by_division, cdd_by_division def estimate_regional_shares(self): """Spreadsheet equivalent: Commercial --> 'Regional Shares' assumed commercial floorspace in each region follows same trends as population or housing units""" regions = ['Northeast', 'Midwest', 'South', 'West'] cbecs_data = pd.read_csv('./Data/cbecs_data_millionsf.csv').set_index('Year') cbecs_data.index = cbecs_data.index.astype(str) cbecs_years = list(cbecs_data.index) cbecs_data = cbecs_data.rename(columns={'Midwest ': 'Midwest', ' South': 'South', ' West': 'West'}) cbecs_data.loc['1979', regions] = cbecs_data.loc['1983', regions].subtract([826, 972, 2665, 1212]) cbecs_data.loc['1979', ['U.S.']] = sum(cbecs_data.loc['1979', regions].values) cbecs_data['U.S. (calc)'] = cbecs_data.sum(axis=1) comm_regional_shares = cbecs_data.drop(['U.S.', 'U.S. (calc)'], axis=1).divide(cbecs_data['U.S. (calc)'].values.reshape(len(cbecs_data), 1)) comm_regional_shares_ln = np.log(comm_regional_shares) residential_data = ResidentialFloorspace(end_year=self.end_year) # change to pull from residential().activity() final_results_total_floorspace_regions, regional_estimates_all, avg_size_all_regions = residential_data.final_floorspace_estimates() regional_dfs = [regional_estimates_all[r][['Total']].rename(columns={'Total': r}) for r in regions] residential_housing_units = reduce(lambda x, y: pd.merge(x, y, left_index=True, right_index=True, how='outer'), regional_dfs) residential_housing_units['U.S.'] = residential_housing_units.sum(axis=1) residential_housing_units.index = residential_housing_units.index.astype(str) regional_shares_residential_housing_units = residential_housing_units.drop('U.S.', axis=1).divide(residential_housing_units['U.S.'].values.reshape(len(residential_housing_units), 1)) regional_shares_residential_housing_units_ln = np.log(regional_shares_residential_housing_units) regional_shares_residential_housing_units_cbecs_years = regional_shares_residential_housing_units.loc[cbecs_years, :] regional_shares_residential_housing_units_cbecs_years_ln = np.log(regional_shares_residential_housing_units_cbecs_years) predictions_df = pd.DataFrame(columns=comm_regional_shares.columns, index=residential_housing_units.index) for region in comm_regional_shares.columns: x_values = comm_regional_shares_ln[region].values X = x_values.transpose() y = regional_shares_residential_housing_units_cbecs_years_ln[region].values p = np.polyfit(X, y, 1) predictions_df[region] = np.exp(regional_shares_residential_housing_units_ln[region].multiply(p[0]).add(p[1])) predictions_df['Predicted Sum'] = predictions_df.sum(axis=1) normalized_shares = predictions_df.drop('Predicted Sum', axis=1).divide(predictions_df['Predicted Sum'].values.reshape(len(predictions_df), 1)) return normalized_shares def commercial_estimate_regional_floorspace(self): regional_shares = self.estimate_regional_shares() commercial_floorspace = self.activity_data regional_shares_index = regional_shares.index.astype(str) commercial_floorspace_reshape = commercial_floorspace.loc[regional_shares_index, :] regional_floorspace = regional_shares.multiply(commercial_floorspace_reshape.values) return regional_floorspace def commercial_regional_intensity_aggregate(self): """Calculate Energy Intensities (kBtu/sq. ft.) by region and fuel type (i.e. Fuels and Electricity) for use in calculating weather factors Returns: dictionary with keys: 'electricity' and 'fuels', values: dataframes of intensity data for the commercial sector with Year index and Region columns """ regional_floorspace = self.commercial_estimate_regional_floorspace() total_fuels_to_indicators, elec_to_indicators = self.eia_data.get_seds() regional_floorspace_index = regional_floorspace.index elec_to_indicators = elec_to_indicators.loc[regional_floorspace_index, :] total_fuels_to_indicators = total_fuels_to_indicators.loc[regional_floorspace_index, :] fuels_regional = regional_floorspace.multiply(total_fuels_to_indicators.drop('National', axis=1).values) elec_regional = regional_floorspace.multiply(elec_to_indicators.drop('National', axis=1).values) return {'fuels': fuels_regional, 'electricity': elec_regional} def residential_regional_intensity_aggregate(self): """This function does not need to exist if nominal_energy_intensity is properly formated, change formatting here if not Returns: dictionary with keys: 'electricity' and 'fuels', values: dataframes of intensity data for the residential sector with Year index and Region columns i.e. {'fuels': fuels_regional, 'electricity': elec_regional} """ nominal_energy_intensity = self.nominal_energy_intensity # nominal_energy_intensity should already be formated in this way return nominal_energy_intensity def weather_factors(self, region, energy_type, actual_intensity, weights_df, regional_weights): """Estimate a simple regression model to fit the regional intensity to a linear function of time (included squared and cubed values of time) and degree days. -electricity model: constant term, heating degree day (HDD), cooling degree day (CDD), time, time-squared, and time-cubed -fuels model: contant term?, HDD, HDD*Time, Time, Time-squared and composite fuel price index (the composite fuel price index was developed as a weighted average of the national distillate fuel oil price index and a national average price for natural gas) Weather factors are applied at the regional level to generate the weather-normalized intensity indexes for each of the four Census regions -The weather factors for delivered energy and source energy are computed implicitly. For delivered energy, they are calculated as the sum of reported electricity and fuels divided by the sum of the weather-adjusted electricity and weather-adjusted fuels. A similar procedure is followed for source energt. As such, the implied weather factors are a result of the process, not an independent variable that influences the values of intensity indexes for delivered energy and source energy. All of these computation occur within Commercial_Total worksheet. TODO: Input data """ if energy_type == 'electricity': energy_type = 'elec' subregions = self.sub_regions_dict[region] subregions_lower = [s.lower().replace(' ', '_') for s in subregions] hdd_activity_weights = [regional_weights['heating_activity'][r_] for r_ in subregions_lower] hdd_activity_weights_dict = {r : regional_weights['heating_activity'][r_] for r, r_ in zip(subregions, subregions_lower)} cdd_activity_weights = [regional_weights['cooling_activity'][r_] for r_ in subregions_lower] cdd_activity_weights_dict = {r : regional_weights['cooling_activity'][r_] for r, r_ in zip(subregions, subregions_lower)} fuels_weights = [regional_weights['fuels'][r_] for r_ in subregions_lower] hdd_by_division, cdd_by_division = self.heating_cooling_data() heating_degree_days = hdd_by_division[subregions] heating_degree_days = heating_degree_days.reset_index('Year') heating_degree_days[region] = heating_degree_days[subregions].dot(hdd_activity_weights) fuels_heating_degree_days = heating_degree_days fuels_heating_degree_days[region] = fuels_heating_degree_days[subregions].dot(fuels_weights) weather_factors_df = heating_degree_days[['Year', region]].rename(columns={region: 'HDD'}) weather_factors_df['Year'] = weather_factors_df['Year'].astype(int) weather_factors_df['Time'] = weather_factors_df['Year'].values - 1969 weather_factors_df['Time^2'] = weather_factors_df[['Time']].pow(2).values if energy_type == 'elec': cooling_degree_days = cdd_by_division[subregions] cooling_degree_days[region] = cooling_degree_days[subregions].dot(cdd_activity_weights) cooling_degree_days = cooling_degree_days.reset_index('Year') cooling_degree_days['Year'] = cooling_degree_days['Year'].astype(int) weather_factors_df_cooling = cooling_degree_days[['Year', region]].rename(columns={region: 'CDD'}) weather_factors_df = weather_factors_df.merge(weather_factors_df_cooling, on='Year', how='outer') weather_factors_df['Time^3'] = weather_factors_df[['Time']].pow(3).values weather_factors_df = weather_factors_df.set_index('Year') weather_factors_df.index = weather_factors_df.index.astype(int) X_data = weather_factors_df[['HDD', 'CDD', 'Time', 'Time^2', 'Time^3']] elif energy_type == 'fuels': weather_factors_df['HDD*Time'] = heating_degree_days[region].multiply(weather_factors_df['Time']) weather_factors_df['Price'] = self.process_prices(weather_factors_df) weather_factors_df = weather_factors_df.set_index('Year') weather_factors_df.index = weather_factors_df.index.astype(int) X_data = weather_factors_df[['HDD', 'HDD*Time', 'Time', 'Time^2', 'Price']] # elif self.energy_type == 'delivered': # weather_factor = (reported_electricity + fuels) / (weather_adjusted_electrity + weather_adjusted_fuels) # return weather_factor else: raise KeyError(f'Missing valid energy type. Type given: {energy_type}') actual_intensity.index = actual_intensity.index.astype(int) data = X_data.merge(actual_intensity, left_index=True, right_index=True, how='inner').dropna() X = data.drop(region.capitalize(), axis=1) Y = data[[region.capitalize()]] reg = linear_model.LinearRegression() reg.fit(X, Y) coefficients = reg.coef_ coefficients = coefficients[0] intercept = reg.intercept_ predicted_value_intensity_actualdd = reg.predict(X) # Predicted value of the intensity based on actual degree days if energy_type == 'elec': prediction2_weights = self.adjust_data(subregions=subregions, hdd_by_division=heating_degree_days, cdd_by_division=cooling_degree_days, cdd_activity_weights=cdd_activity_weights_dict, hdd_activity_weights=hdd_activity_weights_dict, use_weights_1961_90=True) predicted_value_intensity_ltaveragesdd = intercept + coefficients[0] * prediction2_weights['heating'] + coefficients[1] * prediction2_weights['cooling'] + \ coefficients[2] * data['Time'] + coefficients[3] * data['Time^2'] + coefficients[4] * data['Time^3'] # Predicted value of the intensity based on the long-term averages of the degree days elif energy_type == 'fuels': prediction2_weights = self.adjust_data(subregions=subregions, hdd_by_division=heating_degree_days, hdd_activity_weights=hdd_activity_weights_dict, cooling=False, use_weights_1961_90=True) predicted_value_intensity_ltaveragesdd = intercept + coefficients[0] * prediction2_weights['heating'] + coefficients[1] * data['Time'] + \ coefficients[2] * data['Time'] + coefficients[3] * data['Time^2'] + coefficients[4] * data['Price'] # Predicted value of the intensity based on the long-term averages of the degree days weather_factor = predicted_value_intensity_actualdd.flatten() / predicted_value_intensity_ltaveragesdd.values.flatten() try: weather_normalized_intensity = actual_intensity.loc[data.index].divide(weather_factor.reshape(len(weather_factor), 1)) except Exception: try: weather_normalized_intensity = actual_intensity.loc[data.index].divide(weather_factor) except Exception as e: raise ValueError(f'Failure to divide: {actual_intensity.shape} by {weather_factor.shape}, failed with error {e}') weather_factor_df = pd.DataFrame(data={'Year': data.index, f'{region}_weather_factor': weather_factor}).set_index('Year') return weather_factor_df, weather_normalized_intensity def national_method1_fixed_end_use_share_weights(self, energy_type_): """Used fixed weights to develop from regional factors, weighted by regional energy share from 1995 CBECS """ if self.sector == 'commercial': shares = self.cbecs_1995_shares() intensity_df = self.commercial_regional_intensity_aggregate() elif self.sector == 'residential': intensity_df = self.residential_regional_intensity_aggregate() shares = self.recs_1993_shares() if energy_type_ == 'elec': energy_type = 'electricity' else: energy_type = energy_type_ regional_weather_factors = [] weights_df = self.gather_weights_data() regional_weights = self.regional_shares(dataframe=weights_df, cols=['heating_activity', 'cooling_activity', 'fuels']) for region in self.sub_regions_dict.keys(): region_cap = region.capitalize() if self.sector == 'residential': regional_intensity = intensity_df[region_cap][energy_type_] elif self.sector == 'commercial': regional_intensity = intensity_df[energy_type_][region_cap] weather_factors, weather_normalized_intensity = self.weather_factors(region, energy_type_, actual_intensity=regional_intensity, weights_df=weights_df, regional_weights=regional_weights) regional_weather_factors.append(weather_factors) weather_factors_all = pd.concat(regional_weather_factors, axis=1) weather_factors_all = weather_factors_all.reindex(columns=list(weather_factors_all.columns) + [f'{energy_type_}_weather_factor']) for y in weather_factors_all.index: if energy_type == 'electricity': energy_type = 'elec' share_name = f'{energy_type}_share' year_weather = weather_factors_all.drop(f'{energy_type_}_weather_factor', axis=1).loc[y, :] weights = shares[share_name].drop('Total') year_factor = year_weather.dot(weights.to_numpy()) weather_factors_all.loc[y, [f'{energy_type_}_weather_factor']] = year_factor return weather_factors_all def national_method2_regression_models(self, seds_data, weather_factors): seds_data, weather_factors = CalculateLMDI.ensure_same_indices(seds_data, weather_factors) weather_adjusted_consumption = seds_data.drop('National', axis=1).multiply(weather_factors.values) weather_adjusted_consumption['National'] = weather_adjusted_consumption.sum(axis=1) implicit_national_weather_factor = seds_data[['National']].divide(weather_adjusted_consumption['National'].values.reshape(len(weather_adjusted_consumption), 1)) return implicit_national_weather_factor def adjust_for_weather(self, data, energy_type): """Adjust data by weather factors Parameters ---------- data: dataframe dataset to adjust by weather weather_factors: array? Returns ------- weather_adjusted_data: dataframe ? """ weather_factors = self.national_method1_fixed_end_use_share_weights(energy_type) weather_adjusted_data = data / weather_factors[energy_type] return weather_adjusted_data def get_weather(self, energy_dict=None, energy_type=None, energy_df=None, weather_adjust=False, seds_data=None): if self.sector == 'residential': if weather_adjust: for type, energy_dataframe in energy_dict.items(): weather_adj_energy = self.adjust_for_weather(energy_dataframe, type) energy_dict[f'{type}_weather_adj'] = weather_adj_energy return energy_dict else: weather_factors = dict() for type in energy_dict.keys(): weather_factors_t = self.national_method1_fixed_end_use_share_weights(energy_type_=type) if type == 'electricity': type = 'elec' weather_factors[type] = weather_factors_t return weather_factors elif self.sector == 'commercial': weather_factors = dict() for type in ['electricity', 'fuels']: weather_factors_method1 = self.national_method1_fixed_end_use_share_weights(type) early_years = range(min(weather_factors_method1.index), 1969 + 1) weather_factors_early = weather_factors_method1.loc[early_years, [f'{type}_weather_factor']] weather = weather_factors_method1.drop(f'{type}_weather_factor', axis=1) if type == 'electricity': type = 'elec' type_seds = seds_data[type] weather_factors_method2 = self.national_method2_regression_models(seds_data=type_seds, weather_factors=weather) weather_factors_method2 = weather_factors_method2.rename(columns={'National': f'{type}_weather_factor'}) late_years = range(1970, max(weather_factors_method2.index) + 1) weather_factors_late = weather_factors_method2.loc[late_years] weather_factors_t = pd.concat([weather_factors_early, weather_factors_late], sort=True) weather_factors[type] = weather_factors_t return weather_factors
class ResidentialIndicators(CalculateLMDI): def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.eia_res = GetEIAData('residential') self.sub_categories_list = { 'National': { 'Northeast': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'Midwest': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'South': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None }, 'West': { 'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None } } } self.national_calibration = self.eia_res.national_calibration() self.seds_census_region = self.eia_res.get_seds( ) # energy_consumtpion_data_regional self.ahs_Data = ResidentialFloorspace.update_ahs_data() self.conversion_factors = self.eia_res.conversion_factors() self.regions = ['Northeast', 'South', 'West', 'Midwest', 'National'] self.base_year = base_year self.directory = directory self.end_year = end_year self.energy_types = ['elec', 'fuels', 'deliv', 'source'] super().__init__(sector='residential', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year) # self.AER11_table2_1b_update = GetEIAData.eia_api(id_='711250') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711250' # self.AnnualData_MER_22_Dec2019 = GetEIAData.eia_api(id_='711250') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711250' ? # self.RECS_intensity_data = # '711250' for Residential Sector Energy Consumption def get_seds(self): census_regions = {4: 'West', 3: 'South', 2: 'Midwest', 1: 'Northeast'} total_fuels = self.seds_census_region[0].rename(columns=census_regions) elec = self.seds_census_region[1].rename(columns=census_regions) return total_fuels, elec def fuel_electricity_consumption(self, total_fuels, elec, region): """Combine Energy datasets into one Energy Consumption dataframe in Trillion Btu Data Source: EIA's State Energy Data System (SEDS)""" fuels_dataframe = total_fuels[[region]] elec_dataframe = elec[[region]] energy_data = {'elec': elec_dataframe, 'fuels': fuels_dataframe} return energy_data def get_floorspace(self): residential_data = ResidentialFloorspace(end_year=self.end_year) floorspace_square_feet, occupied_housing_units, household_size_square_feet_per_hu = residential_data.final_floorspace_estimates( ) final_floorspace_results = { 'occupied_housing_units': occupied_housing_units, 'floorspace_square_feet': floorspace_square_feet, 'household_size_square_feet_per_hu': household_size_square_feet_per_hu } return final_floorspace_results def activity(self, floorspace): """Combine Energy datasets into one Energy Consumption Occupied Housing Units """ all_activity = dict() for region in self.sub_categories_list['National'].keys(): region_activity = dict() for variable, data in floorspace.items(): df = data[region] if variable == 'household_size_square_feet_per_hu': df = df.rename( columns={ 'avg_size_sqft_mf': 'Multi-Family', 'avg_size_sqft_mh': 'Manufactured-Homes', 'avg_size_sqft_sf': 'Single-Family' }) else: df = df.rename( columns={ 'occupied_units_mf': 'Multi-Family', 'occupied_units_mh': 'Manufactured-Homes', 'occupied_units_sf': 'Single-Family' }) print(variable, df.columns) region_activity[variable] = df all_activity[region] = region_activity return all_activity def collect_weather(self, energy_dict, nominal_energy_intensity): weather = WeatherFactors( sector='residential', directory=self.directory, nominal_energy_intensity=nominal_energy_intensity) weather_factors = weather.get_weather( energy_dict, weather_adjust=False ) # What should this return?? (e.g. weather factors or weather adjusted data, both?) return weather_factors def collect_data(self): total_fuels, elec = self.get_seds() floorspace = self.get_floorspace() activity = self.activity(floorspace) all_data = dict() nominal_energy_intensity_by_r = dict() for r in self.sub_categories_list['National'].keys(): region_activity = activity[r] energy_data = self.fuel_electricity_consumption(total_fuels, elec, region=r) nominal_energy_intensity_by_e = dict() for e, e_df in energy_data.items(): e_df = e_df.rename_axis(columns=None) floorspace = region_activity['floorspace_square_feet'] total_floorspace = floorspace.sum(axis=1) nominal_energy_intensity = self.lmdi( model=None, activity_input_data=total_floorspace, energy_input_data=e_df, unit_conversion_factor=1, return_nominal_energy_intensity=True ) # shouldn't rely on multiplicative? nominal_energy_intensity_by_e[e] = nominal_energy_intensity region_data = {'energy': energy_data, 'activity': region_activity} nominal_energy_intensity_by_r[r] = nominal_energy_intensity_by_e all_data[r] = region_data weather_factors = self.collect_weather( energy_dict=energy_data, nominal_energy_intensity=nominal_energy_intensity_by_r ) # need to integrate this into the data passed to LMDI for region, r_dict_ in all_data.items(): weather_factors_by_e_type = dict() for e_ in r_dict_['energy'].keys(): weather_factors_by_e_type[e] = weather_factors r_dict_['weather_factors'] = weather_factors_by_e_type all_data[region] = r_dict_ return all_data def main(self, breakout, save_breakout, calculate_lmdi): unit_conversion_factor = 1 data_dict = self.collect_data() results_dict, formatted_results = self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, save_breakout=save_breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, account_for_weather=True) return results
class CO2EmissionsDecomposition(CalculateLMDI): """Class to decompose CO2 emissions by sector of the U.S. economy. LMDI aspects: - total activity (Q), - activity share for subsector i (structure) (=Q_i/Q), - energy intensity for subsector i (=E_i/Q_i), - energy share of type j in subsector i (=E_ij/E_i), also called fuel mix - emissions rate of energy type j and subsector i (=C_ij/E_ij), also called emissions coefficient (for time series, all variables have t subscripts (i.e. no constants-- constant emissions rates cancel out)) """ def __init__(self, directory, output_directory, sector, config_path, categories_dict, level_of_aggregation): self.sector = sector self.eia = GetEIAData('emissions') self.config_path = config_path self.level_of_aggregation = level_of_aggregation super().__init__(sector, level_of_aggregation=self.level_of_aggregation, categories_dict=categories_dict, directory=directory, output_directory=output_directory, primary_activity=None, unit_conversion_factor=1, weather_activity=None, use_yaml_config=True, config_path=self.config_path) def collect_emissions(self, sector=None, fuel_type=None, state_abbrev=None): """Collect emissions from the EIA API (through GetEIAData). Args: sector (str, optional): Economic sector. Defaults to None. fuel_type (str, optional): Fuel Type. Defaults to None. state_abbrev (str, optional): State (e.g. 'AK'). Defaults to None. Returns: data (DataFrame): Emissions data for sector, fuel type and state """ eia_co2_emiss = f'EMISS.CO2-TOTV-{sector}-{fuel_type}-{state_abbrev}.A' data = self.eia.eia_api(id_=eia_co2_emiss, id_type='series') return data @staticmethod def get_fuel_mix(region_data): """Calculate shares of total fuel by fuel type Args: region_data (DataFrame): Fuel use data by fuel for a region Returns: fuel_mix (DataFrame): Fuel mix (i.e. share of total by fuel) """ region_data = region_data.drop('Census Region', axis=1, errors='ignore') region_data = df_utils().create_total_column(region_data, total_label='total') fuel_mix = \ df_utils().calculate_shares(region_data, total_label='total') return fuel_mix @staticmethod def weighted(x, cols, w="weights"): """Calculate weighted average Args: x ([type]): [description] cols ([type]): [description] w (str, optional): [description]. Defaults to "weights". Returns: (pd.Series): Weighted average """ return pd.Series(np.average(x[cols], weights=x[w], axis=0), cols) def get_mean_factor(self, emissions_factors, input_cols, new_name, portions=None): """Calculate average of given emissions factors Args: emissions_factors (pd.DataFrame): emissions factors input_cols (list): List of emissions factors to average new_name (str): Name of resulting factor portions (dict): Optional. Weights for weighted average (keys are cols) Returns: ef (pd.DataFrame): emissions factors with new type """ subset = \ emissions_factors[emissions_factors['Fuel Type'].isin(input_cols)] subset['weights'] = np.nan if portions: subset_ = [] for k, v in portions.items(): fuel_ = subset[subset['Fuel Type'] == k] fuel_['weights'] = v subset_.append(fuel_) subset = pd.concat(subset_, axis=0) subset.loc[:, 'weighted_value'] = \ subset['value'].multiply(subset['weights'].values) grouped = \ subset.groupby(by=['Unit', 'Variable']) mean_df = grouped.sum() mean_df = mean_df.drop('value', axis=1) mean_df['value'] = \ mean_df['weighted_value'].divide(mean_df['weights'].values) mean_df = mean_df.drop('weighted_value', axis=1) else: grouped = \ subset.groupby(by=['Unit', 'Variable']) mean_df = grouped.mean() mean_df.loc[:, 'Fuel Type'] = new_name mean_df.loc[:, 'Category'] = 'Merged Category' mean_df = mean_df.reset_index() mean_df = mean_df[[ 'Category', 'Fuel Type', 'Unit', 'value', 'Variable' ]] ef = pd.concat([emissions_factors, mean_df], axis=0) return ef def epa_emissions_data(self): """Read and process EPA emissions factors data Returns: emissions_factors (DataFrame): EPA emissions factors data """ try: ef = pd.read_csv( './EnergyIntensityIndicators/Data/EPA_emissions_factors.csv') except FileNotFoundError: os.chdir('..') print('changed dir:', os.getcwd()) ef = pd.read_csv( './EnergyIntensityIndicators/Data/EPA_emissions_factors.csv') df_cols = ef.columns dfs = [] grouped = ef.groupby(ef['Unit Type']) for g in ef['Unit Type'].unique(): unit_data = grouped.get_group(g) unit_data.columns = unit_data.iloc[0] units_dict = dict(zip(unit_data.columns, df_cols)) unit_data = unit_data.drop(g, axis=1) unit_data = unit_data.drop(unit_data.index[0]) unit_data = unit_data.melt(id_vars=['Units', 'Fuel Type'], var_name='Unit') unit_data = unit_data.rename(columns={'Units': 'Category'}) unit_data.loc[:, 'Variable'] = unit_data['Unit'].map(units_dict) dfs.append(unit_data) emissions_factors = pd.concat(dfs, axis=0) emissions_factors['value'] = \ emissions_factors['value'].apply(lambda x: str(x).replace(',', '')) emissions_factors['value'] = emissions_factors['value'].astype(float) ef = self.get_mean_factor(emissions_factors, input_cols=[ 'Distillate Fuel Oil No. 1', 'Distillate Fuel Oil No. 2', 'Distillate Fuel Oil No. 4' ], new_name='Distillate Fuel Oil') ef = self.get_mean_factor( ef, input_cols=['Residual Fuel Oil No. 5', 'Residual Fuel Oil No. 6'], new_name='Residual Fuel Oil') ef = self.get_mean_factor( ef, input_cols=['Blast Furnace Gas', 'Coke Oven Gas'], new_name='Blast Furnace/Coke Oven Gases') ef = self.get_mean_factor( ef, input_cols=['North American Softwood', 'North American Hardwood'], new_name='Pulping Liquor or Black Liquor') ef = self.get_mean_factor( ef, input_cols=['Motor Gasoline', 'Ethanol (100%)'], new_name='Gasohol', portions={ 'Ethanol (100%)': 0.16, 'Motor Gasoline': 0.84 }) ef = self.get_mean_factor(ef, input_cols=['Diesel Fuel', 'Motor Gasoline'], new_name='School', portions={ 'Diesel Fuel': 0.9, 'Motor Gasoline': 0.1 }) ef = self.get_mean_factor(ef, input_cols=[ 'Diesel Fuel', 'Distillate Fuel Oil No. 1', 'Distillate Fuel Oil No. 2', 'Distillate Fuel Oil No. 4' ], new_name='Diesel Fuel & Distillate') ef = self.get_mean_factor(ef, input_cols=[ 'Distillate Fuel Oil No. 1', 'Distillate Fuel Oil No. 2', 'Distillate Fuel Oil No. 4', 'Residual Fuel Oil No. 5', 'Residual Fuel Oil No. 6' ], new_name='Petroleum') return ef @staticmethod def mecs_epa_mapping(mecs_data): """Rename mecs_data columns so that labels match EPA emissions factors labels Args: mecs_data (DataFrame): Industrial sector energy use by fuel type Returns: mecs_data (DataFrame): MECS data with column names that match EPA emissions data labels """ rename_dict = {col: col.strip() for col in mecs_data.columns} mecs_data = mecs_data.rename(columns=rename_dict) mapping_ = { 'Waste Gas': 'Fuel Gas', 'Petroleum Coke': 'Petroleum Coke', 'Wood Chips, Bark': 'Wood and Wood Residuals', 'Waste Oils/Tars and Waste Materials': 'Used Oil', 'steam': 'Steam and Heat', # From Table 7 'Net Electricity': 'Us Average', # From Table 6, Total Output Emissions Factors CO2 Factor 'Electricity': 'US Average', 'Net Electricity(b)': 'US Average', 'Residual': 'Residual Fuel Oil', 'Distillate': 'Distillate Fuel Oil', 'Distillate Fuel Oil(c)': 'Distillate Fuel Oil', 'Nat. Gas': 'Natural Gas', 'Natural Gas(d)': 'Natural Gas', 'Natural Gas': 'Natural Gas', 'HGL (excluding natural gasoline)': 'Liquefied Petroleum Gases (LPG)', 'Coal': 'Mixed (Industrial Sector)', 'Coke Coal and Breeze': 'Coal Coke', 'Coke and Breeze': 'Coal Coke', 'Coke': 'Coal Coke', 'LPG': 'Liquefied Petroleum Gases (LPG)', 'Diesel': 'Diesel Fuel', 'LP Gas': 'Liquefied Petroleum Gases (LPG)', 'HGL (excluding natural gasoline)(e)': 'Liquefied Petroleum Gases (LPG)', 'Gasoline': 'Motor Gasoline', 'Gas': 'Natural Gas' } mecs_data = mecs_data.rename(columns=mapping_) mecs_data = mecs_data.drop('Total Fuel', axis=1, errors='ignore') return mecs_data @staticmethod def electric_epa_mapping(elec_data): """Rename elec power sector data (from EIA) columns so that labels match EPA emissions factors labels Args: elec_data (pd.DataFrame): Energy consumption for the elec power sector by fuel type Returns: elec_data (pd.DataFrame): Elec data with column names that match EPA emissions data labels """ rename_dict = { col: col[:col.find('Consumption')].strip() for col in elec_data.columns if 'Consumption' in col } rename_dict2 = { col: col[:col.find('Consumed')].strip() for col in elec_data.columns if 'Consumed' in col } intersect = [] for key, value in rename_dict2.items(): if value in rename_dict.values(): intersect.append(key) elec_data = elec_data.drop(intersect, axis=1) rename_dict.update(rename_dict2) others = { 'Electricity Net Generation From Wood, Electric Power Sector, Annual, Million Kilowatthours': 'Wood', 'Electricity Net Generation From Waste, Electric Power Sector, Annual, Million Kilowatthours': 'Waste' } rename_dict.update(others) elec_data = elec_data.rename(columns=rename_dict) mapping_ = { 'Coal': 'Mixed (Electric Power Sector)', 'Natural Gas': 'Natural Gas', 'Other Gases': 'Fuel Gas', 'Other Gas': 'Fuel Gas', 'Waste': 'Municipal Solid Waste', 'Wood': 'Wood and Wood Residuals', 'hydroelectric': 'Hydroelectric', 'Other': 'US Average', 'Other Petroleum Liquids': 'Liquefied Petroleum Gases (LPG)' } # might be wrong elec_data = elec_data.rename(columns=mapping_) elec_data = elec_data.drop('Total Petroleum', axis=1, errors='ignore') return elec_data @staticmethod def tedb_epa_mapping(tedb_data): """Rename transportation sector data (from TEDB) columns so that labels match EPA emissions factors labels Args: tedb_data (pd.DataFrame): Energy consumption for the transportation sector by fuel type Returns: tedb_data_ (pd.DataFrame): transportation data with column names that match EPA emissions data labels """ unit_coversion = { 'Diesel Fuel & Distillate (1,000 bbl)': 1000 / 42, 'Residual Fuel Oil (1,000 bbl)': 1000 / 42 } unit_coversion = { k: unit_coversion[k] for k in unit_coversion.keys() if k in tedb_data.columns } if len(unit_coversion) > 0: tedb_data = tedb_data.assign(**unit_coversion).mul(tedb_data) if 'Domestic Operations ' and 'International Operations ' in tedb_data.columns: tedb_data['Air'] = \ tedb_data[['Domestic Operations ', 'International Operations ']].sum(axis=1) tedb_data = tedb_data[['Air']] mapping = { 'Gasoline': 'Motor Gasoline', # ef is in gallon 'Gasoline (million gallons)': 'Motor Gasoline', # ef is in gallon 'Gasohol': 'Gasohol', # ef is in gallon 'Diesel (million gallons)': 'Diesel Fuel', # ef is in gallon 'Diesel': 'Diesel Fuel', # ef is in gallon 'CNG': 'Compressed Natural Gas (CNG)', # ef is in scf 'LNG': 'Liquefied Natural Gas (LNG)', # ef is in gallons 'Bio Diesel ': 'Biodiesel (100%)', # ef is in gallons 'Diesel Fuel & Distillate (1,000 bbl)': # ef is in gallon (42 gallons in a barrel) 'Diesel Fuel', 'Residual Fuel Oil (1,000 bbl)': 'Residual Fuel Oil', # ef is in gallon (42 gallons in a barrel) 'Jet fuel (million gallons)': 'Aviation Gasoline', # ef is per gallon 'Electricity (GWhrs)': 'US Average', # ef is kg/MWh 'Distillate Fuel Oil': 'Diesel Fuel', # ef is per gallon 'Natural Gas (million cu. ft.)': 'Natural Gas', # ef is per scf 'Electricity (million kWh)': 'US Average', # ef is kg/MWh 'Diesel fuel': 'Diesel Fuel', # ef is per gallon 'Liquefied petroleum gas': 'Liquefied Petroleum Gases (LPG)', # ef is per gallon 'LPG': 'Liquefied Petroleum Gases (LPG)', # ef is per gallon 'Air': 'Aviation Gasoline', # ef is per gallon 'Jet fuel': 'Aviation Gasoline', # ef is per gallon 'Residual fuel oil': 'Residual Fuel Oil', # ef is per gallon 'Natural gas': 'Natural Gas', # ef is per scf 'Electricity': 'US Average', # ef is kg/MWh 'Intercity': 'Diesel Fuel' # ef is in gallon } tedb_data_ = tedb_data.rename(columns=mapping) tedb_data_ = tedb_data_.drop('School (million bbl)', axis=1, errors='ignore') tedb_data_ = \ tedb_data_.drop(['Total Energy (Tbtu)', 'Total Energy (Tbtu) ', 'Total Energy (Tbtu) - old series'], axis=1, errors='ignore') return tedb_data_ @staticmethod def get_factor(factors_df, emissions_type): """Lookup emissions factor for given fuel and emissions type Args: factors_df (df): EPA emisions hub data fuel_name (str): Fuel type to look up in factors_df emissions_type (str): Type of emissions to return (e.g CO2) Returns: emissions_factor (float): emissions factor for given params """ factors_df = \ factors_df[factors_df['Variable'].isin( ['Heat Content (HHV)', emissions_type])] factors_df['value'] = factors_df['value'].fillna(1) factors_df = \ factors_df.groupby(['Category', 'Fuel Type']).prod() factors_df = factors_df.reset_index() fuel_factor_df = factors_df[['Fuel Type', 'value']] fuel_factor_df.loc[:, 'value'] = fuel_factor_df['value'].astype(float) new_row = {'Fuel Type': 'Census Region', 'value': 1} fuel_factor_df = fuel_factor_df.append(new_row, ignore_index=True) fuel_factor_df = fuel_factor_df.set_index('Fuel Type') no_emissions = [ 'Solar', 'Wind', 'Nuclear', 'Geothermal', 'Hydroelectric' ] e_data = [0] * len(no_emissions) no_emissions_df = pd.DataFrame(data=e_data, index=no_emissions, columns=['value']) no_emissions_df.index.name = 'Fuel Type' fuel_factor_df = pd.concat([fuel_factor_df, no_emissions_df], axis=0) fuel_factor_df = fuel_factor_df.transpose() return fuel_factor_df def calculate_emissions(self, energy_data, emissions_type='CO2 Factor', datasource='SEDS'): """Calculate emissions from the product of energy_data and emissions_factor Parameters: energy_data (pd.DataFrame): energy consumption by fuel data emissions_type (str): Type of emissions factor. Defaults to 'CO2 Factor'. datasource (str): 'SEDS', 'MECS', 'eia_elec', or 'TEDB'. Data source for energy consumption by fuel data. Defaults to 'SEDS'. Returns: emissions_data (pd.DataFrame): Emissions by fuel type energy_data (pd.DataFrame): Energy consumption by fuel data converted to MMBtu, if necessary, with columns matching emissions data TODO: Handle Other category (should not be dropped) """ # Also need to convert TEDB fuel (in gal or ft3) to MMBtu mapping_heat = { 'Motor Gasoline': 125000, # ef is in gallon 'Gasohol': 120900, # ef is in gallon 'Diesel Fuel': 138700, # ef is in gallon 'Compressed Natural Gas (CNG)': 129400, # ef is in scf 'Liquefied Natural Gas (LNG)': 84800, # ef is in gallons 'Liquefied Petroleum Gases (LPG)': 91300, 'Biodiesel (100%)': 128520, # ef is in gallons 'Residual Fuel Oil': 149700, # ef is in gallon (42 gallons in a barrel) 'Aviation Gasoline': 120900, # ef is per gallon 'US Average': 10339, # ef is kg/MWh; Btu/kWh 'Natural Gas': 1031, # ef is per scf 'Diesel Fuel': 138700, # ef is per gallon } energy_data = energy_data.drop('region', axis=1, errors='ignore') emissions_factors = self.epa_emissions_data() if datasource == 'SEDS': energy_data = self.epa_eia_crosswalk(energy_data) elif datasource == 'MECS': energy_data = self.mecs_epa_mapping(energy_data) elif datasource == 'eia_elec': energy_data = self.electric_epa_mapping(energy_data) elif datasource == 'TEDB': energy_data = self.tedb_epa_mapping(energy_data) energy_data = energy_data.drop('Total', axis=1, errors='ignore') ## TEMPORARY!! (need to add these factors to csv) energy_data = energy_data.drop(['Other'], axis=1, errors='ignore') emissions_factors = self.get_factor(emissions_factors, emissions_type) try: emissions_factors = \ emissions_factors[energy_data.columns.tolist()] except KeyError: print('energy_data.columns.tolist() not in dataframe:', energy_data.columns.tolist()) for t in energy_data.columns.tolist(): if t not in emissions_factors.columns.tolist(): print('t not in list:', t) print('emissions_factors columns:', emissions_factors.index) raise KeyError('Emissions data does not contain ' + 'all energy sources') emissions_data = \ energy_data.multiply(emissions_factors.to_numpy()) try: energy_data.loc[:, 'Census Region'] = \ energy_data.loc[:, 'Census Region'].astype(int).astype(str) census_region = True except KeyError: census_region = False try: # ensure string Census Regions are ints (at least some # start as floats-- need to match) emissions_data.loc[:, 'Census Region'] = \ emissions_data['Census Region'].astype(int).astype(str) except KeyError: if census_region: energy_data = energy_data[energy_data['Census Region'] == '0'] else: pass # Convert TEDB fuels to MMBtu if datasource == 'TEDB': try: energy_data = \ energy_data.apply(lambda x: mapping_heat[x.name]*x, axis=0) except KeyError: print('energy_data.columns.tolist() not in dataframe:', energy_data.columns.tolist()) for t in energy_data.columns.tolist(): if t not in mapping_heat.keys(): print('t not in list:', t) raise KeyError( 'Heat content conversion data does not contain ' + 'all energy sources') return emissions_data, energy_data def calc_lmdi(self, breakout, calculate_lmdi, data_dict): """Calculate decomposition of CO2 emissions for the U.S. economy TODO: Could simply call lmdi_gen main with a few slight adjustments """ results_dict, formatted_results = \ self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, lmdi_type=self.gen.lmdi_type) return results_dict
'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_FRP_NA_NA_NA_BLNY09DLR.A'}}, 'Miscellaneous': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_BMF_NA_NA_NA_TRLBTU.A', 'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_BMF_ELC_NA_NA_TRLBTU.A'}, # Energy data from "Balance of Manufacturing" 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_MISC_NA_NA_NA_BLNY09DLR.A'}}}, 'Nonmanufacturing': {'Agriculture/Forestry/Fishing/Hunting': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_AGG_NA_NA_NA_TRLBTU.A', 'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_AGG_PRC_NA_NA_TRLBTU.A'}, 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_AFF_NA_NA_NA_BLNY09DLR.A'}}, # here elec is purchased electricity, Note: try to find total elec 'Mining': {{'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_NA_NA_NA_TRLBTU.A', 'purchased_electricity': {'excluding_oil_shale': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_PEO_NA_NA_TRLBTU.A', 'including_oil_shale': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_PES_NA_NA_TRLBTU.A'}}, 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_MING_NA_NA_NA_BLNY09DLR.A'}}}, 'Construction': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_CNS_NA_NA_NA_TRLBTU.A', 'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_CNS_PRC_NA_NA_TRLBTU.A'}, 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_CNS_NA_NA_NA_BLNY09DLR.A'}}}} industrial_eia = GetEIAData('industrial') energy_use_industrial_electricity_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_ELC_NA_NA_QBTU.A', id_type='series') energy_use_industrial_total_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_TOT_NA_NA_QBTU.A', id_type='series') energy_use_industrial_delivered_energy_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_DELE_NA_NA_QBTU.A', id_type='series') industrial_value_of_shipments_total = activity_data = energy_data = pass def transportation_projections(self): """activity: - Passenger-miles [P-M] (Passenger)
def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model='multiplicative', tedb_date='04302020', base_year=1985, end_year=2018): self.transit_eia = GetEIAData('transportation') self.mer_table25_dec_2019 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.mer_table_43_nov2019 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.aer_2010_table_65 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.tedb_date = tedb_date self.energy_types = ['deliv'] self.sub_categories_list = { 'All_Passenger': { 'Highway': { 'Passenger Cars and Trucks': { 'Passenger Car – SWB Vehicles': { 'Passenger Car': None, 'SWB Vehicles': None }, 'Light Trucks – LWB Vehicles': { 'Light Trucks': None, 'LWB Vehicles': None }, 'Motorcycles': None }, 'Buses': { 'Urban Bus': None, 'Intercity Bus': None, 'School Bus': None }, 'Paratransit': None }, 'Rail': { 'Urban Rail': { 'Commuter Rail': None, 'Heavy Rail': None, 'Light Rail': None }, 'Intercity Rail': None }, 'Air': { 'Commercial Carriers': None, 'General Aviation': None } }, 'All_Freight': { 'Highway': { 'Single-Unit Truck': None, 'Combination Truck': None }, 'Rail': None, 'Air': None, # 'Waterborne': None, 'Pipeline': { 'Oil Pipeline': None, 'Natural Gas Pipeline': None } } } super().__init__(sector='transportation', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
class TransportationIndicators(CalculateLMDI): """Class to calculate Energy Intensity indicators for the U.S. Transportation Sector """ def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model='multiplicative', tedb_date='04302020', base_year=1985, end_year=2018): self.transit_eia = GetEIAData('transportation') self.mer_table25_dec_2019 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.mer_table_43_nov2019 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.aer_2010_table_65 = self.transit_eia.eia_api( id_='711272' ) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272' self.tedb_date = tedb_date self.energy_types = ['deliv'] self.sub_categories_list = { 'All_Passenger': { 'Highway': { 'Passenger Cars and Trucks': { 'Passenger Car – SWB Vehicles': { 'Passenger Car': None, 'SWB Vehicles': None }, 'Light Trucks – LWB Vehicles': { 'Light Trucks': None, 'LWB Vehicles': None }, 'Motorcycles': None }, 'Buses': { 'Urban Bus': None, 'Intercity Bus': None, 'School Bus': None }, 'Paratransit': None }, 'Rail': { 'Urban Rail': { 'Commuter Rail': None, 'Heavy Rail': None, 'Light Rail': None }, 'Intercity Rail': None }, 'Air': { 'Commercial Carriers': None, 'General Aviation': None } }, 'All_Freight': { 'Highway': { 'Single-Unit Truck': None, 'Combination Truck': None }, 'Rail': None, 'Air': None, # 'Waterborne': None, 'Pipeline': { 'Oil Pipeline': None, 'Natural Gas Pipeline': None } } } super().__init__(sector='transportation', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year) # self.transportation_data = {'Passenger Car – SWB Vehicles': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': '4_01', 'header_starts': 8, 'column_name': 'Fuel use'}, # Table4_01_{date} # 'total_energy': # {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_18'}, # 'vehicle_miles': # {'unit': 'miles', 'source': 'TEDB', 'table_number': '4_01', 'header_starts': 8, 'column_name': None}, # 'passenger_miles': # {'unit': 'miles', 'source': 'TEDB', 'table_number': 'A_18'}}, # 'Light Trucks – LWB Vehicles': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': '4_02', 'header_starts': 7}, # 'total_energy': # {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_05'}, # 'vehicle_miles': # {'unit': 'miles', 'source': 'TEDB', 'table_number': '4_02'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'TEDB', 'table_number': 'A_19'}}, # 'Motorcycles': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_02'}, # 'total_energy': # {'unit': 'tbtu', 'source': None, 'table_number': None, 'Assumptions': 'Assume all motorcycle fuel is gasoline'}, # 'vehicle_miles': # {'unit': 'miles', 'source': 'BTS', 'table_number': '1-35'}, # with some interpolation prior to 1990' # 'passenger_miles': # {'unit': 'miles', 'source': None, 'table_number': None, 'Assumptions': vehicle_miles * 1.1}}, # 'Urban Bus': {'total_fuel': # {'unit': 'gallons', 'source': 'APTA Public Transportation Fact Book'}, # 'total_energy': # {'unit': '', 'source': None, 'table_number': None, 'Assumption': 'Apply conversion factors (Btu/gallon)'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'APTA'}}, # 'Intercity Bus': {'total_fuel': # {'unit': 'gallons', 'source': 'Earlier data from TEDB-19 later from ABA'}, # 'total_energy': # {'unit': 'tbtu', 'source': None, 'Assumptions': 'All fuel assumed to be diesel'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'Multiple'}}, # 'School Bus': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_04'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_04', 'Assumptions': 'Fuel assumed to be 90% diesel, 10% gasoline'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'Multiple'}}, # 'Commuter Rail': {'total_fuel': # {'unit': 'kwh_gallons', 'source': 'APTA Public Transportation Fact Book'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'TEDB', 'Assumptions': 'Conversion to TBtu with electricity and diesel energy conversion factors (electricity conversion at 10,339 Btu/kWh from TEDB'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'APTA', 'table_number': ' Table 3 of Appendix A'}}, # 'Transit Rail (Heavy and Light Rail)': {'total_fuel': # {'unit': 'kwh_gallons', 'source': '(APTA, Fact Book)' , 'table_number': 'Tables 38 and 39 in Appendix A'}, # 'total_energy': # {'unit': 'tbtu', 'source': None, 'table_number': None, 'Assumptions': 'Conversion to TBtu as for commuter rail'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'APTA', 'table_number': 'source as avober for fuel use. Passenger-miles in Table 3 of Appendix'}}, # 'Intercity Rail': {'total_fuel': # {'unit': 'kwh_gallons', 'source': '1994-2016" TEDB-37 Table A.16; prior data extrapolated from TEDB-30, Table 9.10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to TBtu as for commuter rail'}, # 'passenger_miles': # {'unit': 'miles', 'source': 'APTA source as avober for fuel use. Passenger-miles in Table 3 of Appendix'}}, # 'Commercial Carriers': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_09'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'passenger_miles': # {'unit': 'miles', 'source': '1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use'}}, # 'General Aviation': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'passenger_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}}, # 'Single-Unit Truck': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_09'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}, # 'ton_miles': {'unit': None, 'source': None}}, # 'Combination Truck': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}}, # 'Rail': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}}, # 'Air Carriers': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}}, # 'Waterborne': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}}, # 'Natural Gas Pipelines': {'total_fuel': # {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'}, # 'total_energy': # {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'}, # 'vehicle_miles': # {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},} # def import_tedb_data(self, table_number): # file_url = f'https://tedb.ornl.gov/wp-content/uploads/2020/04/Table{table_number}_{self.tedb_date}.xlsx' # xls = pd.read_excel(file_url) # , sheetname=None, header=11 # return xls # def get_data_from_nested_dict(self, nested_dictionary, subcategory_data_sources): # for key, value in nested_dictionary.items(): # if type(value) is dict: # get_data_from_nested_dict(value) # elif value is None: # data_sources = self.transportation_data[key] # subcategory_data_sources[key] = data_sources # return key, data_sources # def load_data(self): # energy_mode_data = dict() # fuel_mode_data = dict() # vehcile_miles_mode_data = dict() # for mode_name, mode_dict in self.transportation_data.items(): # could be more efficient with pandas? # for variable_name, variable_dict in mode_dict.items(): # if variable_name == 'total_fuel': # pass # elif variable_name == 'total_energy': # pass # elif variable_name == 'vehicle_miles': # pass # if variable_dict['source'] == 'TEDB': # mode_source_df = import_tedb_data(variable_dict['table_number']) # get_data_from_nested_dict(self.categories_dict, subcategory_data_sources) # pass # def fuel_heat_content(self, parameter_list): # """Assumed Btu content of the various types of petroleum products. This dataframe is not time variant (no need to update it with subsquent indicator updates) # Parameters # ---------- # Returns # ------- # """ # pass # def fuel_consump(self, parameter_list): # """Time series of fuel consumption for the major transportation subsectors. Data are generally in # millions gallons or barrels of petroleum. # Parameters # ---------- # Returns # ------- # """ # # swb_vehciles_all_fuel = [0] # 2007-2017 FHA Highway Statistics Table VM-1 # # motorcycles_all_fuel_1949_1969 = [0] # Based upon data published in Table 4_11 of Bureau of Transportation Statistics (BTS), fuel consumption is based assumption of 50 miles per gallon # # motorcycles_all_fuel_1970_2017 = import_tedb_data(table_number='A_02') # Alternative: 2017 data from Highway Statistics, Table VM-1. # # light_trucks_all_fuel = # # lwb_vehicles_all_fuel = # # bus_urban_diesel = # # bus_urban_cng = # # bus_urban_gasoline = # # bus_urban_lng = # # bus_urban_bio_diesel = # # bus_urban_other = # # bus_urban_lpg = # # paratransit = # # bus_school = # # bus_school_million_bbl = # # bus_intercity = [0] # Not used # # waterborne = # # air = # # rail_intercity_diesel_million_gallons = # # rail_intercity_diesel_electricity_gwhrs = # # rail_intercity_total_energy_tbtu = # # rail_intercity_total_energy_tbtu_old = # # rail_intercity_total_energy_tbtu_adjusted = # # rail_commuter_diesel = # # rail_commuter_electricity_gwhrs = # # rail_heavy_electricity_gwhrs = # # rail_light_electricity_gwhrs = # # class_I_freight_distillate_fuel_oil = # # pipeline_natural_gas_million_cu_ft = # # pipeline_natutral_gas_electricity_million_kwh = # pass # def adjust_truck_freight(self, parameter_list): # """purpose # Parameters # ---------- # Returns # ------- # DOES THIS WORK ? dataframes # """ # gross_output = bls_data[] # Note: Gross output in million 2005 dollars from BLS database for their employment projections input-output model, # # PNNL spreadsheet: BLS_output_data.xlsx in folder BLS_Industry_Data) # vehicle_miles_fhwa_tvm1 = [0] # old_methodology_2007_extrapolated = gross_output.iloc[2007] / gross_output.iloc[2006] * vehicle_miles_fhwa_tvm1.iloc[2006, :] # old_series_scaled_to_new = vehicle_miles_fhwa_tvm1 * vehicle_miles_fhwa_tvm1.iloc[2007, :] / old_methodology_2007_extrapolated # # **come back** # pass # def passenger_based_energy_use(self): # """Calculate the fuel consumption in Btu for passenger transportation # """ # all_passenger_categories = self.categories_dict['All_Passenger'] # for passenger_category in all_passenger_categories.keys(): # for transportation_mode in passenger_category.keys(): # if transportation_mode == 'Passenger Car – SWB Vehicles': # # Passenger Car and SWB Vehicles have separate data sources, later aggregated? # elif transportation_mode == 'Light Trucks – LWB Vehicles': # # Light Trucks and LWB Vehicles have separate data sources, later aggregated? # else: # urban_rail_categories = list(all_passenger_categories['Rail']['Urban Rail'].values()]) # passenger_based_energy_use_df['Urban Rail (Hvy, Lt, Commuter)'] = passenger_based_energy_use_df[urban_rail_categories].sum(1) # pass # def passenger_based_activity_data(self): # # Passenger cars and light trucks # fha_table_vm201a = pd.read_excel('https://www.fhwa.dot.gov/ohim/summary95/vm201a.xlw', sheetname='UnknownSheet4', header=11) # Passenger car and light truck VMT 1981-95 # fha_table_vm1 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', header=4) # fha_table_vm1_1996 = pd.read_excel('https://www.fhwa.dot.gov/ohim/1996/vm1.xlw') # fha_table_vm1_1997 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs97/xls/vm1.xls', sheetname='1997 VM1', header=6) # fha_table_vm1_1998 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/1998/xls/vm1.xls', sheetname='Pub 98 VM1', header=6) # fha_table_vm1_1999 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs99/excel/vm1.xls', sheetname='1999 VM1', header=3) # fha_table_vm1_2000 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs00/xls/vm1.xls', sheetname='Sheet1', header=4) # fha_table_vm1_2001 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs01/xls/vm1.xls', sheetname='Sheet1', header=4) # vmt2002_2006_passenger_car_light_truck = pd.read_excel('https://www.bts.gov/sites/bts.dot.gov/files/table_01_35_013020.xlsx', header=1) # Note, this link has changed since usage in the spreadsheets # vmt2007_2008_passenger_car = import_tedb_data(table_number='4_01') # vmt2007_2008_light_truck = import_tedb_data(table_number='4_02') # load_factors_1966_2011 = # # Short Wheelbase and Long Wheelbase Vehicles # 2007-2017 FHA VM1 # fha_table_vm1_2014 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2014/xls/vm1.xlsx', sheetname='final VM-1' , header=4) # # Motorcycles # 1970-2010 = vmt2002_2006_passenger_car_light_truck # fha_tables # # Bus / Transit # # 1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46. # apta_table3 = pd.read_excel('https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', sheetname='3', header=2) # 1997-2017 # # # Bus / Intercity # # see revised_intercity_bus_estimates # # # Bus / School # # see revised_intercity_bus_estimates # # Paratransit # paratransit_activity_1984_2017 = apta_table3 # # Commercial Air Carrier # commercial_air_carrier_activity_1970_1974 = [0] # TEDB 13 Table 6.2 page 6-7 # commercial_air_carrier_activity_1975_1984 = [0] # TEDB 19 Table 12.1 page 12-2 (is the 2 a typo?) # commercial_air_carrier_activity_1985_1999 = [0] # TEDB 21 Table 12.1 page 12-2 (is the 2 a typo?) # commercial_air_carrier_activity_2000_2018 = import_tedb_data(table_number='10_02') # domestic_passenger_miles_2011 = [0] # from Table 2.12 in 2013 TEDB # # Urban Rail (Commuter) # urban_rail_commuter_activity_1970_1976 = [0] # Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 6.13, p. 6-31. # urban_rail_commuter_activity_1977_2017 = apta_table3 # # Urban Rail (Light, Heavy) # urban_rail_light_heavy_1970_1977 = import_tedb_data(table_number='10_10') # urban_rail_light_heavy_1977_2017 = apta_table3 # # Intercity Rail (Amtrak) Note: Amtrak data is compiled by fiscal year rather than calendar year # intercity_rail_1970 = None # Amtrak not established until May 1971. Data for 1970 assumes same passenger activity as 1971, primarily included # # to fill out all transportation-related time series back to 1970 # intercity_rail_1971_2016 = [0] # intercity_rail_2017 = [0] # 'Bureau of Transportation Statistics, National Transportation Statistics, Table 1-40: U.S. Passenger-Miles # # General Aviation # # !!!! # return passenger_based_activity_input_data # def passenger_based_activity(self): # """ Time series for the activity measures for passenger transportation # # Highway # self.intercity_buses = 'Detailed data_Intercity buses' # Column E # 1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46. # 1977-2017 American Public Transportation Association. 2019 Public Transportation Fact Book. Appendix A, Table 3, Pt. A # Note: Transit bus does not include trolley bus because APTA did not not report electricity consumption for trolley buses separately for all years. # https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/ # Note: Series not continuous for transit bus between 2006 and 2007; estimation procedures changed for bus systems outside urban areas # Note: Series not continuous between 1983 and 1984. # Note: Transit Bus == Urban Bus # This method is horribly structured. # """ # all_passenger_categories = self.categories_dict['All_Passenger'] # for passenger_category in all_passenger_categories.keys(): # for transportation_mode in all_passenger_categories[passenger_category].keys(): # if transportation_mode == 'Passenger Car – SWB Vehicles': # # Passenger Car and SWB Vehicles have separate data sources, later aggregated? # elif transportation_mode == 'Light Trucks – LWB Vehicles': # # Light Trucks and LWB Vehicles have separate data sources, later aggregated? # else: # urban_rail_categories = list(all_passenger_categories['Rail']['Urban Rail'].values()]) # passenger_based_energy_use_df['Urban Rail (Hvy, Lt, Commuter)'] = passenger_based_energy_use_df[urban_rail_categories].sum(1) # pass # def freight_based_energy_use(self): # """Calculate the fuel consumption in Btu for freight transportation # Need FuelConsump, Fuel Heat Content # """ # all_freight_categories = self.categories_dict['All_Freight'] # for freight_category in all_freight_categories.keys(): # for freight_mode in freight_category.keys(): # pass # def freight_based_activity(self): # """Time series for the activity measures for passenger transportation # """ # all_freight_categories = self.categories_dict['All_Freight'] # for freight_category in all_freight_categories.keys(): # for freight_mode in freight_category.keys(): # if freight_category == 'Waterborne': # domestic_vessel = [0] # international_vessel_in_us_waters = [0] # total_commerce_us_waters = domestic_vessel + international_vessel_in_us_waters # elif freight_category == 'Highway': # highway_published = [0] # if freight_mode == 'Combination Truck': # freight_based_energy_use_df['Old Series from 2001 Eno, Trans. In America'] = [0] # # 1950-1989: # freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0] # 1990 value for this column divided by 1990 value for 'Old Series from 2001 Eno, Trans. In America' * contemporary year from old series # # 1990-2003 # freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0] # # 2004-2017 # vehicle_miles_combination_trucks_adjusted = [0] # from adjust truck freight column K # freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0] # contemporary value of vehicle_miles_combination_trucks_adjusted divided by 2003 value of vehicle_miles_combination_trucks_adjusted times 2003 value of this # elif freight_mode == 'Single-Unit Truck': # # 1970-2006 # freight_based_energy_use_df['Single-Unit Truck (million vehicle-miles), adjusted'] = [0] # Adjust_truck_Freight Column J # # 2007-2017 # freight_based_energy_use_df['Single-Unit Truck (million vehicle-miles), adjusted'] = highway_published['Single-Unit Truck (million vehicle-miles)'] # if freight_mode == 'Natural Gas Pipeline': # natrual_gas_delivered_to_end_users = self.table65_AER2010 # Column AH, million cu. ft. # natural_gas_delivered_lease_plant_pipeline_fuel = self.MER_Table43_Nov2019 # Column M - column D - column I # natural_gas_delivered_lease_plant_pipeline_fuel.at[0] = 0.000022395 # natural_gas_consumption_million_tons = natrual_gas_delivered_to_end_users * natural_gas_delivered_lease_plant_pipeline_fuel[0] # avg_length_natural_gas_shipment_miles = 620 # freight_based_energy_use_df[freight_mode] = natural_gas_consumption_million_tons * 620 # pass # def water_freight_regression(self, ): # """purpose # Parameters # ---------- # Returns # ------- # """ # intensity = # X = math.log(intensity) # Y = years # reg = linear_model.LinearRegression() # reg.fit(X, Y) # coefficients = reg.coef_ # predicted_value = reg.predict(X_test) # Predicted value of the intensity based on actual degree days # def detailed_data_school_buses(self, parameter_list): # """purpose # Parameters # ---------- # Returns # ------- # """ # pass # def detailed_data_intercity_buses(self, parameter_list): # """purpose # Parameters # ---------- # Returns # ------- # """ # passenger_miles = [0] # revised_passenger_miles = [0] # number_of_buses = [0] # number_of_buses_old = [0] # Not used # vehicle_miles_commercial = [0] # Not used # vehicle_miles_intercity = [0] # implied_load_factor = [0] # energy_use_tedb_19_32 = [0] # Not used # implied_mpg = [0] # Not used # blended_mpg_miles_per_gallon = [0] # # revised_energy_use = [0] # number_of_motorcoaches = [0] # number_of_motorcoaches['Ratio'] = number_of_motorcoaches['US'] / number_of_motorcoaches['Total N.A.'] # pass # def mpg_check(self, parameter_list): # """purpose # Parameters # ---------- # Returns # ------- # """ # pass def energy_consumption(self): """Gather Energy Data to use in LMDI Calculation TBtu """ pass def activity(self): """Gather """ pass def collect_data(self): """Method to collect freight and passenger energy and activity data. This method should be adjusted to call dataframe building methods rather than reading csvs, when those are ready Gather Energy Data to use in LMDI Calculation TBtu and Activity Data to use in LMDI Calculation passenger-miles [P-M], ton-miles [T-M] Returns: [type]: [description] """ # passenger_based_energy_use = self.energy_consumption('All_Passenger') # passenger_based_activity = self.activity('All_Passenger') # freight_based_energy_use = self.energy_consumption('All_Freight') # freight_based_activity = self.activity('All_Freight') passenger_based_energy_use = pd.read_csv( './EnergyIntensityIndicators/Transportation/passenger_based_energy_use.csv' ).set_index('Year') passenger_based_activity = pd.read_csv( './EnergyIntensityIndicators/Transportation/passenger_based_activity.csv' ).set_index('Year') freight_based_energy_use = pd.read_csv( './EnergyIntensityIndicators/Transportation/freight_based_energy_use.csv' ).set_index('Year') freight_based_activity = pd.read_csv( './EnergyIntensityIndicators/Transportation/freight_based_activity.csv' ).set_index('Year') data_dict = { 'All_Passenger': { 'energy': { 'deliv': passenger_based_energy_use }, 'activity': passenger_based_activity }, 'All_Freight': { 'energy': { 'deliv': freight_based_energy_use }, 'activity': freight_based_activity } } print(freight_based_energy_use) return data_dict def main(self, breakout, save_breakout, calculate_lmdi): # base_year=None, """potentially refactor later Args: _base_year ([type], optional): [description]. Defaults to None. """ # if not base_year: # _base_year = self.base_year # else: # _base_year = _base_year data_dict = self.collect_data() if self.level_of_aggregation == 'all_transportation': freight_activity, freight_energy = self.get_nested_lmdi( level_of_aggregation='All_Freight', breakout=breakout, save_breakout=save_breakout, calculate_lmdi=False, raw_data=data_dict) passenger_activity, passenger_energy = self.get_nested_lmdi( level_of_aggregation='All_Passenger', breakout=breakout, save_breakout=save_breakout, calculate_lmdi=False, raw_data=data_dict) all_transportation_activity = freight_activity[['All_Freight']].merge(passenger_activity[['All_Passenger']], \ left_index=True, right_index=True, how='outer') all_transportation_energy = freight_energy[['All_Freight']].merge(passenger_energy[['All_Passenger']], \ left_index=True, right_index=True, how='outer') results = self.call_lmdi(unit_conversion_factor=1, lmdi_models=self.lmdi_models) elif self.level_of_aggregation == 'personal_vehicles_aggregate': # THIS CASE NEEDS A DIFFERENT FORMAT (NOT SAME NESTED DICTIONARY STRUCTURE), need to come back categories = ['Passenger Car', 'Light Truck', 'Motorcycles'] results = None else: results_dict, results = self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, save_breakout=save_breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict) results.to_csv(f"{self.output_directory}/transportation_results2.csv") print('RESULTS:\n', results) return results def compare_aggregates(self, parameter_list): """purpose Parameters ---------- Returns ------- """ total_fuel_tbtu_published_mer = self.mer_table_25_dec_2019[ 'Total Energy Consumed by the Transportation Sector'] # j sum_of_modes = self.total_transportation[ 'Energy_Consumption_Total'] # F difference = sum_of_modes - total_fuel_tbtu_published_mer pct_difference = difference / total_fuel_tbtu_published_mer return pct_difference
class ResidentialIndicators(CalculateLMDI): """Class to decompose changes in Energy Consumption from the Residential Sector of the US Economy """ def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.eia_res = GetEIAData('residential') housing_types = \ {'Single-Family': None, 'Multi-Family': None, 'Manufactured-Homes': None} self.sub_categories_list = \ {'National': {'Northeast': housing_types, 'Midwest': housing_types, 'South': housing_types, 'West': housing_types}} self.national_calibration = \ self.eia_res.national_calibration() self.seds_census_region = \ self.eia_res.get_seds() # energy_consumtpion_data_regional RF = ResidentialFloorspace() self.ahs_Data = RF.update_ahs_data() self.regions = ['Northeast', 'South', 'West', 'Midwest', 'National'] self.base_year = base_year self.directory = directory self.end_year = end_year self.energy_types = ['elec', 'fuels', 'deliv', 'source'] super().__init__(sector='residential', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, energy_types=self.energy_types, directory=directory, output_directory=output_directory, primary_activity='occupied_housing_units', base_year=base_year, end_year=end_year, weather_activity='floorspace_square_feet') print("self.dir()):", dir(self)) # self.AER11_table2_1b_update = GetEIAData.eia_api(id_='711250') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711250' # self.AnnualData_MER_22_Dec2019 = GetEIAData.eia_api(id_='711250') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711250' ? # self.RECS_intensity_data = # '711250' for Residential Sector Energy Consumption def get_seds(self): """Collect SEDS data Returns: total_fuels (pd.DataFrame): Fuels data elec (pd.DataFrame): Elec Data """ census_regions = {4: 'West', 3: 'South', 2: 'Midwest', 1: 'Northeast'} total_fuels = self.seds_census_region[0].rename(columns=census_regions) elec = self.seds_census_region[1].rename(columns=census_regions) return total_fuels, elec def fuel_electricity_consumption(self, total_fuels, elec, region): """Combine Energy datasets into one Energy Consumption dataframe in Trillion Btu Data Source: EIA's State Energy Data System (SEDS) Args: total_fuels ([type]): [description] elec ([type]): [description] region ([type]): [description] Returns: energy_data (dict): [description] """ fuels_dataframe = total_fuels[[region]] elec_dataframe = elec[[region]] energy_data = {'elec': elec_dataframe, 'fuels': fuels_dataframe} return energy_data def get_floorspace(self): """Collect floorspace data for the Residential sector Returns: final_floorspace_results (dict): [description] """ residential_data = ResidentialFloorspace(end_year=self.end_year) floorspace_square_feet, \ occupied_housing_units, \ household_size_square_feet_per_hu = \ residential_data.final_floorspace_estimates() final_floorspace_results = \ {'occupied_housing_units': occupied_housing_units, 'floorspace_square_feet': floorspace_square_feet, 'household_size_square_feet_per_hu': household_size_square_feet_per_hu} return final_floorspace_results def activity(self, floorspace): """Combine Energy datasets into one Energy Consumption Occupied Housing Units Args: floorspace ([type]): [description] Returns: all_activity (dict): [description] """ all_activity = dict() for region in self.sub_categories_list['National'].keys(): region_activity = dict() for variable, data in floorspace.items(): df = data[region] if variable == 'household_size_square_feet_per_hu': df = df.rename( columns={ 'avg_size_sqft_mf': 'Multi-Family', 'avg_size_sqft_mh': 'Manufactured-Homes', 'avg_size_sqft_sf': 'Single-Family' }) else: df = df.rename( columns={ 'occupied_units_mf': 'Multi-Family', 'occupied_units_mh': 'Manufactured-Homes', 'occupied_units_sf': 'Single-Family' }) region_activity[variable] = df all_activity[region] = region_activity return all_activity def collect_weather(self, energy_dict, nominal_energy_intensity): """Collect weather data for the Residential Sector Args: energy_dict ([type]): [description] nominal_energy_intensity ([type]): [description] Returns: weather_factors (dict): [description] """ weather = \ WeatherFactors(sector='residential', directory=self.directory, nominal_energy_intensity=nominal_energy_intensity) # What should this return?? (e.g. weather factors or weather adjusted data, both?) weather_factors = weather.get_weather(energy_dict, weather_adjust=False) return weather_factors def collect_data(self): """Gather all input data for you in decomposition of energy use for the Residential sector Returns: all_data (dict): All input data for the Residential Sector Energy Decomposition """ total_fuels, elec = self.get_seds() floorspace = self.get_floorspace() activity = self.activity(floorspace) all_data = dict() nominal_energy_intensity_by_r = dict() for r in self.sub_categories_list['National'].keys(): region_activity = activity[r] energy_data = \ self.fuel_electricity_consumption(total_fuels, elec, region=r) nominal_energy_intensity_by_e = dict() for e, e_df in energy_data.items(): e_df = e_df.rename_axis(columns=None) floorspace = region_activity['floorspace_square_feet'] total_floorspace = floorspace.sum(axis=1) nominal_energy_intensity = \ self.nominal_energy_intensity( energy_input_data=e_df, activity_data_=total_floorspace) nominal_energy_intensity_by_e[e] = \ nominal_energy_intensity region_data = {'energy': energy_data, 'activity': region_activity} nominal_energy_intensity_by_r[r] = nominal_energy_intensity_by_e all_data[r] = region_data weather_factors = self.collect_weather( energy_dict=energy_data, nominal_energy_intensity=nominal_energy_intensity_by_r ) # need to integrate this into the data passed to LMDI national_weather_dict = dict() for region, r_dict_ in all_data.items(): weather_factors_by_e_type = dict() for e_ in r_dict_['energy'].keys(): national_weather_dict[e_] = \ weather_factors[e_][[f'{e_}_weather_factor']] e_r_weather =\ weather_factors[e_][[f'{region.lower()}_weather_factor']] weather_factors_by_e_type[e_] = e_r_weather r_dict_['weather_factors'] = weather_factors_by_e_type all_data[region] = r_dict_ all_data = {'National': all_data} return all_data def main(self, breakout, calculate_lmdi): """Calculate decomposition for the Residential sector Args: breakout ([type]): [description] calculate_lmdi ([type]): [description] Returns: [type]: [description] """ unit_conversion_factor = 1 data_dict = self.collect_data() results_dict, formatted_results = \ self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, lmdi_type='LMDI-I') return results_dict
class IndustrialIndicators(CalculateLMDI): """Some of the specific steps to download and process the census data on construction energy costs are explained in the following paragraphs. The top-level census bureau website for the Economic Census is: https://www.census.gov/programs-surveys/economic-census.html. Scroll down the page until the words “2017 Data Tables” are found. After clicking on that link, the user will end up at https://www.census.gov/programs-surveys/economic-census/news-updates/updates/2017-datatables.html. The “2017 Data Table pages” now include direct links into data.census.gov and large ftp downloads. After clicking on pages, the webpage https://www.census.gov/programssurveys/economic-census/data/tables.html comes up. Scroll down this page until the entry “Construction (NAICS Sector 23)” is found. After selecting this entry, the user is then automatically transferred to: https://www.census.gov/data/tables/2017/econ/economic-census/naics-sector23.html. """ def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018, naics_digits=3): self.sub_categories_list = \ {'Industry': {'Manufacturing': {'Food and beverage and tobacco products': None, 'Textile mills and textile product mills': None, 'Apparel and leather and allied products': None, 'Wood products': None, 'Paper products': None, 'Printing and related support activities': None, 'Petroleum and coal products': None, 'Chemical products': None, 'Plastics and rubber products': None, 'Nonmetallic mineral products': None, 'Primary metals': None, 'Fabricated metal products': None, 'Machinery': None, 'Computer and electronic products': None, 'Electrical equipment, appliances, and components': None, 'Motor vehicles, bodies and trailers, and parts': None, 'Furniture and related products': None, 'Miscellaneous manufacturing': None}, 'Nonmanufacturing': {'Agriculture, Forestry & Fishing': None, 'Mining': {'Petroleum and Natural Gas': None, 'Other Mining': None, 'Support Activities': None}, 'Construction': None}}} self.ind_eia = GetEIAData('industry') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.MER_Nov19_Table24 = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER10_Table21d = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER11_Table21d_MER0816 = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.mer_dataT0204 = \ self.ind_eia.eia_api(id_='711252') self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] self.naics_digits = naics_digits super().__init__(sector='industry', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year, primary_activity='value_added') def reconcile_physical_units(self, ): """Convert physical units to Btu. (Prior to 2005, the data on energy consumption fuels to produce electricity were supplied in physical units (e.g. mcf of natural gas, tons of coal, etc)) Data Source: EIA's Annual Energy Review (AER)""" pass def manufacturing(self): """Gather manufacturing data Returns: [type]: [description] """ manufacturing_data = \ Manufacturing(self.naics_digits).manufacturing() return manufacturing_data def non_manufacturing(self): """Gather non-manufacturing data Primary Data Sources: Economic Census (previously the Census of Manufactures, Census of Agriculture, and Census of Mining) Prior to 1985, primary data source is the National Energy Accounts (NEA) http://www.nass.usda.gov/Statistics_by_Subject/index.php Returns: [type]: [description] """ non_manufacturing_data = \ NonManufacturing(self.naics_digits).nonmanufacturing_data() return non_manufacturing_data def collect_data(self): """Gather all input data for decomposition of the energy use in the Industrial sector Returns: data_dict (dict): [description] """ non_man = self.non_manufacturing() man = self.manufacturing() data_dict = { 'Industry': { 'Manufacturing': man, 'Nonmanufacturing': non_man } } return data_dict def total_industrial_util_adj_lmdi(self): """[summary] Returns: util_adj_categories (list): [description] """ # This case is quite different from the others util_adj_categories = [ 'Fuels', 'Delivered Electricity', 'Source Electricity', 'Total Source' ] return util_adj_categories def main(self, breakout, calculate_lmdi): """Calculate decomposition for the Industrial sector Args: breakout ([type]): [description] calculate_lmdi ([type]): [description] Returns: [type]: [description] """ # unit_conversion_factor = 1 data_dict = self.collect_data() results_dict, formatted_results = \ self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, lmdi_type='LMDI-I') return results_dict
class IndustrialIndicators(CalculateLMDI): """Some of the specific steps to download and process the census data on construction energy costs are explained in the following paragraphs. The top-level census bureau website for the Economic Census is: https://www.census.gov/programs-surveys/economic-census.html. Scroll down the page until the words “2017 Data Tables” are found. After clicking on that link, the user will end up at https://www.census.gov/programs-surveys/economic-census/news-updates/updates/2017-datatables.html. The “2017 Data Table pages” now include direct links into data.census.gov and large ftp downloads. After clicking on pages, the webpage https://www.census.gov/programssurveys/economic-census/data/tables.html comes up. Scroll down this page until the entry “Construction (NAICS Sector 23)” is found. After selecting this entry, the user is then automatically transferred to: https://www.census.gov/data/tables/2017/econ/economic-census/naics-sector23.html. """ def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018): self.sub_categories_list = {'Manufacturing': {'Food, Beverages, & Tobacco': None, 'Textile Mills and Products': None, 'Apparel & Leather': None, 'Wood Products': None, 'Paper': None, 'Printing & Allied Support': None, 'Petroleum & Coal Products': None, 'Chemicals': None, 'Plastics & Rubber Products': None, 'Nonmetallic Mineral Products': None, 'Primary Metals': None, 'Fabricated Metal Products': None, 'Machinery': None, 'Computer & Electronic Products': None, 'Electical Equip. & Appliances': None, 'Transportation Equipment': None, 'Furniture & Related Products': None, 'Miscellaneous': None}, 'Nonmanufacturing': {'Agriculture, Forestry & Fishing': None, 'Mining': {'Petroleum and Natural Gas': None, 'Other Mining': None, 'Petroleum drilling and Mining Services': None}, 'Construction': None}} self.ind_eia = GetEIAData('industry') self.conversion_factors = self.ind_eia.conversion_factors() self.MER_Nov19_Table24 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER10_Table21d = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER11_Table21d_MER0816 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.mer_dataT0204 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.BEA_Output_data = [0] # Chain-type Quantity Indexes for Value Added by Industry from Bureau of Economic Analysis self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] super().__init__(sector='industry', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \ energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year) def reconcile_physical_units(self, ): """Convert physical units to Btu. (Prior to 2005, the data on energy consumption fuels to produce electricity were supplied in physical units (e.g. mcf of natural gas, tons of coal, etc)) Data Source: EIA's Annual Energy Review (AER)""" pass def manufacturing(self): # categories = self.sub_categories_list['Manufacturing'] # manufacturing_data = Manufacturing().manufacturing() # print('manufacturing_data: \n', manufacturing_data) manufacturing_data = None return manufacturing_data def non_manufacturing(self): """Primary Data Sources: Economic Census (previously the Census of Manufactures, Census of Agriculture, and Census of Mining) Prior to 1985, primary data source is the National Energy Accounts (NEA) http://www.nass.usda.gov/Statistics_by_Subject/index.php """ # categories = self.sub_categories_list['Nonmanufacturing'] non_manufacturing_data = NonManufacturing().nonmanufacturing_data() print('non_manufacturing_data: \n', non_manufacturing_data) return non_manufacturing_data def collect_data(self): man = self.manufacturing() non_man = self.non_manufacturing() data_dict = {'Manufacturing': man, 'Nonmanufacturing': non_man} return data_dict def total_industrial_util_adj_lmdi(self): util_adj_categories = ['Fuels', 'Delivered Electricity', 'Source Electricity', 'Total Source'] # This case is quite different from the others return util_adj_categories def main(self, breakout, save_breakout, calculate_lmdi): unit_conversion_factor = 1 data_dict = self.collect_data() results_dict, formatted_results = self.get_nested_lmdi(level_of_aggregation=self.level_of_aggregation, breakout=breakout, save_breakout=save_breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, account_for_weather=False) return formatted_results
def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018, naics_digits=3): self.sub_categories_list = \ {'Industry': {'Manufacturing': {'Food and beverage and tobacco products': None, 'Textile mills and textile product mills': None, 'Apparel and leather and allied products': None, 'Wood products': None, 'Paper products': None, 'Printing and related support activities': None, 'Petroleum and coal products': None, 'Chemical products': None, 'Plastics and rubber products': None, 'Nonmetallic mineral products': None, 'Primary metals': None, 'Fabricated metal products': None, 'Machinery': None, 'Computer and electronic products': None, 'Electrical equipment, appliances, and components': None, 'Motor vehicles, bodies and trailers, and parts': None, 'Furniture and related products': None, 'Miscellaneous manufacturing': None}, 'Nonmanufacturing': {'Agriculture, Forestry & Fishing': None, 'Mining': {'Petroleum and Natural Gas': None, 'Other Mining': None, 'Support Activities': None}, 'Construction': None}}} self.ind_eia = GetEIAData('industry') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.MER_Nov19_Table24 = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER10_Table21d = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.AER11_Table21d_MER0816 = \ self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252' self.mer_dataT0204 = \ self.ind_eia.eia_api(id_='711252') self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj'] self.naics_digits = naics_digits super().__init__(sector='industry', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year, primary_activity='value_added')
class EmissionsDataExploration: """Class to visualize changes over time in Emissions and Emissions factors using EIA data """ def __init__(self): self.eia = GetEIAData('emissions') def eia_data(self, id_, label): """Make EIA API call Args: id_ (str): EIA API endpoint label (str): Label to use as column name in resulting df Returns: data (DataFrame): data resulting from API call """ data = self.eia.eia_api(id_=id_, id_type='series', new_name=label) return data def all_fuels_data(self): """Collect EIA CO2 Emissions data from all fuels data for each Sector Returns: all_data (dict): All CO2 emissions data Dictionary with {sector}_co2 as keys and dataframe as value """ commercial_co2 = 'EMISS.CO2-TOTV-CC-TO-US.A' electric_power_co2 = 'EMISS.CO2-TOTV-EC-TO-US.A' industrial_co2 = 'EMISS.CO2-TOTV-IC-TO-US.A' residential_co2 = 'EMISS.CO2-TOTV-RC-TO-US.A' transportation_co2 = 'EMISS.CO2-TOTV-TC-TO-US.A' sectors = { 'commercial_co2': commercial_co2, 'electric_power_co2': electric_power_co2, 'industrial_co2': industrial_co2, 'residential_co2': residential_co2, 'transportation_co2': transportation_co2 } all_data = { s: self.eia_data(id_=sectors[s], label='CO2 Emissions') for s in sectors } return all_data def all_fuels_all_sector_data(self): """Collect EIA CO2 Emissions data from all fuels and all sectors Returns: all_sector (DataFrame): CO2 Emissions data for all fuels and all sectors in the US """ all_fuels_all_sector = 'EMISS.CO2-TOTV-TT-TO-US.A' all_sector = self.eia_data(id_=all_fuels_all_sector, label='CO2 Emissions') return all_sector @staticmethod def lineplot(datasets, y_label): """Plot 'CO2 Emissions from All Fuels by Sector' data Args: datasets (dict): [description] y_label (str): Label to use for y-axis of resulting plot """ plt.style.use('seaborn-darkgrid') palette = plt.get_cmap('Set2') for i, label in enumerate(datasets.keys()): df = datasets[label] plt.plot(df.index, df, marker='', color=palette(i), linewidth=1, alpha=0.9, label=label) title = 'CO2 Emissions from All Fuels by Sector' plt.title(title, fontsize=12, fontweight=0) plt.xlabel('Year') plt.ylabel(y_label) plt.legend(loc=2, ncol=2) plt.show() def get_emissions_plots(self): """Collect CO2 Emisisons data and plot it """ sectors = self.all_fuels_data() # total = self.all_fuels_all_sector_data() # sectors['total'] = total self.lineplot(sectors, y_label='CO2 Emissions (Million Metric Tons)') def get_emissions_factors_plots(self): """Collect CO2 Emissions and Energy data by sector, calculate Emissions factors (CO2/Energy) and plot the results """ emissions = self.all_fuels_data() energy = self.economy_wide() sectors = [ 'commercial', 'industrial', 'residential', 'transportation', 'electric_power' ] emissions_factors = dict() for s in sectors: em = emissions[f'{s}_co2'] en = energy[f'{s}_energy'] em, en = df_utils().ensure_same_indices(em, en) factor = em.divide(en.values, axis=1) factor = factor.rename( columns={ 'CO2 Emissions': 'Million Metric Tons per Trillion Btu' }) emissions_factors[s] = factor self.lineplot(emissions_factors, y_label='Million Metric Tons CO2 per Trillion Btu') def economy_wide(self): """Collect Energy Consumption data for each sector from the EIA API Returns: all_data (dict): Dictionary with sectors as keys and df as values """ commercial_energy = 'TOTAL.TECCBUS.A' electric_power_energy = 'TOTAL.TXEIBUS.A' industrial_energy = 'TOTAL.TEICBUS.A' residential_energy = 'TOTAL.TERCBUS.A' transportation_energy = 'TOTAL.TEACBUS.A' sectors = { 'commercial_energy': commercial_energy, 'electric_power_energy': electric_power_energy, 'industrial_energy': industrial_energy, 'residential_energy': residential_energy, 'transportation_energy': transportation_energy } all_data = { s: self.eia_data(id_=sectors[s], label='Energy') for s in sectors } return all_data
def electricity_projections(): """Gather Commercial projections data activity: Million kWh energy: Energy Consumption Trillion Btu """ electricity_eia = GetEIAData('electricity') data_dict = {'Elec Power Sector': {'Electricity Only': {'Fossil Fuels': {'Coal': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_STC_NA_NA_QBTU.A', id_type='series', new_name='Coal')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_CL_NA_USA_BLNKWH.A', id_type='series', new_name='Coal')}, 'Petroleum': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_LFL_NA_NA_QBTU.A', id_type='series', new_name='Petroleum')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_PET_NA_USA_BLNKWH.A', id_type='series', new_name='Petroleum')}, 'Natural Gas': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NG_NA_NA_QBTU.A', id_type='series', new_name='Natural Gas')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_NG_NA_USA_BLNKWH.A', id_type='series', new_name='Natural Gas')}}, 'Nuclear': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NUC_NA_NA_QBTU.A', id_type='series', new_name='Nuclear')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_NUP_NA_USA_BLNKWH.A', id_type='series', new_name='Nuclear')}, # 'Hydro Electric': {'energy': '', # 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_HYD_CNV_NA_BLNKWH.A', id_type='series')}, 'Renewable': {'Wood': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_ELEP_NA_WBM_NA_NA_QBTU.A', id_type='series', new_name='Wood')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_WBM_NA_NA_BLNKWH.A', id_type='series', new_name='Wood')}, # This is wood and other biomass 'Waste': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NBMSW_NA_NA_QBTU.A', id_type='series', new_name='Waste'), 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_BGM_NA_NA_BLNKWH.A', id_type='series', new_name='Waste')}}, # This is Biogenic Municipal Waste # 'Solar': {'Solar Photovoltaic': {'energy': electricity_eia.eia_api(id_='', id_type='series'), # 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_SLR_PHTVL_NA_BLNKWH.A', id_type='series')}, # 'Solar Thermal': {'energy': electricity_eia.eia_api(id_='', id_type='series'), # 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_SLR_THERM_NA_BLNKWH.A', id_type='series')}}, 'Wind': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_ELEP_NA_OFW_NA_NA_QBTU.A', id_type='series', new_name='Wind')}, 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_WND_NA_NA_BLNKWH.A', id_type='series', new_name='Wind')}}}}} return data_dict
def __init__(self): self.eia = GetEIAData('emissions')
class TransportationIndicators(CalculateLMDI): """ Class to calculate Energy Intensity indicators for the U.S. Transportation Sector """ def __init__(self, directory, output_directory, level_of_aggregation, lmdi_model='multiplicative', tedb_date='04302020', base_year=1985, end_year=2018): self.transit_eia = GetEIAData('transportation') self.mer_table25_dec_2019 = \ self.transit_eia.eia_api(id_='711272', id_type='category') self.mer_table_43_nov2019 = \ self.transit_eia.eia_api(id_='711272', id_type='category') self.aer_2010_table_65 = \ self.transit_eia.eia_api(id_='711272', id_type='category') self.tedb_date = tedb_date self.energy_types = ['deliv'] self.sub_categories_list = \ {'All_Transportation': {'All_Passenger': {'Highway': {'Passenger Cars and Trucks': {'Passenger Car – SWB Vehicles': {'Passenger Car': None, 'SWB Vehicles': None}, 'Light Trucks – LWB Vehicles': {'Light Trucks': None, 'LWB Vehicles': None}, 'Motorcycles': None}, 'Buses': {'Urban Bus': None, 'Intercity Bus': None, 'School Bus': None}, 'Paratransit': None}, 'Rail': {'Urban Rail': {'Commuter Rail': None, 'Heavy Rail': None, 'Light Rail': None}, 'Intercity Rail': None}, 'Air': {'Commercial Carriers': None, 'General Aviation': None}}, 'All_Freight': {'Highway': {'Single-Unit Truck': None, 'Combination Truck': None}, 'Rail': None, 'Air': None, 'Waterborne': None, 'Pipeline': {'Oil Pipeline': None, 'Natural Gas Pipeline': None}}}} super().__init__(sector='transportation', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year, end_year=end_year, unit_conversion_factor=1000000) def import_tedb_data(self, table_number, skip_footer=None, skiprows=None, sheet_name=None, usecols=None, index_col=None): try: file_url = f'https://tedb.ornl.gov/wp-content/uploads/2020/04/Table{table_number}_{self.tedb_date}.xlsx' xls = pd.read_excel(file_url, skipfooter=skip_footer, skiprows=skiprows, sheet_name=sheet_name, usecols=usecols, index_col=index_col) return xls except urllib.error.HTTPError as e: print('error with table', table_number, e) raise e # def adjust_truck_freight(self, parameter_list): # """purpose # Parameters # ---------- # Returns # ------- # DOES THIS WORK ? dataframes # """ # gross_output = pd.read_excel('./EnergyIntensityIndicators/Industry/Data/BLS_BEA_Data.xlsx', sheet_name='BLS_Data_011920', index_col=0) # gross_output = gross_output.transpose() # Note: Gross output in million 2005 dollars from BLS database for their employment projections input-output model, # # PNNL spreadsheet: BLS_output_data.xlsx in folder BLS_Industry_Data) # vehicle_miles_fhwa_tvm1 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', header=4, index_col=0) # old_methodology_2007_extrapolated = gross_output.iloc[2007] / gross_output.iloc[2006] * vehicle_miles_fhwa_tvm1.iloc[2006, :] # old_series_scaled_to_new = vehicle_miles_fhwa_tvm1 * vehicle_miles_fhwa_tvm1.iloc[2007, :] / old_methodology_2007_extrapolated # return old_series_scaled_to_new def passenger_based_activity(self): """ Time series for the activity measures for passenger transportation 1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46. 1977-2017 American Public Transportation Association. 2019 Public Transportation Fact Book. Appendix A, Table 3, Pt. A Note: Transit bus does not include trolley bus because APTA did not not report electricity consumption for trolley buses separately for all years. https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/ Note: Series not continuous for transit bus between 2006 and 2007; estimation procedures changed for bus systems outside urban areas Note: Series not continuous between 1983 and 1984. """ # Historical data from PNNL passenger_based_activity = \ pd.read_csv( './EnergyIntensityIndicators/Transportation/passenger_based_activity.csv' ).set_index('Year').rename( columns={'Motor-cycles': 'Motorcycles', 'Light Truck': 'Light Trucks', 'Transit Bus': 'Urban Bus', 'Para-Transit': 'Paratransit', 'All Carriers (pass miles), millions': 'Commercial Carriers'} ) # # Passenger cars and light trucks # fha_table_vm1 = pd.read_excel( # 'https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', # header=4, index_col=0 # ) # # Bus / Transit # apta_table3 = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='3', skiprows=7, skipfooter=42, index_col=0 # ) # 1997-2017 apta_table3 # # Bus / Intercity: see revised_intercity_bus_estimates # # Bus / School: see revised_intercity_bus_estimates # # Paratransit # paratransit_activity_1984_2017 = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='3', skiprows=7, skipfooter=42, usecols='A,G', index_col=0 # ) # 1997-2017 apta_table3 # # Commercial Air Carrier # commercial_air_carrier = pd.read_excel( # 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_02_01312021.xlsx', # skiprows=9, skip_footer=42, index_col=0, use_cols='B:D' # ) # # Urban Rail (Commuter) # urban_rail_commuter = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='3', skiprows=7, skipfooter=42, index_col=0, # use_cols='A,L') # 1997-2017 apta_table3 # # Urban Rail (Light, Heavy) # urban_rail_light_heavy = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='3', skiprows=7, skipfooter=42, usecols='A,O,P', index_col=0 # ) # 1997-2017 apta_table3 # # Intercity Rail (Amtrak) Note: Amtrak data is compiled by fiscal year rather than calendar year # intercity_rail_2017 = pd.read_excel( # 'https://www.bts.gov/sites/bts.dot.gov/files/table_01_40_091820.xlsx', # skiprows=1, skipfooter=25, index_col=0 # ) # intercity_rail_2017 = intercity_rail_2017.transpose() # intercity_rail_2017 = intercity_rail_2017[['Intercity/Amtraki']] # not sure how the i superscript shows up # 'Bureau of Transportation Statistics, National Transportation Statistics, Table 1-40: U.S. Passenger-Miles return passenger_based_activity def passenger_based_energy_use(self): """Calculate the fuel consumption in Btu for passenger transportation """ passenger_based_energy_use = pd.read_csv( './EnergyIntensityIndicators/Transportation/passenger_based_energy_use.csv' ).set_index('Year').rename( columns={ 'Light Truck': 'Light Trucks', 'Para-transit': 'Paratransit', 'Intercity Rail (Amtrak)': 'Intercity Rail' }) passenger_based_energy_use['Paratransit'] = \ passenger_based_energy_use['Paratransit'].fillna(0.000000001) return passenger_based_energy_use def fuel_heat_content(self, gross=True): """Assumed Btu content of the various types of petroleum products. This dataframe is not time variant (no need to update it with subsquent indicator updates) Parameters ---------- gross : bool if True use gross fuel heat content, if False use net """ fuel_heat_content = pd.read_csv( './EnergyIntensityIndicators/Transportation/Data/fuel_heat_content.csv' ) if gross: fuel_heat_content_df = fuel_heat_content[[ 'Fuel', 'Gross Content Fmt' ]] else: fuel_heat_content_df = fuel_heat_content[['Fuel', 'Net Content']] return fuel_heat_content_df # def fuel_consump(self, parameter_list): # """ # Time series of fuel consumption for the major transportation # subsectors. Data are generally in # millions gallons or barrels of petroleum. # Parameters # ---------- # Returns # ------- # """ # fha_table_vm1 = pd.read_excel( # 'https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', # header=4, index_col=0 # ) # 2007-2017 FHA Highway Statistics Table VM-1 # apta_table59 = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='59', skiprows=2, skipfooter=24, index_col=0 # ) # 1997-2017 apta_table3 # apta_table60 = pd.read_excel( # 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', # sheet_name='60', skiprows=2, skipfooter=24, index_col=0 # ) # 1997-2017 apta_table3 # # swb_vehciles_all_fuel = fha_table_vm1 # # motorcycles_all_fuel_1970_2017 = self.import_tedb_data(table_number='A_02') # Alternative: 2017 data from Highway Statistics, Table VM-1. # # lwb_vehicles_all_fuel = fha_table_vm1 # 2007-2017 FHA Highway Statistics Table VM-1 # # bus_urban_diesel = apta_table59 #[] #Table 59 in Appendix A, Bus Fuel Consumption (downloaded Excel file) # # from: https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/ # # #Excel file: 2019-APTA-Fact-Book-Appendix-A.xlsx # # bus_urban_cng = apta_table59 #[] #same as bus_urban_diesel # # bus_urban_lng = apta_table59 #[] #same as bus_urban_diesel # # bus_urban_other = apta_table59 #[] #same as bus_urban_diesel # # bus_urban_lpg = apta_table59 #[] #same as bus_urban_diesel # # paratransit_diesel = apta_table60 #[] # http://www.apta.com/resources/statistics/Pages/transitstats.aspx # # # https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/ # # Table 60 in Appendix A, Demand Response Fuel Consumption # # general_aviation = self.import_tedb_data(table_number='A_08') # # commercial_air_carrier = self.import_tedb_data(table_number='A_09') # # rail_commuter = self.import_tedb_data(table_number='A_13') # # rail_urban = self.import_tedb_data(table_number='A_15') # # class_I_freight_distillate_fuel_oil = self.import_tedb_data(table_number='A_13') # # pipeline_natural_gas_million_cu_ft = self.import_tedb_data(table_number='A_12') # fuel_consump_df = [] # return fuel_consump_df def freight_based_energy_use(self, freight_based_activity): """Calculate the fuel consumption in Btu for freight transportation Need FuelConsump, Fuel Heat Content """ # Historical data from PNNL freight_based_energy_use = pd.read_csv( './EnergyIntensityIndicators/Transportation/' + 'freight_based_energy_use.csv').set_index('Year') # This was used by PNNL to negate oil pipeline data (no reliable data source) freight_based_energy_use['Oil Pipeline'] = \ freight_based_activity['Oil Pipeline'].multiply(0.000001) # natural_gas_delivered_to_end_users = \ # self.aer_2010_table_65 # Column AH, million cu. ft. # natural_gas_delivered_lease_plant_pipeline_fuel = \ # self.mer_table_43_nov2019 # Column M - column D - column I # natural_gas_delivered_lease_plant_pipeline_fuel.at[0] = 0.000022395 # natural_gas_consumption_million_tons = \ # natural_gas_delivered_to_end_users.multiply( # natural_gas_delivered_lease_plant_pipeline_fuel, axis=1) # avg_length_natural_gas_shipment_miles = 620 # freight_based_energy_use['Natural Gas Pipeline'] = \ # natural_gas_consumption_million_tons.multiply( # avg_length_natural_gas_shipment_miles # ) return freight_based_energy_use def freight_based_activity(self): """Time series for the activity measures for passenger transportation """ # Historical data, from PNNL freight_based_activity = \ pd.read_csv('./EnergyIntensityIndicators/Transportation/' + 'freight_based_activity.csv').set_index('Year') freight_based_activity['Single-Unit Truck'] = \ freight_based_activity['Single-Unit Truck'].multiply(3) # Assumes 3-ton avg. load freight_based_activity['Oil Pipeline'] = \ freight_based_activity['Oil Pipeline'].multiply(0.000001) # USDOT, Bureau of Transportation Statistics # https://www.bts.gov/content/energy-intensity-class-i-railroad-freight-service Table 4-25 # class_1_rail = pd.read_excel('https://www.bts.gov/sites/bts.dot.gov/files/table_04_25_112219.xlsx', # skiprows=1, skipfooter=9, index_col=0) # class_1_rail = class_1_rail.transpose() # class_1_rail = class_1_rail[['Revenue freight ton-miles (millions)']] # air_carrier = \ # pd.read_excel( # 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_02_01312021.xlsx', # skiprows=9, skip_footer=42, index_col=1, use_cols='B,H') # waterborne_vessles = \ # pd.read_excel( # 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_05_01312021.xlsx', # skiprows=9, skip_footer=42, index_col=1, use_cols='B,D') # gas_pipeline = self.mer_table_43_nov2019 return freight_based_activity # def water_freight_regression(self, intensity, actual_dd): # """[summary] # Args: # intensity ([type]): [description] # actual_dd ([type]): [description] # Returns: # predicted_value ([type]): [description] # """ # X = np.log(intensity['intensity']) # Y = intensity['Year'] # reg = linear_model.LinearRegression() # reg.fit(X, Y) # coefficients = reg.coef_ # predicted_value = reg.predict(actual_dd) # Predicted value of the intensity based on actual degree days # return predicted_value # def detailed_data_school_buses(self, parameter_list): # """ # Adjusted miles - use number of buses times # interpolated values for annual miles per bus. # For years ending 0 and 5, use published data. # Thus, adjusted values match in years ending 0 and 5 with Column 4, # and for 1994. After 1994, values taken directly from # School Bus Fleet Magazine estimates, adjusted for missing values # Args: # parameter_list ([type]): [description] # Returns: # revised_pass_miles (pd.DataFrame): [description] # """ # avg_load_factor = 23 # adjusted_vehicle_miles = pd.read_csv( # './EnergyIntensityIndicators/Transportation/Data/adjusted_data_school_bus.csv' # ) # revised_pass_miles = adjusted_vehicle_miles.multiply(avg_load_factor * 0.001) # return revised_pass_miles def collect_data(self): """Method to collect freight and passenger energy and activity data. This method should be adjusted to call dataframe building methods rather than reading csvs, when those are ready Gather Energy Data to use in LMDI Calculation TBtu and Activity Data to use in LMDI Calculation passenger-miles [P-M], ton-miles [T-M] Returns ------- data_dict : dict """ passenger_based_energy_use = self.passenger_based_energy_use() passenger_based_activity = self.passenger_based_activity() freight_based_activity = self.freight_based_activity() freight_based_energy_use = \ self.freight_based_energy_use(freight_based_activity) data_dict = { 'All_Passenger': { 'Highway': { 'Passenger Cars and Trucks': { 'Passenger Car – SWB Vehicles': { 'Passenger Car': { 'energy': { 'deliv': passenger_based_energy_use[[ 'Passenger Car' ]] }, 'activity': passenger_based_activity[['Passenger Car']] }, 'SWB Vehicles': { 'energy': { 'deliv': passenger_based_energy_use[[ 'SWB Vehicles' ]] }, 'activity': passenger_based_activity[['SWB Vehicles']] } }, 'Light Trucks – LWB Vehicles': { 'Light Trucks': { 'energy': { 'deliv': passenger_based_energy_use[[ 'Light Trucks' ]] }, 'activity': passenger_based_activity[['Light Trucks']] }, 'LWB Vehicles': { 'energy': { 'deliv': passenger_based_energy_use[[ 'LWB Vehicles' ]] }, 'activity': passenger_based_activity[['LWB Vehicles']] } }, 'Motorcycles': { 'energy': { 'deliv': passenger_based_energy_use[['Motorcycles']] }, 'activity': passenger_based_activity[['Motorcycles']] } }, 'Buses': { 'Urban Bus': { 'energy': { 'deliv': passenger_based_energy_use[['Urban Bus']] }, 'activity': passenger_based_activity[['Urban Bus']] }, 'Intercity Bus': { 'energy': { 'deliv': passenger_based_energy_use[['Intercity Bus']] }, 'activity': passenger_based_activity[['Intercity Bus']] }, 'School Bus': { 'energy': { 'deliv': passenger_based_energy_use[['School Bus']] }, 'activity': passenger_based_activity[['School Bus']] } }, 'Paratransit': { 'energy': { 'deliv': passenger_based_energy_use[['Paratransit']] }, 'activity': passenger_based_activity[['Paratransit']] } }, 'Rail': { 'Urban Rail': { 'Commuter Rail': { 'energy': { 'deliv': passenger_based_energy_use[['Commuter Rail']] }, 'activity': passenger_based_activity[['Commuter Rail']] }, 'Heavy Rail': { 'energy': { 'deliv': passenger_based_energy_use[['Heavy Rail']] }, 'activity': passenger_based_activity[['Heavy Rail']] }, 'Light Rail': { 'energy': { 'deliv': passenger_based_energy_use[['Light Rail']] }, 'activity': passenger_based_activity[['Light Rail']] } }, 'Intercity Rail': { 'energy': { 'deliv': passenger_based_energy_use[['Intercity Rail']] }, 'activity': passenger_based_activity[['Intercity Rail']] } }, 'Air': { 'Commercial Carriers': { 'energy': { 'deliv': passenger_based_energy_use[['Carrier']] }, 'activity': passenger_based_activity[['Commercial Carriers']] }, 'General Aviation': { 'energy': { 'deliv': passenger_based_energy_use[['General Aviation']] }, 'activity': passenger_based_activity[['General Aviation']] } } }, 'All_Freight': { 'Highway': { 'Single-Unit Truck': { 'energy': { 'deliv': freight_based_energy_use[['Single-Unit Truck']] }, 'activity': freight_based_activity[['Single-Unit Truck']] }, 'Combination Truck': { 'energy': { 'deliv': freight_based_energy_use[['Combination Truck']] }, 'activity': freight_based_activity[['Combination Truck']] } }, 'Rail': { 'energy': { 'deliv': freight_based_energy_use[['Rail']] }, 'activity': freight_based_activity[['Rail']] }, 'Air': { 'energy': { 'deliv': freight_based_energy_use[['Air']] }, 'activity': freight_based_activity[['Air']] }, 'Waterborne': { 'energy': { 'deliv': freight_based_energy_use[['Waterborne']] }, 'activity': freight_based_activity[['Waterborne']] }, 'Pipeline': { 'Oil Pipeline': { 'energy': { 'deliv': freight_based_energy_use[['Oil Pipeline']] }, 'activity': freight_based_activity[['Oil Pipeline']] }, 'Natural Gas Pipeline': { 'energy': { 'deliv': freight_based_energy_use[['Natural Gas Pipeline']] }, 'activity': freight_based_activity[['Natural Gas Pipeline']] } } } } data_dict = {'All_Transportation': data_dict} return data_dict def main(self, breakout, calculate_lmdi): # base_year=None, """Decompose Energy use for the Transportation sector """ data_dict = self.collect_data() if self.level_of_aggregation == 'personal_vehicles_aggregate': categories = { 'Passenger Car': None, 'Light Truck': None, 'Motorcycles': None } results = None else: results_dict, results = self.get_nested_lmdi( level_of_aggregation=self.level_of_aggregation, breakout=breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, lmdi_type='LMDI-I') return results_dict