def commercial_projections(self):
"""activity: floorspace
energy: consumption trillion Btu
"""
# Commercial : Total Floorspace : New Additions, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_NADN_USA_BLNSQFT.A'
# Commercial : Total Floorspace : Surviving, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_SURV_USA_BLNSQFT.A'
# Commercial : Total Floorspace : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_TOT_USA_BLNSQFT.A'
commercial_categories = {'Commercial_Total': None, 'Total_Commercial_LMDI_UtilAdj': None}
commercial_eia = GetEIAData('commercial')
energy_use_commercial_electricity_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_ELC_NA_NA_QBTU.A', id_type='series')
energy_use_commercial_total_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_TOT_NA_NA_QBTU.A', id_type='series')
energy_use_commercial_delivered_energy_us = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_COMM_NA_DELE_NA_NA_QBTU.A', id_type='series')
commercial_total_floorspace = commercial_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_NA_COMM_NA_TFP_TOT_USA_BLNSQFT.A', id_type='series')
data_dict = {'Commercial_Total': {'energy': {'elec': energy_use_commercial_electricity_us, 'deliv': energy_use_commercial_delivered_energy_us},
'activity': commercial_total_floorspace},
'Total_Commercial_LMDI_UtilAdj': None}
return data_dict
def residential_projections(self):
"""Gather Residential projections data
activity:
- Occupied Housing Units
- Floorspace, Square Feet
energy: Energy Consumption Trillion Btu
"""
# Residential : Key Indicators : Average House Square Footage, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_NA_NA_NA_USA_SQFT.A'
# Residential : Key Indicators : Households : Mobile Homes, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_MBH_NA_NA_USA_MILL.A'
# Residential : Key Indicators : Households : Multifamily, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_MFR_NA_NA_USA_MILL.A'
# Residential : Key Indicators : Households : Single-Family, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_SFR_NA_NA_USA_MILL.A'
# Residential : Key Indicators : Households : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.KEI_HHD_RESD_TEN_NA_NA_USA_MILL.A'
self.sub_regions_dict = {'Northeast': {'New England': 'NENGL', 'Middle Atlantic': 'MATL'}, 'Midwest': {'East North Central': 'ENC', 'West North Central': 'WNC'},
'South': {'South Atlantic': 'SATL', 'East South Central': 'ESC', 'West South Central': 'WSC'}, 'West': {'Mountain': 'MTN', 'Pacific': 'PCF'}}
residential_eia = GetEIAData('residential')
energy_use_residential_electricity_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_ELC_NA_NA_QBTU.A', id_type='series')
energy_use_residential_total_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_TOT_NA_NA_QBTU.A', id_type='series')
energy_use_residential_delivered_energy_us = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_DELE_NA_NA_QBTU.A', id_type='series')
res_housholds= residential_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_HHD_RESD_TEN_NA_NA_USA_MILL.A', id_type='series', new_name='Residential')
res_avg_size = residential_eia.eia_api(id_='AEO.2020.AEO2019REF.KEI_HHD_RESD_NA_NA_NA_USA_SQFT.A', id_type='series', new_name='Residential')
residential_floorspace = res_avg_size.multiply(res_housholds.values)
activity_data = {'households': res_housholds,
'average_size': res_avg_size,
'total_floorspace': residential_floorspace}
energy_data = {'elec': residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Residential'),
'deliv': residential_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_RESD_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Residential')}
data_dict = {'energy': energy_data, 'activity': activity_data}
return data_dict
class CO2EmissionsDecomposition(CalculateLMDI):
"""Class to decompose CO2 emissions by
sector of the U.S. economy.
LMDI aspects:
- total activity (Q),
- activity share for subsector i (structure) (=Q_i/Q),
- energy intensity for subsector i (=E_i/Q_i),
- energy share of type j in subsector i (=E_ij/E_i),
also called fuel mix
- emissions rate of energy type j and subsector i (=C_ij/E_ij),
also called emissions coefficient
(for time series, all variables have t subscripts
(i.e. no constants-- constant emissions rates cancel out))
"""
def __init__(self, directory, output_directory, sector, config_path,
categories_dict, level_of_aggregation):
self.sector = sector
self.eia = GetEIAData('emissions')
self.config_path = config_path
self.level_of_aggregation = level_of_aggregation
super().__init__(sector,
level_of_aggregation=self.level_of_aggregation,
categories_dict=categories_dict,
directory=directory,
output_directory=output_directory,
primary_activity=None,
unit_conversion_factor=1,
weather_activity=None,
use_yaml_config=True,
config_path=self.config_path)
def collect_emissions(self,
sector=None,
fuel_type=None,
state_abbrev=None):
"""Collect emissions from the EIA API (through
GetEIAData).
Args:
sector (str, optional): Economic sector. Defaults to None.
fuel_type (str, optional): Fuel Type. Defaults to None.
state_abbrev (str, optional): State (e.g. 'AK'). Defaults to None.
Returns:
data (DataFrame): Emissions data for sector, fuel type and state
"""
eia_co2_emiss = f'EMISS.CO2-TOTV-{sector}-{fuel_type}-{state_abbrev}.A'
data = self.eia.eia_api(id_=eia_co2_emiss, id_type='series')
return data
@staticmethod
def get_fuel_mix(region_data):
"""Calculate shares of total fuel by fuel type
Args:
region_data (DataFrame): Fuel use data by fuel for a region
Returns:
fuel_mix (DataFrame): Fuel mix (i.e. share of total by fuel)
"""
region_data = region_data.drop('Census Region',
axis=1,
errors='ignore')
region_data = df_utils().create_total_column(region_data,
total_label='total')
fuel_mix = \
df_utils().calculate_shares(region_data, total_label='total')
return fuel_mix
@staticmethod
def weighted(x, cols, w="weights"):
"""Calculate weighted average
Args:
x ([type]): [description]
cols ([type]): [description]
w (str, optional): [description]. Defaults to "weights".
Returns:
(pd.Series): Weighted average
"""
return pd.Series(np.average(x[cols], weights=x[w], axis=0), cols)
def get_mean_factor(self,
emissions_factors,
input_cols,
new_name,
portions=None):
"""Calculate average of given emissions factors
Args:
emissions_factors (pd.DataFrame): emissions factors
input_cols (list): List of emissions
factors to average
new_name (str): Name of resulting factor
portions (dict): Optional. Weights for weighted average
(keys are cols)
Returns:
ef (pd.DataFrame): emissions factors with new type
"""
subset = \
emissions_factors[emissions_factors['Fuel Type'].isin(input_cols)]
subset['weights'] = np.nan
if portions:
subset_ = []
for k, v in portions.items():
fuel_ = subset[subset['Fuel Type'] == k]
fuel_['weights'] = v
subset_.append(fuel_)
subset = pd.concat(subset_, axis=0)
subset.loc[:, 'weighted_value'] = \
subset['value'].multiply(subset['weights'].values)
grouped = \
subset.groupby(by=['Unit', 'Variable'])
mean_df = grouped.sum()
mean_df = mean_df.drop('value', axis=1)
mean_df['value'] = \
mean_df['weighted_value'].divide(mean_df['weights'].values)
mean_df = mean_df.drop('weighted_value', axis=1)
else:
grouped = \
subset.groupby(by=['Unit', 'Variable'])
mean_df = grouped.mean()
mean_df.loc[:, 'Fuel Type'] = new_name
mean_df.loc[:, 'Category'] = 'Merged Category'
mean_df = mean_df.reset_index()
mean_df = mean_df[[
'Category', 'Fuel Type', 'Unit', 'value', 'Variable'
]]
ef = pd.concat([emissions_factors, mean_df], axis=0)
return ef
def epa_emissions_data(self):
"""Read and process EPA emissions factors data
Returns:
emissions_factors (DataFrame): EPA emissions factors data
"""
try:
ef = pd.read_csv(
'./EnergyIntensityIndicators/Data/EPA_emissions_factors.csv')
except FileNotFoundError:
os.chdir('..')
print('changed dir:', os.getcwd())
ef = pd.read_csv(
'./EnergyIntensityIndicators/Data/EPA_emissions_factors.csv')
df_cols = ef.columns
dfs = []
grouped = ef.groupby(ef['Unit Type'])
for g in ef['Unit Type'].unique():
unit_data = grouped.get_group(g)
unit_data.columns = unit_data.iloc[0]
units_dict = dict(zip(unit_data.columns, df_cols))
unit_data = unit_data.drop(g, axis=1)
unit_data = unit_data.drop(unit_data.index[0])
unit_data = unit_data.melt(id_vars=['Units', 'Fuel Type'],
var_name='Unit')
unit_data = unit_data.rename(columns={'Units': 'Category'})
unit_data.loc[:, 'Variable'] = unit_data['Unit'].map(units_dict)
dfs.append(unit_data)
emissions_factors = pd.concat(dfs, axis=0)
emissions_factors['value'] = \
emissions_factors['value'].apply(lambda x: str(x).replace(',', ''))
emissions_factors['value'] = emissions_factors['value'].astype(float)
ef = self.get_mean_factor(emissions_factors,
input_cols=[
'Distillate Fuel Oil No. 1',
'Distillate Fuel Oil No. 2',
'Distillate Fuel Oil No. 4'
],
new_name='Distillate Fuel Oil')
ef = self.get_mean_factor(
ef,
input_cols=['Residual Fuel Oil No. 5', 'Residual Fuel Oil No. 6'],
new_name='Residual Fuel Oil')
ef = self.get_mean_factor(
ef,
input_cols=['Blast Furnace Gas', 'Coke Oven Gas'],
new_name='Blast Furnace/Coke Oven Gases')
ef = self.get_mean_factor(
ef,
input_cols=['North American Softwood', 'North American Hardwood'],
new_name='Pulping Liquor or Black Liquor')
ef = self.get_mean_factor(
ef,
input_cols=['Motor Gasoline', 'Ethanol (100%)'],
new_name='Gasohol',
portions={
'Ethanol (100%)': 0.16,
'Motor Gasoline': 0.84
})
ef = self.get_mean_factor(ef,
input_cols=['Diesel Fuel', 'Motor Gasoline'],
new_name='School',
portions={
'Diesel Fuel': 0.9,
'Motor Gasoline': 0.1
})
ef = self.get_mean_factor(ef,
input_cols=[
'Diesel Fuel',
'Distillate Fuel Oil No. 1',
'Distillate Fuel Oil No. 2',
'Distillate Fuel Oil No. 4'
],
new_name='Diesel Fuel & Distillate')
ef = self.get_mean_factor(ef,
input_cols=[
'Distillate Fuel Oil No. 1',
'Distillate Fuel Oil No. 2',
'Distillate Fuel Oil No. 4',
'Residual Fuel Oil No. 5',
'Residual Fuel Oil No. 6'
],
new_name='Petroleum')
return ef
@staticmethod
def mecs_epa_mapping(mecs_data):
"""Rename mecs_data columns so that labels match
EPA emissions factors labels
Args:
mecs_data (DataFrame): Industrial sector energy use
by fuel type
Returns:
mecs_data (DataFrame): MECS data with column names
that match EPA emissions data
labels
"""
rename_dict = {col: col.strip() for col in mecs_data.columns}
mecs_data = mecs_data.rename(columns=rename_dict)
mapping_ = {
'Waste Gas': 'Fuel Gas',
'Petroleum Coke': 'Petroleum Coke',
'Wood Chips, Bark': 'Wood and Wood Residuals',
'Waste Oils/Tars and Waste Materials': 'Used Oil',
'steam': 'Steam and Heat', # From Table 7
'Net Electricity':
'Us Average', # From Table 6, Total Output Emissions Factors CO2 Factor
'Electricity': 'US Average',
'Net Electricity(b)': 'US Average',
'Residual': 'Residual Fuel Oil',
'Distillate': 'Distillate Fuel Oil',
'Distillate Fuel Oil(c)': 'Distillate Fuel Oil',
'Nat. Gas': 'Natural Gas',
'Natural Gas(d)': 'Natural Gas',
'Natural Gas': 'Natural Gas',
'HGL (excluding natural gasoline)':
'Liquefied Petroleum Gases (LPG)',
'Coal': 'Mixed (Industrial Sector)',
'Coke Coal and Breeze': 'Coal Coke',
'Coke and Breeze': 'Coal Coke',
'Coke': 'Coal Coke',
'LPG': 'Liquefied Petroleum Gases (LPG)',
'Diesel': 'Diesel Fuel',
'LP Gas': 'Liquefied Petroleum Gases (LPG)',
'HGL (excluding natural gasoline)(e)':
'Liquefied Petroleum Gases (LPG)',
'Gasoline': 'Motor Gasoline',
'Gas': 'Natural Gas'
}
mecs_data = mecs_data.rename(columns=mapping_)
mecs_data = mecs_data.drop('Total Fuel', axis=1, errors='ignore')
return mecs_data
@staticmethod
def electric_epa_mapping(elec_data):
"""Rename elec power sector data (from EIA) columns so
that labels match EPA emissions factors labels
Args:
elec_data (pd.DataFrame): Energy consumption for the
elec power sector by fuel type
Returns:
elec_data (pd.DataFrame): Elec data with column names
that match EPA emissions data
labels
"""
rename_dict = {
col: col[:col.find('Consumption')].strip()
for col in elec_data.columns if 'Consumption' in col
}
rename_dict2 = {
col: col[:col.find('Consumed')].strip()
for col in elec_data.columns if 'Consumed' in col
}
intersect = []
for key, value in rename_dict2.items():
if value in rename_dict.values():
intersect.append(key)
elec_data = elec_data.drop(intersect, axis=1)
rename_dict.update(rename_dict2)
others = {
'Electricity Net Generation From Wood, Electric Power Sector, Annual, Million Kilowatthours':
'Wood',
'Electricity Net Generation From Waste, Electric Power Sector, Annual, Million Kilowatthours':
'Waste'
}
rename_dict.update(others)
elec_data = elec_data.rename(columns=rename_dict)
mapping_ = {
'Coal': 'Mixed (Electric Power Sector)',
'Natural Gas': 'Natural Gas',
'Other Gases': 'Fuel Gas',
'Other Gas': 'Fuel Gas',
'Waste': 'Municipal Solid Waste',
'Wood': 'Wood and Wood Residuals',
'hydroelectric': 'Hydroelectric',
'Other': 'US Average',
'Other Petroleum Liquids': 'Liquefied Petroleum Gases (LPG)'
} # might be wrong
elec_data = elec_data.rename(columns=mapping_)
elec_data = elec_data.drop('Total Petroleum', axis=1, errors='ignore')
return elec_data
@staticmethod
def tedb_epa_mapping(tedb_data):
"""Rename transportation sector data (from TEDB) columns so
that labels match EPA emissions factors labels
Args:
tedb_data (pd.DataFrame): Energy consumption for the
transportation sector by fuel type
Returns:
tedb_data_ (pd.DataFrame): transportation data with
column names that match
EPA emissions data labels
"""
unit_coversion = {
'Diesel Fuel & Distillate (1,000 bbl)': 1000 / 42,
'Residual Fuel Oil (1,000 bbl)': 1000 / 42
}
unit_coversion = {
k: unit_coversion[k]
for k in unit_coversion.keys() if k in tedb_data.columns
}
if len(unit_coversion) > 0:
tedb_data = tedb_data.assign(**unit_coversion).mul(tedb_data)
if 'Domestic Operations ' and 'International Operations ' in tedb_data.columns:
tedb_data['Air'] = \
tedb_data[['Domestic Operations ',
'International Operations ']].sum(axis=1)
tedb_data = tedb_data[['Air']]
mapping = {
'Gasoline': 'Motor Gasoline', # ef is in gallon
'Gasoline (million gallons)': 'Motor Gasoline', # ef is in gallon
'Gasohol': 'Gasohol', # ef is in gallon
'Diesel (million gallons)': 'Diesel Fuel', # ef is in gallon
'Diesel': 'Diesel Fuel', # ef is in gallon
'CNG': 'Compressed Natural Gas (CNG)', # ef is in scf
'LNG': 'Liquefied Natural Gas (LNG)', # ef is in gallons
'Bio Diesel ': 'Biodiesel (100%)', # ef is in gallons
'Diesel Fuel & Distillate (1,000 bbl)': # ef is in gallon (42 gallons in a barrel)
'Diesel Fuel',
'Residual Fuel Oil (1,000 bbl)':
'Residual Fuel Oil', # ef is in gallon (42 gallons in a barrel)
'Jet fuel (million gallons)':
'Aviation Gasoline', # ef is per gallon
'Electricity (GWhrs)': 'US Average', # ef is kg/MWh
'Distillate Fuel Oil': 'Diesel Fuel', # ef is per gallon
'Natural Gas (million cu. ft.)': 'Natural Gas', # ef is per scf
'Electricity (million kWh)': 'US Average', # ef is kg/MWh
'Diesel fuel': 'Diesel Fuel', # ef is per gallon
'Liquefied petroleum gas':
'Liquefied Petroleum Gases (LPG)', # ef is per gallon
'LPG': 'Liquefied Petroleum Gases (LPG)', # ef is per gallon
'Air': 'Aviation Gasoline', # ef is per gallon
'Jet fuel': 'Aviation Gasoline', # ef is per gallon
'Residual fuel oil': 'Residual Fuel Oil', # ef is per gallon
'Natural gas': 'Natural Gas', # ef is per scf
'Electricity': 'US Average', # ef is kg/MWh
'Intercity': 'Diesel Fuel' # ef is in gallon
}
tedb_data_ = tedb_data.rename(columns=mapping)
tedb_data_ = tedb_data_.drop('School (million bbl)',
axis=1,
errors='ignore')
tedb_data_ = \
tedb_data_.drop(['Total Energy (Tbtu)',
'Total Energy (Tbtu) ',
'Total Energy (Tbtu) - old series'],
axis=1, errors='ignore')
return tedb_data_
@staticmethod
def get_factor(factors_df, emissions_type):
"""Lookup emissions factor for given fuel and emissions
type
Args:
factors_df (df): EPA emisions hub data
fuel_name (str): Fuel type to look up in factors_df
emissions_type (str): Type of emissions to return (e.g CO2)
Returns:
emissions_factor (float): emissions factor for given params
"""
factors_df = \
factors_df[factors_df['Variable'].isin(
['Heat Content (HHV)', emissions_type])]
factors_df['value'] = factors_df['value'].fillna(1)
factors_df = \
factors_df.groupby(['Category', 'Fuel Type']).prod()
factors_df = factors_df.reset_index()
fuel_factor_df = factors_df[['Fuel Type', 'value']]
fuel_factor_df.loc[:, 'value'] = fuel_factor_df['value'].astype(float)
new_row = {'Fuel Type': 'Census Region', 'value': 1}
fuel_factor_df = fuel_factor_df.append(new_row, ignore_index=True)
fuel_factor_df = fuel_factor_df.set_index('Fuel Type')
no_emissions = [
'Solar', 'Wind', 'Nuclear', 'Geothermal', 'Hydroelectric'
]
e_data = [0] * len(no_emissions)
no_emissions_df = pd.DataFrame(data=e_data,
index=no_emissions,
columns=['value'])
no_emissions_df.index.name = 'Fuel Type'
fuel_factor_df = pd.concat([fuel_factor_df, no_emissions_df], axis=0)
fuel_factor_df = fuel_factor_df.transpose()
return fuel_factor_df
def calculate_emissions(self,
energy_data,
emissions_type='CO2 Factor',
datasource='SEDS'):
"""Calculate emissions from the product of energy_data and
emissions_factor
Parameters:
energy_data (pd.DataFrame): energy consumption by fuel data
emissions_type (str): Type of emissions factor.
Defaults to 'CO2 Factor'.
datasource (str): 'SEDS', 'MECS', 'eia_elec', or 'TEDB'.
Data source for energy consumption by
fuel data. Defaults to 'SEDS'.
Returns:
emissions_data (pd.DataFrame): Emissions by fuel type
energy_data (pd.DataFrame): Energy consumption by fuel data
converted to MMBtu, if necessary,
with columns matching emissions data
TODO: Handle Other category (should not be dropped)
"""
# Also need to convert TEDB fuel (in gal or ft3) to MMBtu
mapping_heat = {
'Motor Gasoline': 125000, # ef is in gallon
'Gasohol': 120900, # ef is in gallon
'Diesel Fuel': 138700, # ef is in gallon
'Compressed Natural Gas (CNG)': 129400, # ef is in scf
'Liquefied Natural Gas (LNG)': 84800, # ef is in gallons
'Liquefied Petroleum Gases (LPG)': 91300,
'Biodiesel (100%)': 128520, # ef is in gallons
'Residual Fuel Oil':
149700, # ef is in gallon (42 gallons in a barrel)
'Aviation Gasoline': 120900, # ef is per gallon
'US Average': 10339, # ef is kg/MWh; Btu/kWh
'Natural Gas': 1031, # ef is per scf
'Diesel Fuel': 138700, # ef is per gallon
}
energy_data = energy_data.drop('region', axis=1, errors='ignore')
emissions_factors = self.epa_emissions_data()
if datasource == 'SEDS':
energy_data = self.epa_eia_crosswalk(energy_data)
elif datasource == 'MECS':
energy_data = self.mecs_epa_mapping(energy_data)
elif datasource == 'eia_elec':
energy_data = self.electric_epa_mapping(energy_data)
elif datasource == 'TEDB':
energy_data = self.tedb_epa_mapping(energy_data)
energy_data = energy_data.drop('Total', axis=1, errors='ignore')
## TEMPORARY!! (need to add these factors to csv)
energy_data = energy_data.drop(['Other'], axis=1, errors='ignore')
emissions_factors = self.get_factor(emissions_factors, emissions_type)
try:
emissions_factors = \
emissions_factors[energy_data.columns.tolist()]
except KeyError:
print('energy_data.columns.tolist() not in dataframe:',
energy_data.columns.tolist())
for t in energy_data.columns.tolist():
if t not in emissions_factors.columns.tolist():
print('t not in list:', t)
print('emissions_factors columns:', emissions_factors.index)
raise KeyError('Emissions data does not contain ' +
'all energy sources')
emissions_data = \
energy_data.multiply(emissions_factors.to_numpy())
try:
energy_data.loc[:, 'Census Region'] = \
energy_data.loc[:, 'Census Region'].astype(int).astype(str)
census_region = True
except KeyError:
census_region = False
try:
# ensure string Census Regions are ints (at least some
# start as floats-- need to match)
emissions_data.loc[:, 'Census Region'] = \
emissions_data['Census Region'].astype(int).astype(str)
except KeyError:
if census_region:
energy_data = energy_data[energy_data['Census Region'] == '0']
else:
pass
# Convert TEDB fuels to MMBtu
if datasource == 'TEDB':
try:
energy_data = \
energy_data.apply(lambda x: mapping_heat[x.name]*x, axis=0)
except KeyError:
print('energy_data.columns.tolist() not in dataframe:',
energy_data.columns.tolist())
for t in energy_data.columns.tolist():
if t not in mapping_heat.keys():
print('t not in list:', t)
raise KeyError(
'Heat content conversion data does not contain ' +
'all energy sources')
return emissions_data, energy_data
def calc_lmdi(self, breakout, calculate_lmdi, data_dict):
"""Calculate decomposition of CO2 emissions for the U.S. economy
TODO: Could simply call lmdi_gen main with a few slight adjustments
"""
results_dict, formatted_results = \
self.get_nested_lmdi(
level_of_aggregation=self.level_of_aggregation,
breakout=breakout, calculate_lmdi=calculate_lmdi,
raw_data=data_dict, lmdi_type=self.gen.lmdi_type)
return results_dict
class WeatherFactors:
def __init__(self, sector, directory, activity_data=None, residential_floorspace=None, nominal_energy_intensity=None, \
end_year=2018):
self.end_year = end_year
self.directory = directory
self.sector = sector
self.activity_data = activity_data
self.nominal_energy_intensity = nominal_energy_intensity
self.residential_floorspace = residential_floorspace
self.eia_data = GetEIAData(self.sector)
self.lmdi_prices = pd.read_excel(f'{self.directory}/EnergyPrices_by_Sector_010820_DBB.xlsx',
sheet_name='LMDI-Prices', header=14, usecols='A:B, EY')
self.regions_subregions = ['northeast', 'new_england', 'middle_atlantic', 'midwest',
'east_north_central', 'west_north_central', 'south',
'south_atlantic', 'east_south_central', 'west_south_central',
'west', 'mountain', 'pacific']
self.sub_regions_dict = {'northeast': ['New England', 'Middle Atlantic'],
'midwest': ['East North Central', 'West North Central'],
'south': ['South Atlantic', 'East South Central', 'West South Central'],
'west': ['Mountain', 'Pacific']}
@staticmethod
def adjust_data(subregions, hdd_by_division, hdd_activity_weights, cooling=True, cdd_by_division=None, \
cdd_activity_weights=None, use_weights_1961_90=True):
"""Calculate weights for adjusted weather factors prediction
"""
years_1961_90 = list(range(1961, 1990 + 1))
years_1981_2010 = list(range(1981, 1990 + 1))
if cooling:
cdd_by_division = cdd_by_division.set_index('Year')
cdd_by_division.index = cdd_by_division.index.astype(int)
averages_1961_90_cooling = cdd_by_division.loc[years_1961_90, :].mean(axis=0)
averages_1981_2010_cooling = cdd_by_division.loc[years_1981_2010, :].mean(axis=0)
hdd_by_division = hdd_by_division.set_index('Year')
hdd_by_division.index = hdd_by_division.index.astype(int)
averages_1961_90_heating = hdd_by_division.loc[years_1961_90, :].mean(axis=0)
averages_1981_2010_heating = hdd_by_division.loc[years_1981_2010, :].mean(axis=0)
all_s_weights_heating = []
all_s_weights_cooling = []
for s in subregions:
if use_weights_1961_90:
subregion_weights_heating = averages_1961_90_heating.loc[s] * hdd_activity_weights[s]
if cooling:
subregion_weights_cooling = averages_1961_90_cooling.loc[s] * cdd_activity_weights[s]
all_s_weights_cooling.append(subregion_weights_cooling)
else:
subregion_weights_heating = averages_1981_2010_heating.loc[s] * hdd_activity_weights[s]
if cooling:
subregion_weights_cooling = averages_1981_2010_cooling.loc[s] * cdd_activity_weights[s]
all_s_weights_cooling.append(subregion_weights_cooling)
all_s_weights_heating.append(subregion_weights_heating)
weights_dict = dict()
if cooling:
weights_cooling = sum(all_s_weights_cooling)
weights_dict['cooling'] = weights_cooling
weights_heating = sum(all_s_weights_heating)
weights_dict['heating'] = weights_heating
return weights_dict
def process_prices(self, weather_factors_df):
"""TODO: Are distributed lag and time cubed ever the desired variable?
Does this method need to exist?
"""
lmdi_prices = self.lmdi_prices
# distributed_lag =
# time_cubed =
selected_variable = [1] * len(weather_factors_df)
return selected_variable
@staticmethod
def cbecs_1995_shares():
"""Calculate fuels and elec shares for the commercial sector from CBECS 1995 data
"""
electricty_consumption_tbtu = {'Northeast': 436, 'Midwest': 558, 'South': 1027, 'West': 587}
electricty_consumption_tbtu['Total'] = sum(electricty_consumption_tbtu.values())
electricity_df = pd.DataFrame.from_dict(electricty_consumption_tbtu, orient='index', \
columns=['electricity_consumption_tbtu'])
energy_tbtu = {'Northeast': 1035, 'Midwest': 1497, 'South': 1684, 'West': 1106}
energy_tbtu['Total'] = sum(energy_tbtu.values())
energy_df = pd.DataFrame.from_dict(energy_tbtu, orient='index', columns=['energy'])
shares_df = energy_df.merge(electricity_df, left_index=True, right_index=True, how='outer')
shares_df['elec_share'] = shares_df.electricity_consumption_tbtu.divide(shares_df.loc['Total', \
'electricity_consumption_tbtu'])
shares_df['fuel_consumption'] = shares_df.energy.subtract(shares_df.electricity_consumption_tbtu)
shares_df['fuels_share'] = shares_df.fuel_consumption.divide(shares_df.loc['Total', 'fuel_consumption'])
return shares_df
@staticmethod
def recs_1993_shares():
"""Calculate fuels and elec shares for the residential sector from RECS 1993 data
"""
electricty_consumption_tbtu = {'Northeast': 470, 'Midwest': 740, 'South': 1510, 'West': 560}
electricty_consumption_tbtu['Total'] = sum(electricty_consumption_tbtu.values())
electricity_df = pd.DataFrame.from_dict(electricty_consumption_tbtu, orient='index', \
columns=['electricity_consumption_tbtu'])
energy_tbtu = {'Northeast': 2380, 'Midwest': 3130, 'South': 2950, 'West': 1550}
energy_tbtu['Total'] = sum(energy_tbtu.values())
energy_df = pd.DataFrame.from_dict(energy_tbtu, orient='index', columns=['energy'])
shares_df = energy_df.merge(electricity_df, left_index=True, right_index=True, how='outer')
shares_df['elec_share'] = shares_df.electricity_consumption_tbtu.divide(shares_df.loc['Total', \
'electricity_consumption_tbtu'])
shares_df['fuel_consumption'] = shares_df.energy.subtract(shares_df.electricity_consumption_tbtu)
shares_df['fuels_share'] = shares_df.fuel_consumption.divide(shares_df.loc['Total', 'fuel_consumption'])
return shares_df
def regional_shares(self, dataframe, cols):
"""Calulate shares of regional totals by subregion
"""
dataframe = dataframe.set_index('regions_subregions')
weights_data = dict()
for col in cols:
shares_dict = dict()
for r_, subregions in self.sub_regions_dict.items():
subregions = [s.lower().replace(' ', '_') for s in subregions]
regions_ = subregions + [r_]
region_total = dataframe.loc[r_, col]
for r in regions_:
share_value = dataframe.loc[r, col] / region_total
shares_dict[r] = share_value
weights_data[col] = shares_dict
return weights_data
def gather_weights_data(self):
"""Calculate weights to aggregate subregions into four regions
"""
if self.sector == 'residential':
electricity_data = {'total_elec_tbtu': {'northeast': 470, 'midwest': 740,
'south': 1510, 'west': 560},
'heating_tbtu': {'northeast': 12 * 3.412, 'midwest': 22 * 3.412,
'south': 61 * 3.412, 'west': 25 * 3.412},
'cooling_tbtu': {'northeast': 40, 'midwest': 80,
'south': 310, 'west': 30}}
fuels_data = {'all_energy_tbtu': {'northeast': 2380, 'midwest': 3130,
'south': 2950, 'west': 1550},
'electricity_tbtu': {'northeast': 470, 'midwest': 740,
'south': 1510, 'west': 560},
'heating_all_energy_tbtu': {'northeast': 1490, 'midwest': 1920,
'south': 1210, 'west': 700}}
# Residential Heating Households Millions
heating_activity = [4.1, 1, 3.1, 5.8, 3.5, 2.4, 18.8, 10.7, 3.4, 4.8, 8.3, 2, 6.3]
# Residential Cooling Households Millions
cooling_activity = [10.9, 2.1, 8.8, 16.4, 10.8, 5.6, 29.4, 15, 5.3, 9.2, 7.1, 2.1, 5.1]
all_energy = [19.1, 4.9, 14.2, 23.2, 16.3, 6.9, 32.8, 16.8, 5.9, 10.1, 19.4, 5.3, 14.1]
electricity = [1.9, 0.5, 1.4, 2.9, 1.6, 1.3, 14.6, 8.7, 2.5, 3.4, 5.6, 1.4, 4.2]
elif self.sector == 'commercial':
electricity_data = {'total_elec_tbtu': {'northeast': 436, 'midwest': 558,
'south': 1027, 'west': 587},
'heating_tbtu': {'northeast': 18, 'midwest': 23,
'south': 43, 'west': 28},
'cooling_tbtu': {'northeast': 44, 'midwest': 60,
'south': 172, 'west': 64}}
fuels_data = {'all_energy_tbtu': {'northeast': 1035, 'midwest': 1497,
'south': 1684, 'west': 1106},
'electricity_tbtu': {'northeast': 436, 'midwest': 558,
'south': 1027, 'west': 587},
'heating_all_energy_tbtu': {'northeast': 385, 'midwest': 668, 'south': 376,
'west': 275}}
# Commercial Heating Floorspace Million SF
heating_activity = [657, 137, 520, 779, 345, 434, 3189, 1648, 1140, 401, 1219, 469, 750]
# Commercial Cooling Floorspace Million SF
cooling_activity = [5919, 1472, 4447, 10860, 7301, 3559, 13666, 6512, 3265, 3889, 7058, 2812, 4246]
all_energy = [7661, 2031, 5630, 10860, 7301, 3559, 13666, 6512, 3265, 3889, 7065, 2819, 4246]
electricity = [657, 137, 520, 779, 345, 434, 3189, 1648, 1140, 401, 1219, 469, 750]
else:
return None
weights_data_ = {'regions_subregions': self.regions_subregions, 'heating_activity': heating_activity,
'cooling_activity': cooling_activity, 'all_energy': all_energy, 'electricity': electricity}
weights_df = pd.DataFrame(data=weights_data_)
weights_df['fuels'] = weights_df['all_energy'].subtract(weights_df['electricity'])
return weights_df
def heating_cooling_data(self):
hdd_by_division_historical = pd.read_csv('./Data/historical_hdd_census_division.csv').set_index('Year')
cdd_by_division_historical = pd.read_csv('./Data/historical_cdd_census_division.csv').set_index('Year')
hdd_by_division = self.eia_data.eia_api(id_='1566347', id_type='category')
hdd_to_drop = [c for c in list(hdd_by_division.columns) if 'Monthly' in c]
hdd_by_division = hdd_by_division.drop(hdd_to_drop, axis=1)
hdd_rename_dict = {c: c.replace(', Annual, Number', '') for c in list(hdd_by_division.columns)}
hdd_by_division = hdd_by_division.rename(columns=hdd_rename_dict)
hdd_by_division = pd.concat([hdd_by_division_historical, hdd_by_division], sort=True)
cdd_by_division = self.eia_data.eia_api(id_='1566348', id_type='category')
cdd_to_drop = [c for c in list(cdd_by_division.columns) if 'Monthly' in c]
cdd_by_division = cdd_by_division.drop(cdd_to_drop, axis=1)
cdd_rename_dict = {c: c.replace(', Annual, Number', '') for c in list(cdd_by_division.columns)}
cdd_by_division = cdd_by_division.rename(columns=cdd_rename_dict)
cdd_by_division = pd.concat([cdd_by_division_historical, cdd_by_division], sort=True)
title_case_regions = [s.replace('_', ' ').title() for s in self.regions_subregions]
hdd_names = [f'Heating Degree-Days, {r}' for r in title_case_regions]
cdd_names = [f'Cooling Degree-Days, {r}' for r in title_case_regions]
hdd_new_names_dict = {name: name_title for name, name_title in zip(hdd_names, title_case_regions)}
cdd_new_names_dict = {name: name_title for name, name_title in zip(cdd_names, title_case_regions)}
hdd_by_division = hdd_by_division.rename(columns=hdd_new_names_dict)
cdd_by_division = cdd_by_division.rename(columns=cdd_new_names_dict)
return hdd_by_division, cdd_by_division
def estimate_regional_shares(self):
"""Spreadsheet equivalent: Commercial --> 'Regional Shares'
assumed commercial floorspace in each region follows same trends as population or housing units"""
regions = ['Northeast', 'Midwest', 'South', 'West']
cbecs_data = pd.read_csv('./Data/cbecs_data_millionsf.csv').set_index('Year')
cbecs_data.index = cbecs_data.index.astype(str)
cbecs_years = list(cbecs_data.index)
cbecs_data = cbecs_data.rename(columns={'Midwest ': 'Midwest', ' South': 'South', ' West': 'West'})
cbecs_data.loc['1979', regions] = cbecs_data.loc['1983', regions].subtract([826, 972, 2665, 1212])
cbecs_data.loc['1979', ['U.S.']] = sum(cbecs_data.loc['1979', regions].values)
cbecs_data['U.S. (calc)'] = cbecs_data.sum(axis=1)
comm_regional_shares = cbecs_data.drop(['U.S.', 'U.S. (calc)'], axis=1).divide(cbecs_data['U.S. (calc)'].values.reshape(len(cbecs_data), 1))
comm_regional_shares_ln = np.log(comm_regional_shares)
residential_data = ResidentialFloorspace(end_year=self.end_year) # change to pull from residential().activity()
final_results_total_floorspace_regions, regional_estimates_all, avg_size_all_regions = residential_data.final_floorspace_estimates()
regional_dfs = [regional_estimates_all[r][['Total']].rename(columns={'Total': r}) for r in regions]
residential_housing_units = reduce(lambda x, y: pd.merge(x, y, left_index=True, right_index=True, how='outer'), regional_dfs)
residential_housing_units['U.S.'] = residential_housing_units.sum(axis=1)
residential_housing_units.index = residential_housing_units.index.astype(str)
regional_shares_residential_housing_units = residential_housing_units.drop('U.S.', axis=1).divide(residential_housing_units['U.S.'].values.reshape(len(residential_housing_units), 1))
regional_shares_residential_housing_units_ln = np.log(regional_shares_residential_housing_units)
regional_shares_residential_housing_units_cbecs_years = regional_shares_residential_housing_units.loc[cbecs_years, :]
regional_shares_residential_housing_units_cbecs_years_ln = np.log(regional_shares_residential_housing_units_cbecs_years)
predictions_df = pd.DataFrame(columns=comm_regional_shares.columns, index=residential_housing_units.index)
for region in comm_regional_shares.columns:
x_values = comm_regional_shares_ln[region].values
X = x_values.transpose()
y = regional_shares_residential_housing_units_cbecs_years_ln[region].values
p = np.polyfit(X, y, 1)
predictions_df[region] = np.exp(regional_shares_residential_housing_units_ln[region].multiply(p[0]).add(p[1]))
predictions_df['Predicted Sum'] = predictions_df.sum(axis=1)
normalized_shares = predictions_df.drop('Predicted Sum', axis=1).divide(predictions_df['Predicted Sum'].values.reshape(len(predictions_df), 1))
return normalized_shares
def commercial_estimate_regional_floorspace(self):
regional_shares = self.estimate_regional_shares()
commercial_floorspace = self.activity_data
regional_shares_index = regional_shares.index.astype(str)
commercial_floorspace_reshape = commercial_floorspace.loc[regional_shares_index, :]
regional_floorspace = regional_shares.multiply(commercial_floorspace_reshape.values)
return regional_floorspace
def commercial_regional_intensity_aggregate(self):
"""Calculate Energy Intensities (kBtu/sq. ft.) by region and fuel type (i.e. Fuels and Electricity) for use
in calculating weather factors
Returns:
dictionary with keys: 'electricity' and 'fuels', values: dataframes of intensity data for the commercial sector
with Year index and Region columns
"""
regional_floorspace = self.commercial_estimate_regional_floorspace()
total_fuels_to_indicators, elec_to_indicators = self.eia_data.get_seds()
regional_floorspace_index = regional_floorspace.index
elec_to_indicators = elec_to_indicators.loc[regional_floorspace_index, :]
total_fuels_to_indicators = total_fuels_to_indicators.loc[regional_floorspace_index, :]
fuels_regional = regional_floorspace.multiply(total_fuels_to_indicators.drop('National', axis=1).values)
elec_regional = regional_floorspace.multiply(elec_to_indicators.drop('National', axis=1).values)
return {'fuels': fuels_regional, 'electricity': elec_regional}
def residential_regional_intensity_aggregate(self):
"""This function does not need to exist if nominal_energy_intensity is properly formated, change formatting here if not
Returns:
dictionary with keys: 'electricity' and 'fuels', values: dataframes of intensity data for the residential sector
with Year index and Region columns
i.e. {'fuels': fuels_regional, 'electricity': elec_regional}
"""
nominal_energy_intensity = self.nominal_energy_intensity # nominal_energy_intensity should already be formated in this way
return nominal_energy_intensity
def weather_factors(self, region, energy_type, actual_intensity, weights_df, regional_weights):
"""Estimate a simple regression model to fit the regional intensity to a linear function of time (included squared and cubed values of time) and degree days.
-electricity model: constant term, heating degree day (HDD), cooling degree day (CDD), time, time-squared, and time-cubed
-fuels model: contant term?, HDD, HDD*Time, Time, Time-squared and composite fuel price index (the composite fuel price index was developed as a weighted average of the national distillate
fuel oil price index and a national average price for natural gas)
Weather factors are applied at the regional level to generate the weather-normalized intensity indexes for each of the four Census regions
-The weather factors for delivered energy and source energy are computed implicitly. For delivered energy, they are calculated
as the sum of reported electricity and fuels divided by the sum of the weather-adjusted electricity and weather-adjusted fuels.
A similar procedure is followed for source energt. As such, the implied weather factors are a result of the process, not an independent
variable that influences the values of intensity indexes for delivered energy and source energy. All of these computation occur within Commercial_Total worksheet.
TODO: Input data
"""
if energy_type == 'electricity':
energy_type = 'elec'
subregions = self.sub_regions_dict[region]
subregions_lower = [s.lower().replace(' ', '_') for s in subregions]
hdd_activity_weights = [regional_weights['heating_activity'][r_] for r_ in subregions_lower]
hdd_activity_weights_dict = {r : regional_weights['heating_activity'][r_] for r, r_ in zip(subregions, subregions_lower)}
cdd_activity_weights = [regional_weights['cooling_activity'][r_] for r_ in subregions_lower]
cdd_activity_weights_dict = {r : regional_weights['cooling_activity'][r_] for r, r_ in zip(subregions, subregions_lower)}
fuels_weights = [regional_weights['fuels'][r_] for r_ in subregions_lower]
hdd_by_division, cdd_by_division = self.heating_cooling_data()
heating_degree_days = hdd_by_division[subregions]
heating_degree_days = heating_degree_days.reset_index('Year')
heating_degree_days[region] = heating_degree_days[subregions].dot(hdd_activity_weights)
fuels_heating_degree_days = heating_degree_days
fuels_heating_degree_days[region] = fuels_heating_degree_days[subregions].dot(fuels_weights)
weather_factors_df = heating_degree_days[['Year', region]].rename(columns={region: 'HDD'})
weather_factors_df['Year'] = weather_factors_df['Year'].astype(int)
weather_factors_df['Time'] = weather_factors_df['Year'].values - 1969
weather_factors_df['Time^2'] = weather_factors_df[['Time']].pow(2).values
if energy_type == 'elec':
cooling_degree_days = cdd_by_division[subregions]
cooling_degree_days[region] = cooling_degree_days[subregions].dot(cdd_activity_weights)
cooling_degree_days = cooling_degree_days.reset_index('Year')
cooling_degree_days['Year'] = cooling_degree_days['Year'].astype(int)
weather_factors_df_cooling = cooling_degree_days[['Year', region]].rename(columns={region: 'CDD'})
weather_factors_df = weather_factors_df.merge(weather_factors_df_cooling, on='Year', how='outer')
weather_factors_df['Time^3'] = weather_factors_df[['Time']].pow(3).values
weather_factors_df = weather_factors_df.set_index('Year')
weather_factors_df.index = weather_factors_df.index.astype(int)
X_data = weather_factors_df[['HDD', 'CDD', 'Time', 'Time^2', 'Time^3']]
elif energy_type == 'fuels':
weather_factors_df['HDD*Time'] = heating_degree_days[region].multiply(weather_factors_df['Time'])
weather_factors_df['Price'] = self.process_prices(weather_factors_df)
weather_factors_df = weather_factors_df.set_index('Year')
weather_factors_df.index = weather_factors_df.index.astype(int)
X_data = weather_factors_df[['HDD', 'HDD*Time', 'Time', 'Time^2', 'Price']]
# elif self.energy_type == 'delivered':
# weather_factor = (reported_electricity + fuels) / (weather_adjusted_electrity + weather_adjusted_fuels)
# return weather_factor
else:
raise KeyError(f'Missing valid energy type. Type given: {energy_type}')
actual_intensity.index = actual_intensity.index.astype(int)
data = X_data.merge(actual_intensity, left_index=True, right_index=True, how='inner').dropna()
X = data.drop(region.capitalize(), axis=1)
Y = data[[region.capitalize()]]
reg = linear_model.LinearRegression()
reg.fit(X, Y)
coefficients = reg.coef_
coefficients = coefficients[0]
intercept = reg.intercept_
predicted_value_intensity_actualdd = reg.predict(X) # Predicted value of the intensity based on actual degree days
if energy_type == 'elec':
prediction2_weights = self.adjust_data(subregions=subregions, hdd_by_division=heating_degree_days, cdd_by_division=cooling_degree_days,
cdd_activity_weights=cdd_activity_weights_dict, hdd_activity_weights=hdd_activity_weights_dict,
use_weights_1961_90=True)
predicted_value_intensity_ltaveragesdd = intercept + coefficients[0] * prediction2_weights['heating'] + coefficients[1] * prediction2_weights['cooling'] + \
coefficients[2] * data['Time'] + coefficients[3] * data['Time^2'] + coefficients[4] * data['Time^3'] # Predicted value of the intensity based on the long-term averages of the degree days
elif energy_type == 'fuels':
prediction2_weights = self.adjust_data(subregions=subregions, hdd_by_division=heating_degree_days,
hdd_activity_weights=hdd_activity_weights_dict, cooling=False,
use_weights_1961_90=True)
predicted_value_intensity_ltaveragesdd = intercept + coefficients[0] * prediction2_weights['heating'] + coefficients[1] * data['Time'] + \
coefficients[2] * data['Time'] + coefficients[3] * data['Time^2'] + coefficients[4] * data['Price'] # Predicted value of the intensity based on the long-term averages of the degree days
weather_factor = predicted_value_intensity_actualdd.flatten() / predicted_value_intensity_ltaveragesdd.values.flatten()
try:
weather_normalized_intensity = actual_intensity.loc[data.index].divide(weather_factor.reshape(len(weather_factor), 1))
except Exception:
try:
weather_normalized_intensity = actual_intensity.loc[data.index].divide(weather_factor)
except Exception as e:
raise ValueError(f'Failure to divide: {actual_intensity.shape} by {weather_factor.shape}, failed with error {e}')
weather_factor_df = pd.DataFrame(data={'Year': data.index, f'{region}_weather_factor': weather_factor}).set_index('Year')
return weather_factor_df, weather_normalized_intensity
def national_method1_fixed_end_use_share_weights(self, energy_type_):
"""Used fixed weights to develop from regional factors, weighted by regional energy share from 1995 CBECS
"""
if self.sector == 'commercial':
shares = self.cbecs_1995_shares()
intensity_df = self.commercial_regional_intensity_aggregate()
elif self.sector == 'residential':
intensity_df = self.residential_regional_intensity_aggregate()
shares = self.recs_1993_shares()
if energy_type_ == 'elec':
energy_type = 'electricity'
else:
energy_type = energy_type_
regional_weather_factors = []
weights_df = self.gather_weights_data()
regional_weights = self.regional_shares(dataframe=weights_df, cols=['heating_activity', 'cooling_activity', 'fuels'])
for region in self.sub_regions_dict.keys():
region_cap = region.capitalize()
if self.sector == 'residential':
regional_intensity = intensity_df[region_cap][energy_type_]
elif self.sector == 'commercial':
regional_intensity = intensity_df[energy_type_][region_cap]
weather_factors, weather_normalized_intensity = self.weather_factors(region, energy_type_, actual_intensity=regional_intensity, weights_df=weights_df, regional_weights=regional_weights)
regional_weather_factors.append(weather_factors)
weather_factors_all = pd.concat(regional_weather_factors, axis=1)
weather_factors_all = weather_factors_all.reindex(columns=list(weather_factors_all.columns) + [f'{energy_type_}_weather_factor'])
for y in weather_factors_all.index:
if energy_type == 'electricity':
energy_type = 'elec'
share_name = f'{energy_type}_share'
year_weather = weather_factors_all.drop(f'{energy_type_}_weather_factor', axis=1).loc[y, :]
weights = shares[share_name].drop('Total')
year_factor = year_weather.dot(weights.to_numpy())
weather_factors_all.loc[y, [f'{energy_type_}_weather_factor']] = year_factor
return weather_factors_all
def national_method2_regression_models(self, seds_data, weather_factors):
seds_data, weather_factors = CalculateLMDI.ensure_same_indices(seds_data, weather_factors)
weather_adjusted_consumption = seds_data.drop('National', axis=1).multiply(weather_factors.values)
weather_adjusted_consumption['National'] = weather_adjusted_consumption.sum(axis=1)
implicit_national_weather_factor = seds_data[['National']].divide(weather_adjusted_consumption['National'].values.reshape(len(weather_adjusted_consumption), 1))
return implicit_national_weather_factor
def adjust_for_weather(self, data, energy_type):
"""Adjust data by weather factors
Parameters
----------
data: dataframe
dataset to adjust by weather
weather_factors: array?
Returns
-------
weather_adjusted_data: dataframe ?
"""
weather_factors = self.national_method1_fixed_end_use_share_weights(energy_type)
weather_adjusted_data = data / weather_factors[energy_type]
return weather_adjusted_data
def get_weather(self, energy_dict=None, energy_type=None, energy_df=None, weather_adjust=False, seds_data=None):
if self.sector == 'residential':
if weather_adjust:
for type, energy_dataframe in energy_dict.items():
weather_adj_energy = self.adjust_for_weather(energy_dataframe, type)
energy_dict[f'{type}_weather_adj'] = weather_adj_energy
return energy_dict
else:
weather_factors = dict()
for type in energy_dict.keys():
weather_factors_t = self.national_method1_fixed_end_use_share_weights(energy_type_=type)
if type == 'electricity':
type = 'elec'
weather_factors[type] = weather_factors_t
return weather_factors
elif self.sector == 'commercial':
weather_factors = dict()
for type in ['electricity', 'fuels']:
weather_factors_method1 = self.national_method1_fixed_end_use_share_weights(type)
early_years = range(min(weather_factors_method1.index), 1969 + 1)
weather_factors_early = weather_factors_method1.loc[early_years, [f'{type}_weather_factor']]
weather = weather_factors_method1.drop(f'{type}_weather_factor', axis=1)
if type == 'electricity':
type = 'elec'
type_seds = seds_data[type]
weather_factors_method2 = self.national_method2_regression_models(seds_data=type_seds, weather_factors=weather)
weather_factors_method2 = weather_factors_method2.rename(columns={'National': f'{type}_weather_factor'})
late_years = range(1970, max(weather_factors_method2.index) + 1)
weather_factors_late = weather_factors_method2.loc[late_years]
weather_factors_t = pd.concat([weather_factors_early, weather_factors_late], sort=True)
weather_factors[type] = weather_factors_t
return weather_factors
class ElectricityIndicators(CalculateLMDI):
def __init__(self,
directory,
output_directory,
level_of_aggregation,
lmdi_model=['multiplicative'],
base_year=1985):
self.sub_categories_list = {
'Elec Generation Total': {
'Elec Power Sector': {
'Electricity Only': {
'Fossil Fuels': {
'Coal': None,
'Petroleum': None,
'Natural Gas': None,
'Other Gasses': None
},
'Nuclear': None,
'Hydro Electric': None,
'Renewable': {
'Wood': None,
'Waste': None,
'Geothermal': None,
'Solar': None,
'Wind': None
}
},
'Combined Heat & Power': {
'Fossil Fuels': {
'Coal': None,
'Petroleum': None,
'Natural Gas': None,
'Other Gasses': None
},
'Renewable': {
'Wood': None,
'Waste': None
}
}
},
'Commercial Sector': None,
'Industrial Sector': None
},
'All CHP': {
'Elec Power Sector': {
'Combined Heat & Power': {
'Fossil Fuels': {
'Coal': None,
'Petroleum': None,
'Natural Gas': None,
'Other Gasses': None
},
'Renewable': {
'Wood': None,
'Waste': None
},
'Other': None
}
},
'Commercial Sector': {
'Combined Heat & Power': {
'Fossil Fuels': {
'Coal': None,
'Petroleum': None,
'Natural Gas': None,
'Other Gasses': None
},
'Hydroelectric': None,
'Renewable': {
'Wood': None,
'Waste': None
},
'Other': None
}
},
'Industrial Sector': {
'Combined Heat & Power': {
'Fossil Fuels': {
'Coal': None,
'Petroleum': None,
'Natural Gas': None,
'Other Gasses': None
},
'Hydroelectric': None,
'Renewable': {
'Wood': None,
'Waste': None
},
'Other': None
}
}
}
}
self.elec_power_eia = GetEIAData(sector='electricity')
# self.Table21f = pd.read_excel('https://www.eia.gov/totalenergy/data/browser/xls.php?tbl=T02.06') #self.elec_power_eia.eia_api(id_='711254') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711254'
# self.Table82a = self.elec_power_eia.eia_api(id_='3') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=3'
# self.Table82b = self.elec_power_eia.eia_api(id_='21') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21'
# self.Table82c = self.elec_power_eia.eia_api(id_='1736765') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1736765' ?
# self.Table82d = self.elec_power_eia.eia_api(id_='711282') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711282'
# # self.Table82d_2012_and_later = self.elec_power_eia.eia_api(id_='1017') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=1017'
# self.Table83d_03 = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ?
# self.Table84b = self.elec_power_eia.eia_api(id_='711284') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711284' ?
# self.Table85c = self.elec_power_eia.eia_api(id_='379') # ?
# self.Table86b = self.elec_power_eia.eia_api(id_='463') # ?
# self.MER_T72b_1013_AnnualData = self.elec_power_eia.eia_api(id_='21') #'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=21'
# self.MER_T72c_1013_AnnualData = self.elec_power_eia.eia_api(id_='2') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=2'
self.energy_types = ['primary']
self.Table84c = pd.read_csv(
'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03C'
) # Consumption of combustible fuels for electricity generation: Commercial and industrial sectors (selected fuels) # elec_power_eia.eia_api(id_='456')
self.Table85c = pd.read_csv(
'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.03B'
) # Consumption of combustible fuels for electricity generation: Electric power sector
self.Table82d = pd.read_csv(
'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl=T07.02C')
super().__init__(sector='electric', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \
energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
@staticmethod
def get_eia_aer():
"""Prior to 2012, the data for the indicators were taken directly from tables published (and downloaded in
Excel format) from EIA's Annual Energy Review.
"""
pass
@staticmethod
def get_reconciles():
""" The EIA Annual Energy Review data (pre 2012) for energy consumption to produce
electricity were generally supplied in physical units only (e.g., mcf of natural gas, tons of coal, etc.) The
values needed to be converted to Btu, and still be consistent with aggregate energy consumption for
this sector as published by EIA. For each major fossil fuel, a separate worksheet was developed; these
worksheets are identified with the suffix “reconcile.” Thus, the worksheet “NatGas Reconcile” seeks to
produce an estimate of the Btu consumption of natural gas used to generate electricity. Similar
worksheets were developed for coal, petroleum, and other fuels.
/// Does this need to be done or can download Btu directly instead of converting physical units to Btu --> no
"""
pass
def process_utility_level_data(self):
"""The indicators for electricity are derived entirely from data collected by EIA. Since 2012, the indicators
are based entirely upon te EIA-923 survey
"""
eia_923_schedules = pd.read_excel('./')
net_generation = pd.pivot_table(eia_923_schedules,
columns='EIA Sector Number',
index='AERFuel Type Code',
aggfunc='sum') # page A-71,
# 'Net Generation' lower right-hand quadrant?
net_generation['Grand_Total'] = net_generation.sum(
axis=1, skipna=True
) # Should have 18 rows labeled by type of fuel and seven columns
# plus one for 'Grand Total'. Note: rows is not an arg of pd.pivot_table
elec_btu_consumption = pd.pivot_table(eia_923_schedules,
columns='EIA Sector Number',
index='AERFuel Type Code',
aggfunc='sum') # page A-71,
# 'Elec Fuel ConsumptionMMBTU' lower right-hand quadrant?
elec_btu_consumption['Grand_Total'] = elec_btu_consumption.sum(
axis=1, skipna=True
) # Should have 18 rows labeled by type of fuel and seven columns
# plus one for 'Grand Total'
previous_years_net_gen = pd.read_excel('./')
previous_yeas_elec_btu_consumption = pd.read_excel('./')
master_net_gen = previous_years_net_gen.concat(net_generation)
maseter_elec_btu_consumption = previous_yeas_elec_btu_consumption.concat(
elec_btu_consumption)
# Aggregate data?? page A-72 fpr net generation and elec btu consumption
return None
@staticmethod
def reconcile(total, elec_gen, elec_only_plants, chp_elec,
assumed_conv_factor, chp_heat):
"""total: df, Btu
elec_gen: df, Btu
elec_only_plants: df, Short tons
chp_elec: df, Short tons
assumed_conv_factor: float, MMBtu/Ton
chp_heat
"""
difference = total.subtract(elec_gen) # Btu
implied_conversion_factor = total.divide(elec_only_plants).multiply(
1000) # MMBtu/Ton
elec_only_billionbtu = elec_only_plants.multiply(
assumed_conv_factor).multiply(1000) # Billion Btu
chp_elec_billionbtu = chp_elec.multiply(assumed_conv_factor).multiply(
0.001) # Billion Btu
chp_heat_billionbtu = chp_heat.multiply(assumed_conv_factor).multiply(
0.001) # Billion Btu
total_fuel = elec_only_billionbtu.add(chp_elec_billionbtu).add(
chp_heat_billionbtu)
# Cross Check
total_short_tons = elec_only_plants + chp_elec + chp_heat # Short Tons
implied_conversion_factor_cross = total.divide(
total_short_tons).multiply(1000) # MMBtu/Ton
implied_conversion_factor_revised = elec_gen.divide(
chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton
chp_plants_fuel = implied_conversion_factor_revised.multiply(
chp_elec).multiply(0.000001) # Trillion Btu
elec_only_fuel = elec_gen.multiply(.001).subtract(
chp_plants_fuel) # Trillion Btu
resulting_total = chp_plants_fuel.add(elec_only_fuel)
return chp_plants_fuel, elec_only_fuel
def coal_reconcile(self):
energy_consumption_coal = self.elec_power_eia.eia_api(
id_='TOTAL.CLEIBUS.A', id_type='series') # Table21f11 column b
consumption_for_electricity_generation_coal = self.elec_power_eia.eia_api(
id_='TOTAL.CLEIBUS.A', id_type='series') # Table84b11 column b
consumption_combustible_fuels_electricity_generation_coal = self.elec_power_eia.eia_api(
id_='TOTAL.CLL1PUS.A', id_type='series'
) # Table85c11 column B SHOULD BE separated Elec-only/CHP
consumption_combustible_fuels_useful_thermal_output_coal = self.elec_power_eia.eia_api(
id_='TOTAL.CLEIPUS.A', id_type='series') # Table86b11 column B
assumed_conversion_factor = 20.9
total = energy_consumption_coal
elec_gen = consumption_for_electricity_generation_coal
elec_only_plants = consumption_combustible_fuels_electricity_generation_coal # should be separate part?
chp_elec = consumption_combustible_fuels_electricity_generation_coal # should be separate part?
assumed_conv_factor = assumed_conversion_factor
chp_heat = consumption_combustible_fuels_useful_thermal_output_coal
# eia-923 pivot table
difference = total.subtract(elec_gen) # Btu
implied_conversion_factor = total.divide(elec_only_plants).multiply(
1000) # MMBtu/Ton
elec_only_billionbtu = elec_only_plants.multiply(
assumed_conv_factor).multiply(1000) # Billion Btu
chp_elec_billionbtu = chp_elec.multiply(assumed_conv_factor).multiply(
0.001) # Billion Btu
chp_heat_billionbtu = chp_heat.multiply(assumed_conv_factor).multiply(
0.001) # Billion Btu
total_fuel = elec_only_billionbtu.add(chp_elec_billionbtu).add(
chp_heat_billionbtu)
# Cross Check
total_short_tons = elec_only_plants + chp_elec + chp_heat # Short Tons
implied_conversion_factor_cross = total.divide(
total_short_tons).multiply(1000) # MMBtu/Ton
implied_conversion_factor_revised = elec_gen.divide(
chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton
chp_plants_fuel = implied_conversion_factor_revised.multiply(
chp_elec).multiply(0.000001) # Trillion Btu
elec_only_fuel = elec_gen.multiply(.001).subtract(
chp_plants_fuel) # Trillion Btu
resulting_total = chp_plants_fuel.add(elec_only_fuel)
return chp_plants_fuel, elec_only_fuel
def natgas_reconcile(self):
energy_consumption_natgas = self.elec_power_eia.eia_api(
id_='TOTAL.NNEIBUS.A', id_type='series') # Table21f11 column d
consumption_for_electricity_generation_natgas = self.elec_power_eia.eia_api(
id_='TOTAL.NNEIBUS.A', id_type='series') # Table84b11 column f
consumption_combustible_fuels_electricity_generation_natgas = self.elec_power_eia.eia_api(
id_='TOTAL.NGL1PUS.A', id_type='series') # Table85c11 column N
consumption_combustible_fuels_useful_thermal_output_natgas = self.elec_power_eia.eia_api(
id_='TOTAL.NGEIPUS.A', id_type='series') # Table86b11 column M
# eia-923 pivot table
"""total: df, Billion Btu
elec_gen: df, Billion Btu
elec_only_plants: df, Thou. Cu. Ft.
chp_elec: df, Thou. CF
assumed_conv_factor: float, MMBtu/Ton
chp_heat: df, Thou. CF
"""
total = energy_consumption_natgas
elec_gen = consumption_for_electricity_generation_natgas
elec_only_plants = consumption_combustible_fuels_electricity_generation_natgas
chp_elec = consumption_combustible_fuels_electricity_generation_natgas # different part
assumed_conv_factor = 1.028
chp_heat = consumption_combustible_fuels_useful_thermal_output_natgas
difference = total.subtract(elec_gen) # Billion Btu
implied_conversion_factor = elec_gen.divide(elec_only_plants).multiply(
1000) # kBtu/CF * different from coal_reconcile
elec_only_trillionbtu = elec_only_plants.multiply(
assumed_conv_factor).multiply(0.001) # Trillion Btu
chp_elec_trillionbtu = chp_elec.multiply(assumed_conv_factor).multiply(
0.001) # Trillion Btu
chp_heat_trillionbtu = chp_heat.multiply(assumed_conv_factor).multiply(
0.001) # Trillion Btu
total_fuel = elec_only_trillionbtu.add(chp_elec_trillionbtu).add(
chp_heat_trillionbtu)
# Cross Check
total_thou_cf = elec_only_plants + chp_elec + chp_heat # Thou. CF
implied_conversion_factor_cross = total.divide(total_thou_cf).multiply(
1000) # kBtu/CF
implied_conversion_factor_revised = elec_gen.divide(
chp_elec.add(elec_only_plants)).multiply(1000) # MMBtu/Ton
chp_plants_fuel = implied_conversion_factor_revised.multiply(
chp_elec).multiply(0.000001) # Trillion Btu
elec_only_fuel = elec_gen.multiply(.001).subtract(
chp_plants_fuel) # Trillion Btu
resulting_total = chp_plants_fuel.add(elec_only_fuel)
return chp_plants_fuel, elec_only_fuel
def petroleum_reconcile(self):
energy_consumption_petroleum = self.elec_power_eia.eia_api(
id_='TOTAL.PAEIBUS.A', id_type='series') # Table21f11 column F
consumption_for_electricity_generation_petroleum = self.elec_power_eia.eia_api(
id_='TOTAL.PAEIBUS.A', id_type='series') # Table84b11 column D
consumption_combustible_fuels_electricity_generation_distillate_fuel_oil = self.elec_power_eia.eia_api(
id_='TOTAL.DKL1PUS.A',
id_type='series') # Table85c11 column D, F, H, J
consumption_combustible_fuels_electricity_generation_residual_fuel_oil = self.elec_power_eia.eia_api(
id_='TOTAL.RFL1PUS.A', id_type='series')
consumption_combustible_fuels_electricity_generation_other_liquids = self.elec_power_eia.eia_api(
id_='TOTAL.OLL1PUS.A', id_type='series')
consumption_combustible_fuels_electricity_generation_petroleum_coke = self.elec_power_eia.eia_api(
id_='TOTAL.PCL1MUS.A', id_type='series')
consumption_combustible_fuels_electricity_generation_total_petroleum = self.elec_power_eia.eia_api(
id_='TOTAL.PAL1PUS.A', id_type='series')
fuels_list = [
consumption_combustible_fuels_electricity_generation_distillate_fuel_oil,
consumption_combustible_fuels_electricity_generation_residual_fuel_oil,
consumption_combustible_fuels_electricity_generation_other_liquids,
consumption_combustible_fuels_electricity_generation_petroleum_coke
]
elec_only_plants_petroleum = reduce(
lambda x, y: pd.merge(x, y, on='Period'), fuels_list)
consumption_combustible_fuels_useful_thermal_output_distillate_fuel_oil = self.elec_power_eia.eia_api(
id_='TOTAL.DKEIPUS.A',
id_type='series') # Table86b11 column D, F, G, I
consumption_combustible_fuels_useful_thermal_output_residual_fuel_oil = self.elec_power_eia.eia_api(
id_='TOTAL.RFEIPUS.A', id_type='series')
consumption_combustible_fuels_useful_thermal_output_other_liquids = self.elec_power_eia.eia_api(
id_='TOTAL.OLEIPUS.A', id_type='series')
consumption_combustible_fuels_useful_thermal_output_petroleum_coke = self.elec_power_eia.eia_api(
id_='TOTAL.PCEIMUS.A', id_type='series')
consumption_combustible_fuels_useful_thermal_output_total_petroleum = self.elec_power_eia.eia_api(
id_='TOTAL.PTEIPUS.A', id_type='series')
chp_elec = consumption_combustible_fuels_electricity_generation_total_petroleum
# eia-923 pivot table
assumed_conversion_factors = [5.825, 6.287, 5.67, 30120]
chp_plants_petroleum, elec_only_petroleum = self.reconcile(
total=energy_consumption_petroleum,
elec_gen=consumption_for_electricity_generation_petroleum,
elec_only_plants=elec_only_plants_petroleum,
chp_elec=chp_elec,
assumed_conv_factor=assumed_conversion_factors,
chp_heat=
consumption_combustible_fuels_useful_thermal_output_total_petroleum
)
return chp_plants_petroleum, elec_only_petroleum
def othgas_reconcile(self):
consumption_for_electricity_generation_fossil_fuels = self.elec_power_eia.eia_api(
id_='TOTAL.FFEIBUS.A', id_type='series') # Table84b11 column H
consumption_for_electricity_generation_oth_gas = consumption_for_electricity_generation_fossil_fuels - consumption_for_electricity_generation_petroleum - consumption_for_electricity_generation_natgas - consumption_for_electricity_generation_coal
consumption_combustible_fuels_electricity_generation_oth_gas = self.elec_power_eia.eia_api(
id_='TOTAL.OJL1BUS.A', id_type='series') # Table85c11 column P
consumption_combustible_fuels_useful_thermal_output_othgas = self.elec_power_eia.eia_api(
id_='TOTAL.OJEIBUS.A', id_type='series') # Table86b11 column O
elec_gen = consumption_for_electricity_generation_oth_gas
elec_only_plants = consumption_combustible_fuels_electricity_generation_oth_gas
chp_elec = consumption_combustible_fuels_electricity_generation_oth_gas # ** different part of the series?'
chp_heat = consumption_combustible_fuels_useful_thermal_output_othgas
total_other_gas = elec_only_plants.add(chp_elec).add(chp_heat)
return chp_elec
# consumption_combustible_fuels_electricity_generation =
# print(consumption_combustible_fuels_electricity_generation)
# consumption_combustible_fuels_useful_thermal_output =
# print(consumption_combustible_fuels_useful_thermal_output)
def industrial_sector_chp_renew(self):
wood_energy = self.Table84c[
self.Table84c['Description'] ==
'Wood Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001)
waste_energy = self.Table84c[
self.Table84c['Description'] ==
'Waste Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4C column R # TBtu
wood_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WDI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column R
waste_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WSI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column T
data_dict = {
'Wood': {
'energy': {
'primary': wood_energy
},
'activity': wood_activity
},
'Waste': {
'energy': {
'primary': waste_energy
},
'activity': waste_activity
}
}
return data_dict
def industrial_sector_chp_fossil(self):
coal_energy = self.Table84c[
self.Table84c['Description'] ==
'Coal Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4c column B # TBtu
petroleum_energy = self.Table84c[
self.Table84c['Description'] ==
'Petroleum Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4c column D # TBtu
natgas_energy = self.Table84c[
self.Table84c['Description'] ==
'Natural Gas Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4c column F # TBtu
othgas_energy = self.Table84c[
self.Table84c['Description'] ==
'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4c column H # TBtu
coal_activity = self.elec_power_eia.eia_api(
id_='TOTAL.CLI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column B
petroleum_activity = self.elec_power_eia.eia_api(
id_='TOTAL.PAI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column D
natgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.NGI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column F
othgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.OJI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column H
data_dict = {
'Coal': {
'energy': {
'primary': coal_energy
},
'activity': coal_activity
},
'Petroleum': {
'energy': {
'primary': petroleum_energy
},
'activity': petroleum_activity
},
'Natural Gas': {
'energy': {
'primary': natgas_energy
},
'activity': natgas_activity
},
'Other Gasses': {
'energy': {
'primary': othgas_energy
},
'activity': othgas_activity
}
}
return data_dict
def industrial_sector_total(self):
"""Note: Other includes batteries, chemicals, hydrogen, pitch, purchased steam, sulfur, and miscellaneous technologies
"""
hydroelectric_energy = None #.multiply(0.001) # Table 8.4C11 column N
other_energy = self.Table84c[
self.Table84c['Description'] ==
'Other Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4C11 column AB
hydroelectric_activity = self.elec_power_eia.eia_api(
id_='TOTAL.HVI5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d11 Column P
other_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d11 Column AD
data_dict = {
'Hydroelectric': {
'energy': {
'primary': hydroelectric_energy
},
'activity': hydroelectric_activity
},
'Other': {
'energy': {
'primary': other_energy
},
'activity': other_activity
}
}
return data_dict
def comm_sector_chp_renew(self):
"""As is, these are the same sources as ind sector, but should be different part of columns?
"""
wood_energy = self.Table84c[
self.Table84c['Description'] ==
'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4C column P # TBtu
waste_energy = self.Table84c[
self.Table84c['Description'] ==
'Other Gases Consumption for Electricity Generation, Industrial Sector'].multiply(
0.001) # Table 8.4C column R # TBtu
wood_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d column R
waste_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WSC5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column T
data_dict = {
'Wood': {
'energy': {
'primary': wood_energy
},
'activity': wood_activity
},
'Waste': {
'energy': {
'primary': waste_energy
},
'activity': waste_activity
}
}
return data_dict
def comm_sector_chp_fossil(self):
coal_energy = self.Table84c[
self.Table84c['Description'] ==
'Coal Consumption for Electricity Generation, Commercial Sector'].multiply(
0.001) # Table 8.4c column B # TBtu
petroleum_energy = self.Table84c[
self.Table84c['Description'] ==
'Petroleum Consumption for Electricity Generation, Commercial Sector'].multiply(
0.001) # Table 8.4c column D # TBtu
natgas_energy = self.Table84c[
self.Table84c['Description'] ==
'Natural Gas Consumption for Electricity Generation, Commercial Sector'].multiply(
0.001) # Table 8.4c column F # TBtu
othgas_energy = None #.multiply(0.001) # Table 8.4c column H # TBtu
coal_activity = self.elec_power_eia.eia_api(
id_='TOTAL.CLC5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column B
petroleum_activity = self.elec_power_eia.eia_api(
id_='TOTAL.PAC5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column D
natgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.NGC5PUS.A',
id_type='series').multiply(0.001) # Table 8.2d column F
othgas_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d column H
# Industrial and Commercial have same spreadsheet sources but different parts of the columns
data_dict = {
'Coal': {
'energy': {
'primary': coal_energy
},
'activity': coal_activity
},
'Petroleum': {
'energy': {
'primary': petroleum_energy
},
'activity': petroleum_activity
},
'Natural Gas': {
'energy': {
'primary': natgas_energy
},
'activity': natgas_activity
},
'Other Gasses': {
'energy': {
'primary': othgas_energy
},
'activity': othgas_activity
}
}
return data_dict
def comm_sector_total(self):
"""Note: Other includes batteries, chemicals, hydrogen, pitch, purchased steam, sulfur, and miscellaneous technologies
"""
hydroelectric_energy = None #.multiply(0.001) # Table 8.4C11 column N
other_energy = None # .multiply(0.001) # Table 8.4C11 column AB
hydroelectric_activity = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2d11 Column P
other_activity = None # self.Table82d contains 'Electricity Net Generation Total (including from sources not shown), Commercial Sector',
# could get "other" by subtracting other categories from ^ ?.multiply(0.001) # Table 8.2d11 Column AD
data_dict = {
'Hydroelectric': {
'energy': {
'primary': hydroelectric_energy
},
'activity': hydroelectric_activity
},
'Other': {
'energy': {
'primary': other_energy
},
'activity': other_activity
}
}
return data_dict
def elec_power_sector_chp_renew(self):
wood_energy = self.Table85c[
self.Table85c['Description'] ==
'Wood Consumption for Electricity Generation, Electric Power Sector'].multiply(
0.001) # Table 8.5C column R # TBtu
waste_energy = self.Table85c[
self.Table85c['Description'] ==
'Waste Consumption for Electricity Generation, Electric Power Sector'].multiply(
0.001) # Table 8.5C column T # TBtu
wood_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WDEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column R
waste_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WSEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column T
data_dict = {
'Wood': {
'energy': {
'primary': wood_energy
},
'activity': wood_activity
},
'Waste': {
'energy': {
'primary': waste_energy
},
'activity': waste_activity
}
}
return data_dict
def elec_power_sector_chp_fossil(self):
coal_energy = self.coal_reconcile()
petroleum_energy = self.petroleum_reconcile()
natgas_energy = self.natgas_reconcile()
othgas_energy = self.othgas_reconcile()
coal_activity = self.elec_power_eia.eia_api(
id_='TOTAL.CLEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column B
petroleum_activity = self.elec_power_eia.eia_api(
id_='TOTAL.PAEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column D
natgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.NGEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column F
othgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.OJEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2c column H
data_dict = {
'Coal': {
'energy': {
'primary': coal_energy
},
'activity': coal_activity
},
'Petroleum': {
'energy': {
'primary': petroleum_energy
},
'activity': petroleum_activity
},
'Natural Gas': {
'energy': {
'primary': natgas_energy
},
'activity': natgas_activity
},
'Other Gasses': {
'energy': {
'primary': othgas_energy
},
'activity': othgas_activity
}
}
return data_dict
def elec_power_sector_chp_total(self):
other_energy = self.Table85c[
self.Table85c['Description'] ==
'Other Consumption for Electricity Generation, Electric Power Sector'].multiply(
0.001) # Table 8.5c column V
other_activty = None # self.elec_power_eia.eia_api(id_='', id_type='series').multiply(0.001) # Table 8.2c columnAD
data_dict = {
'Other': {
'energy': {
'primary': other_energy
},
'activity': other_activty
}
}
return data_dict
def electricity_only_renew(self):
wood_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WDL1BUS.A',
id_type='series').multiply(0.001) # Table 8.3d I, 8.5C R
waste_activity = self.elec_power_eia.eia_api(
id_='TOTAL.WSL1BUS.A',
id_type='series').multiply(0.001) # Table 8.3d J, 8.5C T
geothermal_activity = self.elec_power_eia.eia_api(
id_='TOTAL.GEEGBUS.A',
id_type='series').multiply(0.001) # Table 8.4 T
solar_activity = self.elec_power_eia.eia_api(
id_='TOTAL.SOEGBUS.A',
id_type='series').multiply(0.001) # Table 8.4 V
wind_activity = self.elec_power_eia.eia_api(id_='TOTAL.WYEGBUS.A',
id_type='series').multiply(
0.001) # Table 8.4 X
wood_energy = self.elec_power_eia.eia_api(
id_='TOTAL.WDEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2b S, 8.2c R
waste_energy = self.elec_power_eia.eia_api(
id_='TOTAL.WSEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2b T, 8.2c T
geothermal_energy = self.elec_power_eia.eia_api(
id_='TOTAL.GEEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2b V, 8.2c V
solar_energy = self.elec_power_eia.eia_api(
id_='TOTAL.SOEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2b X, 8.2c X
wind_energy = self.elec_power_eia.eia_api(
id_='TOTAL.WYEGPUS.A',
id_type='series').multiply(0.001) # Table 8.2b Z, 8.2c Z
data_dict = {
'Wood': {
'energy': {
'primary': wood_energy
},
'activity': wood_activity
},
'Waste': {
'energy': {
'primary': waste_energy
},
'activity': waste_activity
},
'Geothermal': {
'energy': {
'primary': geothermal_energy
},
'activity': geothermal_activity
},
'Solar': {
'energy': {
'primary': solar_energy
},
'activity': solar_activity
},
'Wind': {
'energy': {
'primary': wind_energy
},
'activity': wind_activity
}
}
return data_dict
def electricity_only_fossil(self):
coal_energy = self.coal_reconcile()
petroleum_energy = self.petroleum_reconcile()
natgas_energy = self.natgas_reconcile()
othgas_energy = self.othgas_reconcile()
coal_activity = self.elec_power_eia.eia_api(
id_='TOTAL.CLEGPUS.A',
id_type='series').multiply(0.001) # 8.2b B, 8.2c B
petroleum_activity = self.elec_power_eia.eia_api(
id_='TOTAL.PAEGPUS.A',
id_type='series').multiply(0.001) # 8.2b D, 8.2c D
natgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.NGEGPUS.A',
id_type='series').multiply(0.001) # 8.2b B, 8.2c F
othgas_activity = self.elec_power_eia.eia_api(
id_='TOTAL.OJEGPUS.A', id_type='series').multiply(0.001) # 8.2c H
data_dict = {
'Coal': {
'energy': {
'primary': coal_energy
},
'activity': coal_activity
},
'Petroleum': {
'energy': {
'primary': petroleum_energy
},
'activity': petroleum_activity
},
'Natural Gas': {
'energy': {
'primary': natgas_energy
},
'activity': natgas_activity
},
'Other Gasses': {
'energy': {
'primary': othgas_energy
},
'activity': othgas_activity
}
}
return data_dict
def electricity_only_total(self):
nuclear_energy = self.elec_power_eia.eia_api(
id_='TOTAL.NUEGBUS.A', id_type='series').multiply(0.001) # 8.4b L
hydroelectric_energy = self.elec_power_eia.eia_api(
id_='TOTAL.HVEGBUS.A', id_type='series').multiply(0.001) # 8.4b N
nuclear_activity = self.elec_power_eia.eia_api(
id_='TOTAL.NUEGPUS.A', id_type='series').multiply(0.001) # 8.2b L
hydroelectric_activity = self.elec_power_eia.eia_api(
id_='TOTAL.HVEGPUS.A', id_type='series').multiply(0.001) # 8.2b O
data_dict = {
'Hydro Electric': {
'energy': {
'primary': hydroelectric_energy
},
'activity': hydroelectric_activity
},
'Nuclear': {
'energy': {
'primary': nuclear_energy
},
'activity': nuclear_activity
}
}
return data_dict
def collect_data(self):
industrial_sector_chp_renew = self.industrial_sector_chp_renew()
industrial_sector_chp_fossil = self.industrial_sector_chp_fossil()
industrial_sector_total = self.industrial_sector_total()
industrial_sector_total['Fossil Fuels'] = industrial_sector_chp_fossil
industrial_sector_total['Renewable'] = industrial_sector_chp_renew
comm_sector_chp_renew = self.comm_sector_chp_renew()
comm_sector_chp_fossil = self.comm_sector_chp_fossil()
comm_sector_total = self.comm_sector_total()
comm_sector_total['Fossil Fuels'] = comm_sector_chp_fossil
comm_sector_total['Renewable'] = comm_sector_chp_renew
elec_power_sector_chp_renew = self.elec_power_sector_chp_renew()
elec_power_sector_chp_fossil = self.elec_power_sector_chp_fossil()
elec_power_sector_chp_total = self.elec_power_sector_chp_total()
elec_power_sector_chp_total[
'Fossil Fuels'] = elec_power_sector_chp_fossil
elec_power_sector_chp_total['Renewable'] = elec_power_sector_chp_renew
all_chp = dict()
all_chp['Industrial Sector'] = industrial_sector_total
all_chp['Commercial Sector'] = comm_sector_total
all_chp['Elec Power Sector'] = elec_power_sector_chp_total
electricity_only_renew = self.electricity_only_renew()
electricity_only_fossil = self.electricity_only_fossil()
electricity_only_total = self.electricity_only_total()
electricity_only_total['Fossil Fuels'] = electricity_only_fossil
electricity_only_total['Renewable'] = electricity_only_total
chp_elec_power_sector = dict()
chp_renew = self.chp_renew()
chp_fossil = self.chp_fossil()
chp_elec_power_sector['Fossil Fuels'] = chp_fossil
chp_elec_power_sector['Renewable'] = chp_renew
elec_power_sector = dict()
elec_power_sector['Electricity Only'] = electricity_only_total
elec_power_sector['Combined Heat & Power'] = chp_elec_power_sector
elec_generation_total = dict(
) # Need to build this from commerical and industrial_totals
elec_generation_total['Elec Power Sector'] = elec_power_sector
elec_generation_total['Commercial Sector'] = comm_sector_total
elec_generation_total['Industrial Sector'] = industrial_sector_total
data_dict = {
'Elec Generation Total': elec_generation_total,
'All CHP': all_chp
}
return data_dict
def main(self, breakout, save_breakout, calculate_lmdi):
data_dict = self.collect_data()
print(data_dict)
results_dict, formatted_results = self.get_nested_lmdi(
level_of_aggregation=self.level_of_aggregation,
breakout=breakout,
save_breakout=save_breakout,
calculate_lmdi=calculate_lmdi,
raw_data=data_dict)
class CommercialIndicators(CalculateLMDI):
"""
Data Sources:
- New construction is based on data from Dodge Data and Analytics. Dodge data on new floor space additions is available
from the published versions of the Statistical Abstract of the United States (SAUS). The Most recent data is from the 2020
SAUS, Table 995 "Construction Contracts Started- Value of the Construction and Floor Space of Buildings by Class of Construction:
2014 to 2018".
"""
def __init__(self,
directory,
output_directory,
level_of_aggregation,
lmdi_model=['multiplicative'],
end_year=2018,
base_year=1985):
self.end_year = end_year
self.sub_categories_list = {
'Commercial_Total': None
} #, 'Total_Commercial_LMDI_UtilAdj': None}
self.eia_comm = GetEIAData('commercial')
self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj']
super().__init__(sector='commercial',
level_of_aggregation=level_of_aggregation,
lmdi_models=lmdi_model,
directory=directory,
output_directory=output_directory,
categories_dict=self.sub_categories_list,
energy_types=self.energy_types,
base_year=base_year,
end_year=end_year)
# self.cbecs =
# self.residential_housing_units = [0] # Use regional estimates of residential housing units as interpolator, extrapolator via regression model
# self.mer_data23_May_2016 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251'
# self.mer_data23_Jan_2017 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251'
# self.mer_data23_Dec_2019 = GetEIAData.eia_api(id_='711251') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711251'
# self.AER11_Table21C_Update = GetEIAData.eia_api(id_='711251') # Estimates?
def collect_input_data(self, dataset_name):
datasets = \
{'national_calibration':
self.eia_comm.national_calibration(),
'SEDS_CensusRgn':
self.eia_comm.get_seds(),
'mer_data_23':
self.eia_comm.eia_api(id_='711251', id_type='category')}
return datasets[dataset_name]
def adjusted_supplier_data(self):
"""
This worksheet adjusts some of commercial energy consumption data
as reported in the Annual Energy Review. These adjustments are
based upon state-by-state analysis of energy consumption in the
industrial and commercial sectors. For electricity, there have been
a number of reclassifications by utilities since 1990 that has moved
sales from the industrial sector to the commercial sector.
The adjustment for electricity consumption is based upon a
state-by-state examination of commercial and electricity
sales from 1990 through 2011. This data is collected
by EIA via Survey EIA-861. Significant discontinuities
in the sales data from one year to the next were removed.
In most cases, these adjustments caused industrial consumption
to increase and commercial consumption to decrease. The
spreadsheet with these adjustments is
Sectoral_reclassification5.xls (10/25/2012).
In 2009, there was a significant decline in commercial
electricity sales in MA and a corresponding increase in industrial
sales
Assuming that industrial consumption would have
fallen by 2% between 2008 and 2009, the adjustment
to both the commercial (+) and industrial sectors (-) was
estimated to be 7.61 TWh. .
The 7.61 TWh converted to Tbtu is 26.0. This value is then added
to the negative 164.0 Tbtu in 2009 and subsequent years.
State Energy Data System (Jan. 2017) via National Calibration worksheet
"""
# 1949-1969
published_consumption_trillion_btu = \
self.eia_comm.eia_api(id_='TOTAL.ESCCBUS.A', id_type='series') # Column W (electricity retail sales to the commercial sector) # for years 1949-69
published_consumption_trillion_btu = \
published_consumption_trillion_btu.rename(
columns={'Electricity Retail Sales to the Commercial Sector, Annual, Trillion Btu':
'published_consumption_trillion_btu'})
# 1970-2018
national_calibration = self.collect_input_data('national_calibration')
published_consumption_trillion_btu.loc['1970':, [
'published_consumption_trillion_btu'
]] = national_calibration.loc['1970':, [
'Final Est. (Trillion Btu)_elec'
]].values # Column G (electricity final est) # for years 1970-2018
# 1977-1989
years = list(
range(
1977,
max(published_consumption_trillion_btu.index.astype(int)) + 1))
years = [str(y) for y in years]
# adjustment_to_commercial_trillion_btu_early = number_for_1990
adjustment_to_commercial_trillion_btu = \
[9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975,
9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975,
9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340312799975, 9.21340654000005,
29.77918535999970, 10.21012680399960, 1.70263235599987, -40.63866012000020, -40.63865670799990,
-117.72073870000000, -117.72073528800000, -117.72073187600000, -117.72072846400000, -162.61452790400100,
-136.25241618800100, -108.91594645600000, -125.97594304400000, -125.97593963200100, -163.95020989600000,
-163.95020648400000, -163.95020307200000, -137.98708428968000, -137.98487966000100, -137.98487966000100,
-137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100, -137.98487966000100,
-137.98487966000100, -137.98487966000100] # First value is for 1977 - 2018
adjustment_df = \
pd.DataFrame([years, adjustment_to_commercial_trillion_btu]).transpose()
adjustment_df.columns = \
['Year', 'adjustment_to_commercial_trillion_btu']
adjusted_supplier_data = \
adjustment_df.merge(published_consumption_trillion_btu, how='outer', on='Year')
adjusted_supplier_data['adjustment_to_commercial_trillion_btu'] = \
adjusted_supplier_data['adjustment_to_commercial_trillion_btu'].fillna(0)
adjusted_supplier_data = adjusted_supplier_data.set_index('Year')
adjusted_supplier_data['adjusted_consumption_trillion_btu'] = \
adjusted_supplier_data['adjustment_to_commercial_trillion_btu'].add(adjusted_supplier_data['published_consumption_trillion_btu'])
adjusted_supplier_data['adjusted_consumption_trillion_btu'] = \
adjusted_supplier_data['adjusted_consumption_trillion_btu'].astype(float)
adjusted_supplier_data = \
adjusted_supplier_data.sort_index(ascending=True)
return adjusted_supplier_data[['adjusted_consumption_trillion_btu']]
@staticmethod
def get_saus():
"""Get Data from the Statistical Abstract of the United States (SAUS)
"""
print('os.getcwd():', os.getcwd())
try:
saus_2002 = \
pd.read_csv('./EnergyIntensityIndicators/Data/SAUS2002_table995.csv').set_index('Year')
except FileNotFoundError:
os.chdir('..')
saus_2002 = \
pd.read_csv('./EnergyIntensityIndicators/Data/SAUS2002_table995.csv').set_index('Year')
saus_1994 = {
1980: 738,
1981: 787,
1982: 631,
1983: 716,
1984: 901,
1985: 1039,
1986: 960,
1987: 933,
1988: 883,
1989: 867,
1990: 694,
1991: 477,
1992: 462,
1993: 479
}
saus_2001 = {
1980: 738,
1981: None,
1982: None,
1983: None,
1984: None,
1985: 1039,
1986: None,
1987: None,
1988: None,
1989: 867,
1990: 694,
1991: 476,
1992: 462,
1993: 481,
1994: 600,
1995: 700,
1996: 723,
1997: 855,
1998: 1106,
1999: 1117,
2000: 1176
}
saus_merged = dict()
for (year, value) in saus_2001.items():
if value == None:
set_value = saus_1994[year]
else:
set_value = value
saus_merged[year] = set_value
saus_merged_df = \
pd.DataFrame.from_dict(
saus_merged, orient='index', columns=['Value'])
return saus_2002, saus_merged_df
@staticmethod
def dod_compare_old():
"""
DODCompareOld Note from PNNL (David B. Belzer):
"These series are of unknown origin--need to check Jackson and Johnson 197 (sic)?
"""
dod_old = pd.read_csv(
'./EnergyIntensityIndicators/Data/DODCompareOld.csv').set_index(
'Year')
cols_list = ['Retail', 'Auto R', 'Office', 'Warehouse']
dod_old['Commercial'] = dod_old[cols_list].sum(axis=1)
dod_old_subset = dod_old.loc[list(range(1960, 1982)), cols_list]
dod_old_hotel = dod_old.loc[list(range(1980, 1990)), ['Commercial']]
return dod_old, dod_old_subset, dod_old_hotel
def dodge_adjustment_ratios(self, dodge_dataframe, start_year, stop_year,
adjust_years, late):
"""(1985, 1990) or (1960, 1970)
"""
year_indices = self.years_to_str(start_year, stop_year)
revision_factor_commercial = dodge_dataframe.loc[year_indices,
['Commercial']].sum(
axis=0).values
categories = ['Retail', 'Auto R', 'Office', 'Warehouse', 'Hotel']
if late:
col = 'Commercial'
else:
col = 'Commercial, Excl Hotel'
for category in categories:
revision_factor_cat = dodge_dataframe.loc[
year_indices,
[category]].sum(axis=0).values / revision_factor_commercial
dodge_dataframe.loc[adjust_years,
[category]] = dodge_dataframe.loc[
adjust_years,
[col]].values * revision_factor_cat[0]
return dodge_dataframe
def west_inflation(self):
"""Jackson and Johnson Estimate of West Census Region Inflation Factor, West Region Shares Based on CBECS
Note from PNNL: "Staff: Based upon CBECS, the percentage of construction in west census region was slightly greater
in the 1900-1919 period than in 1920-1945. Thus, factor is set to 1.12, approximately same as 1925 value published
by Jackson and Johnson"
"""
# hist_stat = self.hist_stat()['Commercial (million SF)'] # hist_stat column E
# # west inflation column Q
ornl_78 = {
1925: 1.127,
1930: 1.144,
1935: 1.12,
1940: 1.182,
1945: 1.393,
1950: 1.216,
1951: 1.237,
1952: 1.224,
1953: 1.209,
1954: 1.213,
1955: 1.229
}
all_years = list(range(min(ornl_78.keys()), max(ornl_78.keys()) + 1))
increment_years = list(ornl_78.keys())
final_factors = {year: 1.12 for year in list(range(1919, 1925))}
for index, y_ in enumerate(increment_years):
if index > 0:
year_before = increment_years[index - 1]
num_years = y_ - year_before
infl_factor_year_before = ornl_78[year_before]
infl_factor_y_ = ornl_78[y_]
increment = 1 / num_years
for delta in range(num_years):
value = infl_factor_year_before * (1 - increment * delta) + \
infl_factor_y_ * (increment * delta)
year = year_before + delta
final_factors[year] = value
final_factors_df = pd.DataFrame.from_dict(final_factors,
columns=['Final Factors'],
orient='index')
return final_factors_df
@staticmethod
def years_to_str(start_year, end_year):
"""Create list of year strings from start_year and end_year range
"""
list_ = list(range(start_year, end_year + 1))
return [str(l) for l in list_]
def hist_stat(self):
"""Historical Dodge Data through 1970
Data Source: Series N 90-100 Historical Statistics of the U.S., Colonial Times to 1970
"""
historical_dodge = pd.read_csv(
'./EnergyIntensityIndicators/Data/historical_dodge_data.csv'
).set_index('Year')
pub_inst_values = historical_dodge.loc[list(range(1919, 1925)),
['Pub&Institutional']].values
total_1925_6 = pd.DataFrame.sum(
historical_dodge.loc[list(range(1925, 1927)), ],
axis=0).drop(index='Commercial (million SF)')
inst_pub_total_1925 = pd.DataFrame.sum(
historical_dodge.loc[1925, ].drop('Commercial (million SF)'),
axis=0)
inst_pub_total_1926 = pd.DataFrame.sum(
historical_dodge.loc[1926, ].drop('Commercial (million SF)'),
axis=0)
inst_pub_total_1925_6 = inst_pub_total_1925 + inst_pub_total_1926
shares = total_1925_6.divide(inst_pub_total_1925_6)
for col in list(total_1925_6.index):
values = historical_dodge.loc[list(range(1919, 1925)),
['Pub&Institutional']].multiply(
shares[col]).values
historical_dodge.at[list(range(1919, 1925)), col] = values
historical_dodge.at[list(range(1919, 1925)),
['Pub&Institutional']] = pub_inst_values
return historical_dodge
def hist_stat_adj(self):
"""Adjust historical Dodge data to account for
omission of data for the West Census Region prior to 1956
"""
hist_data = self.hist_stat()
west_inflation = self.west_inflation()
hist_data = hist_data.merge(west_inflation,
how='outer',
left_index=True,
right_index=True)
hist_data['Final Factors'] = hist_data['Final Factors'].fillna(1)
adjusted_for_west = hist_data.drop(
columns=['Final Factors', 'Pub&Institutional']).multiply(
hist_data['Final Factors'].values, axis=0)
return adjusted_for_west.loc[list(range(1919, 1960)), :]
def dodge_revised(self):
"""Dodge Additions, adjusted for omission of
West Census Region prior to 1956
"""
saus_2002, saus_merged = self.get_saus()
dod_old, dod_old_subset, dod_old_hotel = self.dod_compare_old()
west_inflation = self.hist_stat_adj()
dodge_revised = pd.read_csv(
'./EnergyIntensityIndicators/Data/Dodge_Data.csv').set_index(
'Year')
dodge_revised.index = dodge_revised.index.astype(str)
dodge_revised = dodge_revised.reindex(
dodge_revised.columns.tolist() +
['Commercial, Excl Hotel', 'Hotel'],
axis=1).fillna(np.nan)
years_1919_1989 = self.years_to_str(1919, 1990)
years_1990_1997 = self.years_to_str(1990, 1997)
dodge_revised.loc[self.years_to_str(1960, 1981),
['Retail', 'Auto R', 'Office', 'Warehouse'
]] = dod_old_subset.values
dodge_revised.loc[self.years_to_str(1919, 1959),
['Commercial, Excl Hotel']] = west_inflation[
'Commercial (million SF)'].values.reshape(
41, 1) # hist_stat_adj column Q
hist_adj_cols = [
'Education', 'Hospital', 'Public', 'Religious', 'Soc/Amuse', 'Misc'
]
dodge_revised.loc[self.years_to_str(1919, 1959),
hist_adj_cols] = west_inflation.drop(
'Commercial (million SF)', axis=1).values
dodge_revised.loc[self.years_to_str(1990, 1998),
hist_adj_cols] = saus_2002.loc[
self.years_to_str(1990, 1998), [
'Educational', 'Health', 'Pub. Bldg',
'Religious', 'Soc/Rec', 'Misc.'
]].values
dodge_revised.loc[self.years_to_str(1990, 2003),
['Soc/Misc']] = saus_2002.loc[
self.years_to_str(1990, 2001), ['Soc/Rec']].add(
saus_2002.loc[self.years_to_str(1990, 2001),
['Misc.']].values)
dodge_revised.loc[self.years_to_str(1999, 2001),
'Misc'] = saus_2002.loc[
self.years_to_str(1999, 2001),
['Misc.']].values.reshape(3, )
dodge_revised.loc[self.years_to_str(1961, 1989),
'Misc'] = dodge_revised.loc[
self.years_to_str(1961, 1989),
'Soc/Misc'].subtract(dodge_revised.loc[
self.years_to_str(1961, 1989),
'Soc/Amuse'].values)
dodge_revised.loc[str(2000), 'Hospital'] = saus_2002.loc[str(2000),
'Health']
dodge_revised.loc[self.years_to_str(1960, 1989),
['Commercial, Excl Hotel']] = dodge_revised.loc[
self.years_to_str(1960, 1989),
['Retail', 'Auto R', 'Office', 'Warehouse']].sum(
axis=1)
hotel_80_89 = saus_merged.loc[list(range(1980, 1989 + 1)),
['Value']].subtract(dod_old_hotel.values)
dodge_revised.loc[self.years_to_str(1980, 1989),
['Hotel']] = hotel_80_89
hotel_80_89_ratio = hotel_80_89.sum(axis=0).values / dodge_revised.loc[
self.years_to_str(1980, 1989),
['Commercial, Excl Hotel']].sum(axis=0).values
dodge_revised.loc[
self.years_to_str(1919, 1979), ['Hotel']] = dodge_revised.loc[
self.years_to_str(1919, 1979),
['Commercial, Excl Hotel']].values * hotel_80_89_ratio
dodge_revised.loc[years_1990_1997,
['Commercial, Incl Hotel']] = saus_2002.loc[
years_1990_1997, ['Commercial']].values
dodge_revised.loc[self.years_to_str(1985, 1989),
['Commercial']] = saus_merged.loc[
list(range(1985, 1989 + 1)), ['Value']].values
dodge_revised.loc[self.years_to_str(1990, 2018),
['Commercial']] = dodge_revised.loc[
self.years_to_str(1990, 2018),
['Commercial, Incl Hotel']].values
dodge_revised = self.dodge_adjustment_ratios(
dodge_revised,
1960,
1969,
adjust_years=self.years_to_str(1919, 1959),
late=False)
dodge_revised = self.dodge_adjustment_ratios(
dodge_revised,
1985,
1989,
adjust_years=self.years_to_str(1990, 2018),
late=True)
dodge_revised.loc[years_1919_1989,
['Commercial, Incl Hotel']] = dodge_revised.loc[
years_1919_1989, ['Commercial, Excl Hotel']].add(
dodge_revised.loc[years_1919_1989,
['Hotel']].values)
dodge_revised['Total'] = dodge_revised.drop(
['Commercial, Incl Hotel', 'Commercial, Excl Hotel'],
axis=1).sum(axis=1).values
return dodge_revised
def dodge_to_cbecs(self):
"""Redefine the Dodge building categories more along the lines of CBECS categories. Constant fractions of floor space are moved among categories.
Returns:
dodge_to_cbecs (dataframe): redefined data
"""
# Key Assumptions:
education_floor_space_office = .10
auto_repair_retail = .80
retail_merc_service = .80 # remainder to food service and sales
retail_merc_service_food_sales = .11
retail_merc_service_food_service = .90
education_assembly = .05
education_misc = .05 # (laboratories)
health_transfered_to_cbecs_health = .75 # 25% to lodging (nursing homes)
misc_public_assembly = .10 # (passenger terminals)
dodge_revised = self.dodge_revised() # dataframe
dodge_to_cbecs = pd.DataFrame(dodge_revised[[
'Total', 'Religious', 'Warehouse'
]]).rename(columns={'Total': 'Dodge_Totals'})
dodge_to_cbecs['Office'] = dodge_revised[
'Office'] + education_floor_space_office * dodge_revised[
'Education']
dodge_to_cbecs['Merc/Serv'] = retail_merc_service * (
dodge_revised['Retail'] +
auto_repair_retail * dodge_revised['Auto R'])
dodge_to_cbecs['Food_Sales'] = retail_merc_service_food_sales * (
dodge_revised['Retail'] +
auto_repair_retail * dodge_revised['Auto R'])
dodge_to_cbecs['Food_Serv'] = retail_merc_service_food_service * (
dodge_revised['Retail'] +
auto_repair_retail * dodge_revised['Auto R'])
dodge_to_cbecs['Education'] = (
1 - education_floor_space_office - education_assembly -
education_misc) * dodge_revised['Education']
dodge_to_cbecs[
'Health'] = health_transfered_to_cbecs_health * dodge_revised[
'Hospital']
dodge_to_cbecs['Lodging'] = dodge_revised['Hotel'] + (
1 - health_transfered_to_cbecs_health) * dodge_revised['Hospital']
dodge_to_cbecs['Assembly'] = dodge_revised[
'Soc/Amuse'] + misc_public_assembly * dodge_revised[
'Misc'] + education_assembly * dodge_revised['Education']
dodge_to_cbecs['Other'] = dodge_revised['Public'] + (
1 - misc_public_assembly) * dodge_revised['Misc'] + (
1 - auto_repair_retail) * dodge_revised[
'Auto R'] + education_misc * dodge_revised['Education']
dodge_to_cbecs['Redefined_Totals'] = dodge_to_cbecs.drop(
'Dodge_Totals', axis=1).sum(axis=1).values
# dodge_to_cbecs = dodge_to_cbecs.drop() # don't need totals?
return dodge_to_cbecs
def nems_logistic(self, dataframe, params):
"""
PNNL errors found:
- Column S in spreadsheet has CO-StatePop2.xls are incorrectly aligned with years
- Column AL does not actually scale by 1.28 as suggested in the column header
"""
current_year = dt.datetime.now().year
link_factors = pd.read_excel(
f'{self.directory}/CO-EST_statepop2.xls',
sheet_name='Stock',
usecols='D:E',
header=1,
skiprows=158).rename(columns={1789: 'Year'})
state_pop = link_factors.set_index('Year').rename(
columns={' New': 'state_pop'})
state_pop = state_pop[state_pop.index.notnull()]
state_pop.index = state_pop.index.astype(int)
state_pop.index = state_pop.index.astype(str)
dataframe = dataframe.merge(state_pop,
how='outer',
left_index=True,
right_index=True)
dataframe = dataframe[dataframe.index.notnull()]
dataframe = dataframe.reindex(columns=dataframe.columns.tolist() + [
'adjusted_state_pop', 'adjusted_state_pop_scaled_b',
'adjusted_state_pop_scaled_c', 'scaled_additions_estimate_a',
'scaled_additions_estimate_b', 'scaled_additions_estimate_c',
'removal', 'adjusted_removals', 'old_stk_retain', 'floorspace_bsf'
])
dataframe['Year_Int'] = dataframe.index.astype(int)
dataframe['age'] = dataframe['Year_Int'].subtract(
current_year).multiply(-1)
dataframe['remaining'] = ((dataframe['age'].divide(params[1])).pow(
params[0]).add(1)).pow(-1)
dataframe['inflate_fac'] = dataframe['remaining'].pow(-1)
link_factor = 0.1
adjusted_state_pop_1 = 40
timing_wgts_current_yr = 0.4
timing_wgts_lag_yr = 0.6
benchmark_factor = 1
dataframe.loc[str(1838), ['state_pop']] = 400
dataframe.loc[self.years_to_str(1838, 1919),
['adjusted_state_pop']] = dataframe.loc[
self.years_to_str(1838, 1919),
['state_pop']].values * link_factor
dataframe.loc[self.years_to_str(1920, current_year),
['adjusted_state_pop']] = dataframe.loc[
self.years_to_str(1920, current_year),
['Redefined_Totals']].values
for year in self.years_to_str(1838, current_year):
adjusted_state_pop_value = dataframe.loc[
year, ['adjusted_state_pop']].values
if year == '1838':
vpip_estimate = adjusted_state_pop_value
elif year == '1920':
vpip_estimate = adjusted_state_pop_value
else:
adjusted_state_pop_year_before = dataframe.loc[
str(int(year) - 1), ['adjusted_state_pop']].values
vpip_estimate = (
timing_wgts_current_yr * adjusted_state_pop_value +
timing_wgts_lag_yr *
adjusted_state_pop_year_before) * benchmark_factor
dataframe.loc[year, 'VPIP-Estimate'] = vpip_estimate
_variable = 1.2569 # This should be solved for
x_column_value = _variable
db_estimates = 1.2
db_estimates2 = [1.25 - 0.01 * d for d in list(range(1990, 2021))]
post_1989_scaling_factor = db_estimates # Should choose this
variable_2 = 1.533 # This should be solved for
without_lags = dataframe.loc[self.years_to_str(1990, current_year),
['adjusted_state_pop']].multiply(
post_1989_scaling_factor)
dataframe.loc[:str(1989),
['scaled_additions_estimate_a']] = dataframe.loc[:str(
1989), ['VPIP-Estimate']].values * variable_2
dataframe.loc[self.years_to_str(1990, current_year),
['scaled_additions_estimate_a']] = dataframe.loc[
self.years_to_str(1990, current_year),
['VPIP-Estimate']].values * post_1989_scaling_factor
dataframe.loc[:str(1989),
['adjusted_state_pop_scaled_b']] = dataframe.loc[
self.years_to_str(1790, 1989),
['scaled_additions_estimate_a']].values
dataframe.loc[self.years_to_str(1990, 2001),
['adjusted_state_pop_scaled_b']] = dataframe.loc[
self.years_to_str(1990, 2001),
['scaled_additions_estimate_a']].values * 1.15
dataframe.loc[self.years_to_str(2002, current_year),
['adjusted_state_pop_scaled_b']] = dataframe.loc[
self.years_to_str(2002, current_year),
['scaled_additions_estimate_a']].values
dataframe.loc[str(1790), ['adjusted_state_pop_scaled_c']] = 1
dataframe.loc[self.years_to_str(1791, 1989),
['adjusted_state_pop_scaled_c']] = dataframe.loc[
self.years_to_str(1791, 1989),
['scaled_additions_estimate_a']].values
dataframe.loc[self.years_to_str(1990, 2001),
['adjusted_state_pop_scaled_c']] = dataframe.loc[
self.years_to_str(1990, 2001),
['scaled_additions_estimate_a']].values * 1.28
dataframe.loc[self.years_to_str(2002, current_year),
['adjusted_state_pop_scaled_c']] = dataframe.loc[
self.years_to_str(2002, current_year),
['scaled_additions_estimate_a']].values
for y in self.years_to_str(1839, current_year):
years_diff = current_year - 1870
start_year = int(y) - years_diff
# first_index_year = int(dataframe.index[0])
# if start_year < first_index_year:
# start_year = first_index_year
year_index = self.years_to_str(start_year, int(y))
remaining = dataframe.loc[self.years_to_str(1870, current_year),
['remaining']].values.flatten()
adjusted_state_pop_scaled_b = dataframe.loc[
year_index,
['adjusted_state_pop_scaled_b']].fillna(0).values.flatten()
adjusted_state_pop_scaled_c = dataframe.loc[
year_index,
['adjusted_state_pop_scaled_c']].fillna(0).values.flatten()
b_value = np.dot(adjusted_state_pop_scaled_b, remaining)
c_value = np.dot(adjusted_state_pop_scaled_c, remaining)
dataframe.loc[y, ['scaled_additions_estimate_b']] = b_value
dataframe.loc[y, ['scaled_additions_estimate_c']] = c_value
removal_chg = 1 # Not sure what this is about
fractions = [0.3, 0.4, 0.4, 0.35, 0.35, 0.35, 0.35, 0.3, 0.3, 0.3]
fraction_retained = [f * removal_chg for f in fractions]
for i in dataframe.index:
if i >= '1870':
removal = dataframe.loc[
i, ['scaled_additions_estimate_c']].values - dataframe.loc[
str(int(i) - 1),
['scaled_additions_estimate_c']].values - dataframe.loc[
i, ['scaled_additions_estimate_a']].values
dataframe.loc[i, ['removal']] = removal
dataframe.loc[self.years_to_str(2009, 2009 + len(fractions) - 1),
['adjusted_removals']] = dataframe.loc[
self.years_to_str(2009, 2009 + len(fractions) - 1),
['removal']].values.flatten() * fraction_retained
for y_ in list(range(2009, 2009 + len(fractions))):
if y_ == 2009:
dataframe.loc[str(y_), ['old_stk_retain']] = dataframe.loc[
str(y_), ['adjusted_removals']]
else:
dataframe.loc[str(y_), ['old_stk_retain']] = dataframe.loc[
str(y_ - 1), ['old_stk_retain']].values + dataframe.loc[
str(y_), ['adjusted_removals']].values
dataframe['adjusted_removals'] = dataframe['adjusted_removals'].fillna(
0)
dataframe.loc[
self.years_to_str(1960, current_year),
['floorspace_bsf']] = dataframe.loc[
self.years_to_str(1960, current_year),
['scaled_additions_estimate_c']].values - dataframe.loc[
self.years_to_str(1960, current_year),
['adjusted_removals']].values
return dataframe[['floorspace_bsf']].dropna()
def solve_logistic(self, dataframe):
"""Solve NES logistic parameters
"""
pnnl_coefficients = [3.92276415015621,
73.2238120168849] # [gamma, lifetime]
# popt, pcov = curve_fit(self.nems_logistic, xdata=dataframe[], ydata=dataframe[] , p0=pnnl_coefficients)
# return popt
return pnnl_coefficients
def activity(self):
"""Use logistic parameters to find predicted historical floorspace
Returns:
historical_floorspace_billion_sq_feet (pd.DataFrame): historical floorspace
in the Commercial Sector.
Years:
Units: Billion Square Feet
Data Source:
"""
dodge_to_cbecs = self.dodge_to_cbecs(
) # columns c-m starting with year 1920 (row 17)
coeffs = self.solve_logistic(dodge_to_cbecs)
historical_floorspace_late = self.nems_logistic(
dodge_to_cbecs, coeffs) # properly formatted?
historical_floorspace_early = {
1949: 27235.1487296062,
1950: 27788.6370796569,
1951: 28246.642791733,
1952: 28701.4989706012,
1953: 29253.2282427217,
1954: 29913.8330998026,
1955: 30679.7157232176,
1956: 31512.6191323126,
1957: 32345.382764321,
1958: 33206.8483392728,
1959: 34088.6640247816
}
historical_floorspace_early = pd.DataFrame.from_dict(
historical_floorspace_early,
columns=['floorspace_bsf'],
orient='index')
historical_floorspace_early.index = historical_floorspace_early.index.astype(
str)
historical_floorspace = pd.concat(
[historical_floorspace_early, historical_floorspace_late])
historical_floorspace_billion_sq_feet = historical_floorspace.multiply(
0.001)
return historical_floorspace_billion_sq_feet
def fuel_electricity_consumption(self):
"""
Trillion Btu
Returns:
energy_data (dict): Dictionary of dataframes
with keys 'elec' and 'fuels'
"""
year_range = list(range(1949, 1970))
year_range = [str(y) for y in year_range]
national_calibration = self.collect_input_data('national_calibration')
total_primary_energy_consumption = \
self.eia_comm.eia_api(id_='TOTAL.TXCCBUS.A',
id_type='series') # pre 1969: AER table 2.1c update column U
total_primary_energy_consumption = \
total_primary_energy_consumption.rename(
columns={'Total Primary Energy Consumed by the Commercial Sector, Annual, Trillion Btu': 'total_primary'})
# total_primary_energy_consumption = total_primary_energy_consumption[total_primary_energy_consumption.index.isin(year_range)]
# total_primary_energy_consumption = total_primary_energy_consumption.multiply(0.001)
fuels_dataframe = total_primary_energy_consumption.copy()
replacement_data = national_calibration.loc['1970':, [
'Final Est. (Trillion Btu)_fuels'
]] # >= 1970: National Calibration Column 0
fuels_dataframe.loc['1970':,
['total_primary']] = replacement_data.values
fuels_dataframe = fuels_dataframe.rename(
columns={'total_primary': 'adjusted_consumption_trillion_btu'})
fuels_dataframe['adjusted_consumption_trillion_btu'] = fuels_dataframe[
'adjusted_consumption_trillion_btu'].astype(float)
elec_dataframe = self.adjusted_supplier_data()
energy_data = {'elec': elec_dataframe, 'fuels': fuels_dataframe}
return energy_data
def get_seds(self):
"""Collect SEDS data
Returns:
data (dict): Dictionary of dataframes
with keys 'elec' and 'fuels'
"""
seds = self.collect_input_data('SEDS_CensusRgn')
census_regions = {4: 'West', 3: 'South', 2: 'Midwest', 1: 'Northeast'}
total_fuels = seds[0].rename(columns=census_regions)
elec = seds[1].rename(columns=census_regions)
data = {'elec': elec, 'fuels': total_fuels}
return data
def collect_weather(self, comm_activity):
"""Gather weather data for the Commercial sector
Args:
comm_activity ([type]): [description]
Returns:
weather_data (dict): [description]
"""
seds = self.get_seds()
res = ResidentialIndicators(directory=self.directory,
output_directory=self.output_directory,
base_year=self.base_year)
# residential_activity_data = res.get_floorspace()
# residential_floorspace = residential_activity_data['floorspace_square_feet']
weather = WeatherFactors(sector='commercial',
directory=self.directory,
activity_data=comm_activity)
# residential_floorspace=residential_floorspace)
weather_factors = weather.get_weather(seds_data=seds)
# weather_factors = weather.adjust_for_weather() # What should this return?? (e.g. weather factors or weather adjusted data, both?)
weather_data = dict()
for key, value in weather_factors.items():
value = value.drop('electricity_weather_factor',
axis=1,
errors='ignore')
weather_data[key] = value
return weather_data
def collect_data(self):
"""Gather decomposition input data for the Commercial sector
Returns:
data_dict (dict): Commercial Sector data input to the LMDI model
"""
# Activity: Floorspace_Estimates column U, B
# Energy: Elec --> Adjusted Supplier Data Column D
# Fuels --> AER11 Table 2.1C_Update column U, National Calibration Column O
activity_data = self.activity()
print('Activity data collected without issue')
energy_data = self.fuel_electricity_consumption()
print('Energy data collected without issue')
weather_factors = \
self.collect_weather(comm_activity=activity_data)
data_dict = {
'Commercial_Total': {
'energy': energy_data,
'activity': activity_data,
'weather_factors': weather_factors
}
}
return data_dict
def main(self, breakout, calculate_lmdi):
"""Decompose energy use for the Commercial sector
Args:
breakout ([type]): [description]
calculate_lmdi ([type]): [description]
Returns:
[type]: [description]
"""
data_dict = self.collect_data()
results_dict, formatted_results = \
self.get_nested_lmdi(level_of_aggregation=self.level_of_aggregation,
breakout=breakout,
calculate_lmdi=calculate_lmdi,
raw_data=data_dict,
lmdi_type='LMDI-I')
return results_dict
class IndustrialIndicators(CalculateLMDI):
"""Some of the specific steps to download and process the census data
on construction energy costs are explained in the following paragraphs.
The top-level census bureau website for the Economic Census is:
https://www.census.gov/programs-surveys/economic-census.html. Scroll
down the page until the words “2017 Data Tables” are found. After
clicking on that link, the user will end up at
https://www.census.gov/programs-surveys/economic-census/news-updates/updates/2017-datatables.html.
The “2017 Data Table pages” now include direct links into
data.census.gov and large ftp downloads. After clicking on pages,
the webpage https://www.census.gov/programssurveys/economic-census/data/tables.html
comes up. Scroll down this page until the entry
“Construction (NAICS Sector 23)” is found. After selecting this entry,
the user is then automatically transferred to:
https://www.census.gov/data/tables/2017/econ/economic-census/naics-sector23.html.
"""
def __init__(self,
directory,
output_directory,
level_of_aggregation=None,
lmdi_model='multiplicative',
base_year=1985,
end_year=2018,
naics_digits=3):
self.sub_categories_list = \
{'Industry':
{'Manufacturing':
{'Food and beverage and tobacco products': None,
'Textile mills and textile product mills': None,
'Apparel and leather and allied products': None,
'Wood products': None,
'Paper products': None,
'Printing and related support activities': None,
'Petroleum and coal products': None,
'Chemical products': None,
'Plastics and rubber products': None,
'Nonmetallic mineral products': None,
'Primary metals': None,
'Fabricated metal products': None,
'Machinery': None,
'Computer and electronic products': None,
'Electrical equipment, appliances, and components': None,
'Motor vehicles, bodies and trailers, and parts': None,
'Furniture and related products': None,
'Miscellaneous manufacturing': None},
'Nonmanufacturing':
{'Agriculture, Forestry & Fishing': None,
'Mining':
{'Petroleum and Natural Gas': None,
'Other Mining': None,
'Support Activities': None},
'Construction': None}}}
self.ind_eia = GetEIAData('industry')
# 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.MER_Nov19_Table24 = \
self.ind_eia.eia_api(id_='711252')
# 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.AER10_Table21d = \
self.ind_eia.eia_api(id_='711252')
# 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.AER11_Table21d_MER0816 = \
self.ind_eia.eia_api(id_='711252')
# 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.mer_dataT0204 = \
self.ind_eia.eia_api(id_='711252')
self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj']
self.naics_digits = naics_digits
super().__init__(sector='industry',
level_of_aggregation=level_of_aggregation,
lmdi_models=lmdi_model,
categories_dict=self.sub_categories_list,
energy_types=self.energy_types,
directory=directory,
output_directory=output_directory,
base_year=base_year,
primary_activity='value_added')
def reconcile_physical_units(self, ):
"""Convert physical units to Btu. (Prior to 2005, the data on energy
consumption fuels to produce electricity were supplied in physical
units (e.g. mcf of natural gas, tons of coal, etc))
Data Source: EIA's Annual Energy Review (AER)"""
pass
def manufacturing(self):
"""Gather manufacturing data
Returns:
[type]: [description]
"""
manufacturing_data = \
Manufacturing(self.naics_digits).manufacturing()
return manufacturing_data
def non_manufacturing(self):
"""Gather non-manufacturing data
Primary Data Sources:
Economic Census (previously the Census of
Manufactures, Census of Agriculture, and
Census of Mining) Prior to 1985, primary
data source is the National Energy Accounts (NEA)
http://www.nass.usda.gov/Statistics_by_Subject/index.php
Returns:
[type]: [description]
"""
non_manufacturing_data = \
NonManufacturing(self.naics_digits).nonmanufacturing_data()
return non_manufacturing_data
def collect_data(self):
"""Gather all input data for decomposition
of the energy use in the Industrial sector
Returns:
data_dict (dict): [description]
"""
non_man = self.non_manufacturing()
man = self.manufacturing()
data_dict = {
'Industry': {
'Manufacturing': man,
'Nonmanufacturing': non_man
}
}
return data_dict
def total_industrial_util_adj_lmdi(self):
"""[summary]
Returns:
util_adj_categories (list): [description]
"""
# This case is quite different from the others
util_adj_categories = [
'Fuels', 'Delivered Electricity', 'Source Electricity',
'Total Source'
]
return util_adj_categories
def main(self, breakout, calculate_lmdi):
"""Calculate decomposition for the Industrial sector
Args:
breakout ([type]): [description]
calculate_lmdi ([type]): [description]
Returns:
[type]: [description]
"""
# unit_conversion_factor = 1
data_dict = self.collect_data()
results_dict, formatted_results = \
self.get_nested_lmdi(
level_of_aggregation=self.level_of_aggregation,
breakout=breakout, calculate_lmdi=calculate_lmdi,
raw_data=data_dict, lmdi_type='LMDI-I')
return results_dict
class EmissionsDataExploration:
"""Class to visualize changes over time in Emissions and
Emissions factors using EIA data
"""
def __init__(self):
self.eia = GetEIAData('emissions')
def eia_data(self, id_, label):
"""Make EIA API call
Args:
id_ (str): EIA API endpoint
label (str): Label to use as column name
in resulting df
Returns:
data (DataFrame): data resulting from API call
"""
data = self.eia.eia_api(id_=id_, id_type='series', new_name=label)
return data
def all_fuels_data(self):
"""Collect EIA CO2 Emissions data from
all fuels data for each Sector
Returns:
all_data (dict): All CO2 emissions data
Dictionary with {sector}_co2 as keys and
dataframe as value
"""
commercial_co2 = 'EMISS.CO2-TOTV-CC-TO-US.A'
electric_power_co2 = 'EMISS.CO2-TOTV-EC-TO-US.A'
industrial_co2 = 'EMISS.CO2-TOTV-IC-TO-US.A'
residential_co2 = 'EMISS.CO2-TOTV-RC-TO-US.A'
transportation_co2 = 'EMISS.CO2-TOTV-TC-TO-US.A'
sectors = {
'commercial_co2': commercial_co2,
'electric_power_co2': electric_power_co2,
'industrial_co2': industrial_co2,
'residential_co2': residential_co2,
'transportation_co2': transportation_co2
}
all_data = {
s: self.eia_data(id_=sectors[s], label='CO2 Emissions')
for s in sectors
}
return all_data
def all_fuels_all_sector_data(self):
"""Collect EIA CO2 Emissions data from all fuels and all sectors
Returns:
all_sector (DataFrame): CO2 Emissions data for all
fuels and all sectors in the US
"""
all_fuels_all_sector = 'EMISS.CO2-TOTV-TT-TO-US.A'
all_sector = self.eia_data(id_=all_fuels_all_sector,
label='CO2 Emissions')
return all_sector
@staticmethod
def lineplot(datasets, y_label):
"""Plot 'CO2 Emissions from All Fuels by Sector' data
Args:
datasets (dict): [description]
y_label (str): Label to use for y-axis of resulting
plot
"""
plt.style.use('seaborn-darkgrid')
palette = plt.get_cmap('Set2')
for i, label in enumerate(datasets.keys()):
df = datasets[label]
plt.plot(df.index,
df,
marker='',
color=palette(i),
linewidth=1,
alpha=0.9,
label=label)
title = 'CO2 Emissions from All Fuels by Sector'
plt.title(title, fontsize=12, fontweight=0)
plt.xlabel('Year')
plt.ylabel(y_label)
plt.legend(loc=2, ncol=2)
plt.show()
def get_emissions_plots(self):
"""Collect CO2 Emisisons data and plot it
"""
sectors = self.all_fuels_data()
# total = self.all_fuels_all_sector_data()
# sectors['total'] = total
self.lineplot(sectors, y_label='CO2 Emissions (Million Metric Tons)')
def get_emissions_factors_plots(self):
"""Collect CO2 Emissions and Energy data by sector,
calculate Emissions factors (CO2/Energy) and plot the results
"""
emissions = self.all_fuels_data()
energy = self.economy_wide()
sectors = [
'commercial', 'industrial', 'residential', 'transportation',
'electric_power'
]
emissions_factors = dict()
for s in sectors:
em = emissions[f'{s}_co2']
en = energy[f'{s}_energy']
em, en = df_utils().ensure_same_indices(em, en)
factor = em.divide(en.values, axis=1)
factor = factor.rename(
columns={
'CO2 Emissions': 'Million Metric Tons per Trillion Btu'
})
emissions_factors[s] = factor
self.lineplot(emissions_factors,
y_label='Million Metric Tons CO2 per Trillion Btu')
def economy_wide(self):
"""Collect Energy Consumption data for each sector
from the EIA API
Returns:
all_data (dict): Dictionary with sectors as keys and
df as values
"""
commercial_energy = 'TOTAL.TECCBUS.A'
electric_power_energy = 'TOTAL.TXEIBUS.A'
industrial_energy = 'TOTAL.TEICBUS.A'
residential_energy = 'TOTAL.TERCBUS.A'
transportation_energy = 'TOTAL.TEACBUS.A'
sectors = {
'commercial_energy': commercial_energy,
'electric_power_energy': electric_power_energy,
'industrial_energy': industrial_energy,
'residential_energy': residential_energy,
'transportation_energy': transportation_energy
}
all_data = {
s: self.eia_data(id_=sectors[s], label='Energy')
for s in sectors
}
return all_data
'Miscellaneous': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_BMF_NA_NA_NA_TRLBTU.A',
'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_BMF_ELC_NA_NA_TRLBTU.A'}, # Energy data from "Balance of Manufacturing"
'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_MISC_NA_NA_NA_BLNY09DLR.A'}}},
'Nonmanufacturing': {'Agriculture/Forestry/Fishing/Hunting': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_AGG_NA_NA_NA_TRLBTU.A',
'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_AGG_PRC_NA_NA_TRLBTU.A'},
'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_AFF_NA_NA_NA_BLNY09DLR.A'}}, # here elec is purchased electricity, Note: try to find total elec
'Mining': {{'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_NA_NA_NA_TRLBTU.A',
'purchased_electricity': {'excluding_oil_shale': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_PEO_NA_NA_TRLBTU.A',
'including_oil_shale': 'AEO.2020.REF2020.CNSM_NA_IDAL_MING_PES_NA_NA_TRLBTU.A'}},
'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_MING_NA_NA_NA_BLNY09DLR.A'}}},
'Construction': {'energy': {'total_energy': 'AEO.2020.REF2020.CNSM_NA_IDAL_CNS_NA_NA_NA_TRLBTU.A',
'purchased_electricity': 'AEO.2020.REF2020.CNSM_NA_IDAL_CNS_PRC_NA_NA_TRLBTU.A'},
'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_NMFG_CNS_NA_NA_NA_BLNY09DLR.A'}}}}
industrial_eia = GetEIAData('industrial')
energy_use_industrial_electricity_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_ELC_NA_NA_QBTU.A', id_type='series')
energy_use_industrial_total_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_TOT_NA_NA_QBTU.A', id_type='series')
energy_use_industrial_delivered_energy_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_DELE_NA_NA_QBTU.A', id_type='series')
industrial_value_of_shipments_total =
activity_data =
energy_data =
pass
def transportation_projections(self):
"""activity:
- Passenger-miles [P-M] (Passenger)
- Ton-miles [T-M] (Freight)
class TransportationIndicators(CalculateLMDI):
"""Class to calculate Energy Intensity indicators for the U.S. Transportation Sector
"""
def __init__(self,
directory,
output_directory,
level_of_aggregation,
lmdi_model='multiplicative',
tedb_date='04302020',
base_year=1985,
end_year=2018):
self.transit_eia = GetEIAData('transportation')
self.mer_table25_dec_2019 = self.transit_eia.eia_api(
id_='711272'
) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272'
self.mer_table_43_nov2019 = self.transit_eia.eia_api(
id_='711272'
) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272'
self.aer_2010_table_65 = self.transit_eia.eia_api(
id_='711272'
) # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711272'
self.tedb_date = tedb_date
self.energy_types = ['deliv']
self.sub_categories_list = {
'All_Passenger': {
'Highway': {
'Passenger Cars and Trucks': {
'Passenger Car – SWB Vehicles': {
'Passenger Car': None,
'SWB Vehicles': None
},
'Light Trucks – LWB Vehicles': {
'Light Trucks': None,
'LWB Vehicles': None
},
'Motorcycles': None
},
'Buses': {
'Urban Bus': None,
'Intercity Bus': None,
'School Bus': None
},
'Paratransit': None
},
'Rail': {
'Urban Rail': {
'Commuter Rail': None,
'Heavy Rail': None,
'Light Rail': None
},
'Intercity Rail': None
},
'Air': {
'Commercial Carriers': None,
'General Aviation': None
}
},
'All_Freight': {
'Highway': {
'Single-Unit Truck': None,
'Combination Truck': None
},
'Rail': None,
'Air': None,
# 'Waterborne': None,
'Pipeline': {
'Oil Pipeline': None,
'Natural Gas Pipeline': None
}
}
}
super().__init__(sector='transportation', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \
energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
# self.transportation_data = {'Passenger Car – SWB Vehicles': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': '4_01', 'header_starts': 8, 'column_name': 'Fuel use'}, # Table4_01_{date}
# 'total_energy':
# {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_18'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': 'TEDB', 'table_number': '4_01', 'header_starts': 8, 'column_name': None},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'TEDB', 'table_number': 'A_18'}},
# 'Light Trucks – LWB Vehicles': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': '4_02', 'header_starts': 7},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_05'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': 'TEDB', 'table_number': '4_02'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'TEDB', 'table_number': 'A_19'}},
# 'Motorcycles': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_02'},
# 'total_energy':
# {'unit': 'tbtu', 'source': None, 'table_number': None, 'Assumptions': 'Assume all motorcycle fuel is gasoline'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': 'BTS', 'table_number': '1-35'}, # with some interpolation prior to 1990'
# 'passenger_miles':
# {'unit': 'miles', 'source': None, 'table_number': None, 'Assumptions': vehicle_miles * 1.1}},
# 'Urban Bus': {'total_fuel':
# {'unit': 'gallons', 'source': 'APTA Public Transportation Fact Book'},
# 'total_energy':
# {'unit': '', 'source': None, 'table_number': None, 'Assumption': 'Apply conversion factors (Btu/gallon)'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'APTA'}},
# 'Intercity Bus': {'total_fuel':
# {'unit': 'gallons', 'source': 'Earlier data from TEDB-19 later from ABA'},
# 'total_energy':
# {'unit': 'tbtu', 'source': None, 'Assumptions': 'All fuel assumed to be diesel'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'Multiple'}},
# 'School Bus': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_04'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'TEDB', 'table_number': 'A_04', 'Assumptions': 'Fuel assumed to be 90% diesel, 10% gasoline'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'Multiple'}},
# 'Commuter Rail': {'total_fuel':
# {'unit': 'kwh_gallons', 'source': 'APTA Public Transportation Fact Book'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'TEDB', 'Assumptions': 'Conversion to TBtu with electricity and diesel energy conversion factors (electricity conversion at 10,339 Btu/kWh from TEDB'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'APTA', 'table_number': ' Table 3 of Appendix A'}},
# 'Transit Rail (Heavy and Light Rail)': {'total_fuel':
# {'unit': 'kwh_gallons', 'source': '(APTA, Fact Book)' , 'table_number': 'Tables 38 and 39 in Appendix A'},
# 'total_energy':
# {'unit': 'tbtu', 'source': None, 'table_number': None, 'Assumptions': 'Conversion to TBtu as for commuter rail'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'APTA', 'table_number': 'source as avober for fuel use. Passenger-miles in Table 3 of Appendix'}},
# 'Intercity Rail': {'total_fuel':
# {'unit': 'kwh_gallons', 'source': '1994-2016" TEDB-37 Table A.16; prior data extrapolated from TEDB-30, Table 9.10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to TBtu as for commuter rail'},
# 'passenger_miles':
# {'unit': 'miles', 'source': 'APTA source as avober for fuel use. Passenger-miles in Table 3 of Appendix'}},
# 'Commercial Carriers': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_09'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'passenger_miles':
# {'unit': 'miles', 'source': '1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use'}},
# 'General Aviation': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'passenger_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},
# 'Single-Unit Truck': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_09'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'},
# 'ton_miles': {'unit': None, 'source': None}},
# 'Combination Truck': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},
# 'Rail': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},
# 'Air Carriers': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},
# 'Waterborne': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},
# 'Natural Gas Pipelines': {'total_fuel':
# {'unit': 'gallons', 'source': 'TEDB', 'table_number': 'A_10'},
# 'total_energy':
# {'unit': 'tbtu', 'source': 'Conversion to Btu with factors for aviation fuel and jet fuel (120,2 and 135.0 kBtu/gallon, respectively.)'},
# 'vehicle_miles':
# {'unit': 'miles', 'source': ' 1970-2001: Eno Transportation Foundation, Transportation in America 2001, 19th Edition, p.45 Passenger-miles after 2001 extrapolated by total energy use.'}},}
# def import_tedb_data(self, table_number):
# file_url = f'https://tedb.ornl.gov/wp-content/uploads/2020/04/Table{table_number}_{self.tedb_date}.xlsx'
# xls = pd.read_excel(file_url) # , sheetname=None, header=11
# return xls
# def get_data_from_nested_dict(self, nested_dictionary, subcategory_data_sources):
# for key, value in nested_dictionary.items():
# if type(value) is dict:
# get_data_from_nested_dict(value)
# elif value is None:
# data_sources = self.transportation_data[key]
# subcategory_data_sources[key] = data_sources
# return key, data_sources
# def load_data(self):
# energy_mode_data = dict()
# fuel_mode_data = dict()
# vehcile_miles_mode_data = dict()
# for mode_name, mode_dict in self.transportation_data.items(): # could be more efficient with pandas?
# for variable_name, variable_dict in mode_dict.items():
# if variable_name == 'total_fuel':
# pass
# elif variable_name == 'total_energy':
# pass
# elif variable_name == 'vehicle_miles':
# pass
# if variable_dict['source'] == 'TEDB':
# mode_source_df = import_tedb_data(variable_dict['table_number'])
# get_data_from_nested_dict(self.categories_dict, subcategory_data_sources)
# pass
# def fuel_heat_content(self, parameter_list):
# """Assumed Btu content of the various types of petroleum products. This dataframe is not time variant (no need to update it with subsquent indicator updates)
# Parameters
# ----------
# Returns
# -------
# """
# pass
# def fuel_consump(self, parameter_list):
# """Time series of fuel consumption for the major transportation subsectors. Data are generally in
# millions gallons or barrels of petroleum.
# Parameters
# ----------
# Returns
# -------
# """
# # swb_vehciles_all_fuel = [0] # 2007-2017 FHA Highway Statistics Table VM-1
# # motorcycles_all_fuel_1949_1969 = [0] # Based upon data published in Table 4_11 of Bureau of Transportation Statistics (BTS), fuel consumption is based assumption of 50 miles per gallon
# # motorcycles_all_fuel_1970_2017 = import_tedb_data(table_number='A_02') # Alternative: 2017 data from Highway Statistics, Table VM-1.
# # light_trucks_all_fuel =
# # lwb_vehicles_all_fuel =
# # bus_urban_diesel =
# # bus_urban_cng =
# # bus_urban_gasoline =
# # bus_urban_lng =
# # bus_urban_bio_diesel =
# # bus_urban_other =
# # bus_urban_lpg =
# # paratransit =
# # bus_school =
# # bus_school_million_bbl =
# # bus_intercity = [0] # Not used
# # waterborne =
# # air =
# # rail_intercity_diesel_million_gallons =
# # rail_intercity_diesel_electricity_gwhrs =
# # rail_intercity_total_energy_tbtu =
# # rail_intercity_total_energy_tbtu_old =
# # rail_intercity_total_energy_tbtu_adjusted =
# # rail_commuter_diesel =
# # rail_commuter_electricity_gwhrs =
# # rail_heavy_electricity_gwhrs =
# # rail_light_electricity_gwhrs =
# # class_I_freight_distillate_fuel_oil =
# # pipeline_natural_gas_million_cu_ft =
# # pipeline_natutral_gas_electricity_million_kwh =
# pass
# def adjust_truck_freight(self, parameter_list):
# """purpose
# Parameters
# ----------
# Returns
# -------
# DOES THIS WORK ? dataframes
# """
# gross_output = bls_data[] # Note: Gross output in million 2005 dollars from BLS database for their employment projections input-output model,
# # PNNL spreadsheet: BLS_output_data.xlsx in folder BLS_Industry_Data)
# vehicle_miles_fhwa_tvm1 = [0]
# old_methodology_2007_extrapolated = gross_output.iloc[2007] / gross_output.iloc[2006] * vehicle_miles_fhwa_tvm1.iloc[2006, :]
# old_series_scaled_to_new = vehicle_miles_fhwa_tvm1 * vehicle_miles_fhwa_tvm1.iloc[2007, :] / old_methodology_2007_extrapolated
# # **come back**
# pass
# def passenger_based_energy_use(self):
# """Calculate the fuel consumption in Btu for passenger transportation
# """
# all_passenger_categories = self.categories_dict['All_Passenger']
# for passenger_category in all_passenger_categories.keys():
# for transportation_mode in passenger_category.keys():
# if transportation_mode == 'Passenger Car – SWB Vehicles':
# # Passenger Car and SWB Vehicles have separate data sources, later aggregated?
# elif transportation_mode == 'Light Trucks – LWB Vehicles':
# # Light Trucks and LWB Vehicles have separate data sources, later aggregated?
# else:
# urban_rail_categories = list(all_passenger_categories['Rail']['Urban Rail'].values()])
# passenger_based_energy_use_df['Urban Rail (Hvy, Lt, Commuter)'] = passenger_based_energy_use_df[urban_rail_categories].sum(1)
# pass
# def passenger_based_activity_data(self):
# # Passenger cars and light trucks
# fha_table_vm201a = pd.read_excel('https://www.fhwa.dot.gov/ohim/summary95/vm201a.xlw', sheetname='UnknownSheet4', header=11) # Passenger car and light truck VMT 1981-95
# fha_table_vm1 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', header=4)
# fha_table_vm1_1996 = pd.read_excel('https://www.fhwa.dot.gov/ohim/1996/vm1.xlw')
# fha_table_vm1_1997 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs97/xls/vm1.xls', sheetname='1997 VM1', header=6)
# fha_table_vm1_1998 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/1998/xls/vm1.xls', sheetname='Pub 98 VM1', header=6)
# fha_table_vm1_1999 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs99/excel/vm1.xls', sheetname='1999 VM1', header=3)
# fha_table_vm1_2000 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs00/xls/vm1.xls', sheetname='Sheet1', header=4)
# fha_table_vm1_2001 = pd.read_excel('https://www.fhwa.dot.gov/ohim/hs01/xls/vm1.xls', sheetname='Sheet1', header=4)
# vmt2002_2006_passenger_car_light_truck = pd.read_excel('https://www.bts.gov/sites/bts.dot.gov/files/table_01_35_013020.xlsx', header=1) # Note, this link has changed since usage in the spreadsheets
# vmt2007_2008_passenger_car = import_tedb_data(table_number='4_01')
# vmt2007_2008_light_truck = import_tedb_data(table_number='4_02')
# load_factors_1966_2011 =
# # Short Wheelbase and Long Wheelbase Vehicles
# 2007-2017 FHA VM1
# fha_table_vm1_2014 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2014/xls/vm1.xlsx', sheetname='final VM-1' , header=4)
# # Motorcycles
# 1970-2010 = vmt2002_2006_passenger_car_light_truck
# fha_tables
# # Bus / Transit
# # 1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46.
# apta_table3 = pd.read_excel('https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx', sheetname='3', header=2) # 1997-2017
# # # Bus / Intercity
# # see revised_intercity_bus_estimates
# # # Bus / School
# # see revised_intercity_bus_estimates
# # Paratransit
# paratransit_activity_1984_2017 = apta_table3
# # Commercial Air Carrier
# commercial_air_carrier_activity_1970_1974 = [0] # TEDB 13 Table 6.2 page 6-7
# commercial_air_carrier_activity_1975_1984 = [0] # TEDB 19 Table 12.1 page 12-2 (is the 2 a typo?)
# commercial_air_carrier_activity_1985_1999 = [0] # TEDB 21 Table 12.1 page 12-2 (is the 2 a typo?)
# commercial_air_carrier_activity_2000_2018 = import_tedb_data(table_number='10_02')
# domestic_passenger_miles_2011 = [0] # from Table 2.12 in 2013 TEDB
# # Urban Rail (Commuter)
# urban_rail_commuter_activity_1970_1976 = [0] # Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 6.13, p. 6-31.
# urban_rail_commuter_activity_1977_2017 = apta_table3
# # Urban Rail (Light, Heavy)
# urban_rail_light_heavy_1970_1977 = import_tedb_data(table_number='10_10')
# urban_rail_light_heavy_1977_2017 = apta_table3
# # Intercity Rail (Amtrak) Note: Amtrak data is compiled by fiscal year rather than calendar year
# intercity_rail_1970 = None # Amtrak not established until May 1971. Data for 1970 assumes same passenger activity as 1971, primarily included
# # to fill out all transportation-related time series back to 1970
# intercity_rail_1971_2016 = [0]
# intercity_rail_2017 = [0] # 'Bureau of Transportation Statistics, National Transportation Statistics, Table 1-40: U.S. Passenger-Miles
# # General Aviation
# # !!!!
# return passenger_based_activity_input_data
# def passenger_based_activity(self):
# """ Time series for the activity measures for passenger transportation
# # Highway
# self.intercity_buses = 'Detailed data_Intercity buses' # Column E
# 1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46.
# 1977-2017 American Public Transportation Association. 2019 Public Transportation Fact Book. Appendix A, Table 3, Pt. A
# Note: Transit bus does not include trolley bus because APTA did not not report electricity consumption for trolley buses separately for all years.
# https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/
# Note: Series not continuous for transit bus between 2006 and 2007; estimation procedures changed for bus systems outside urban areas
# Note: Series not continuous between 1983 and 1984.
# Note: Transit Bus == Urban Bus
# This method is horribly structured.
# """
# all_passenger_categories = self.categories_dict['All_Passenger']
# for passenger_category in all_passenger_categories.keys():
# for transportation_mode in all_passenger_categories[passenger_category].keys():
# if transportation_mode == 'Passenger Car – SWB Vehicles':
# # Passenger Car and SWB Vehicles have separate data sources, later aggregated?
# elif transportation_mode == 'Light Trucks – LWB Vehicles':
# # Light Trucks and LWB Vehicles have separate data sources, later aggregated?
# else:
# urban_rail_categories = list(all_passenger_categories['Rail']['Urban Rail'].values()])
# passenger_based_energy_use_df['Urban Rail (Hvy, Lt, Commuter)'] = passenger_based_energy_use_df[urban_rail_categories].sum(1)
# pass
# def freight_based_energy_use(self):
# """Calculate the fuel consumption in Btu for freight transportation
# Need FuelConsump, Fuel Heat Content
# """
# all_freight_categories = self.categories_dict['All_Freight']
# for freight_category in all_freight_categories.keys():
# for freight_mode in freight_category.keys():
# pass
# def freight_based_activity(self):
# """Time series for the activity measures for passenger transportation
# """
# all_freight_categories = self.categories_dict['All_Freight']
# for freight_category in all_freight_categories.keys():
# for freight_mode in freight_category.keys():
# if freight_category == 'Waterborne':
# domestic_vessel = [0]
# international_vessel_in_us_waters = [0]
# total_commerce_us_waters = domestic_vessel + international_vessel_in_us_waters
# elif freight_category == 'Highway':
# highway_published = [0]
# if freight_mode == 'Combination Truck':
# freight_based_energy_use_df['Old Series from 2001 Eno, Trans. In America'] = [0]
# # 1950-1989:
# freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0] # 1990 value for this column divided by 1990 value for 'Old Series from 2001 Eno, Trans. In America' * contemporary year from old series
# # 1990-2003
# freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0]
# # 2004-2017
# vehicle_miles_combination_trucks_adjusted = [0] # from adjust truck freight column K
# freight_based_energy_use_df['Combination Truck, adjusted extrapolated'] = [0] # contemporary value of vehicle_miles_combination_trucks_adjusted divided by 2003 value of vehicle_miles_combination_trucks_adjusted times 2003 value of this
# elif freight_mode == 'Single-Unit Truck':
# # 1970-2006
# freight_based_energy_use_df['Single-Unit Truck (million vehicle-miles), adjusted'] = [0] # Adjust_truck_Freight Column J
# # 2007-2017
# freight_based_energy_use_df['Single-Unit Truck (million vehicle-miles), adjusted'] = highway_published['Single-Unit Truck (million vehicle-miles)']
# if freight_mode == 'Natural Gas Pipeline':
# natrual_gas_delivered_to_end_users = self.table65_AER2010 # Column AH, million cu. ft.
# natural_gas_delivered_lease_plant_pipeline_fuel = self.MER_Table43_Nov2019 # Column M - column D - column I
# natural_gas_delivered_lease_plant_pipeline_fuel.at[0] = 0.000022395
# natural_gas_consumption_million_tons = natrual_gas_delivered_to_end_users * natural_gas_delivered_lease_plant_pipeline_fuel[0]
# avg_length_natural_gas_shipment_miles = 620
# freight_based_energy_use_df[freight_mode] = natural_gas_consumption_million_tons * 620
# pass
# def water_freight_regression(self, ):
# """purpose
# Parameters
# ----------
# Returns
# -------
# """
# intensity =
# X = math.log(intensity)
# Y = years
# reg = linear_model.LinearRegression()
# reg.fit(X, Y)
# coefficients = reg.coef_
# predicted_value = reg.predict(X_test) # Predicted value of the intensity based on actual degree days
# def detailed_data_school_buses(self, parameter_list):
# """purpose
# Parameters
# ----------
# Returns
# -------
# """
# pass
# def detailed_data_intercity_buses(self, parameter_list):
# """purpose
# Parameters
# ----------
# Returns
# -------
# """
# passenger_miles = [0]
# revised_passenger_miles = [0]
# number_of_buses = [0]
# number_of_buses_old = [0] # Not used
# vehicle_miles_commercial = [0] # Not used
# vehicle_miles_intercity = [0]
# implied_load_factor = [0]
# energy_use_tedb_19_32 = [0] # Not used
# implied_mpg = [0] # Not used
# blended_mpg_miles_per_gallon = [0] #
# revised_energy_use = [0]
# number_of_motorcoaches = [0]
# number_of_motorcoaches['Ratio'] = number_of_motorcoaches['US'] / number_of_motorcoaches['Total N.A.']
# pass
# def mpg_check(self, parameter_list):
# """purpose
# Parameters
# ----------
# Returns
# -------
# """
# pass
def energy_consumption(self):
"""Gather Energy Data to use in LMDI Calculation TBtu
"""
pass
def activity(self):
"""Gather
"""
pass
def collect_data(self):
"""Method to collect freight and passenger energy and activity data.
This method should be adjusted to call dataframe building methods rather than reading csvs, when those are ready
Gather Energy Data to use in LMDI Calculation TBtu and Activity Data to use in LMDI Calculation passenger-miles [P-M], ton-miles [T-M]
Returns:
[type]: [description]
"""
# passenger_based_energy_use = self.energy_consumption('All_Passenger')
# passenger_based_activity = self.activity('All_Passenger')
# freight_based_energy_use = self.energy_consumption('All_Freight')
# freight_based_activity = self.activity('All_Freight')
passenger_based_energy_use = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/passenger_based_energy_use.csv'
).set_index('Year')
passenger_based_activity = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/passenger_based_activity.csv'
).set_index('Year')
freight_based_energy_use = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/freight_based_energy_use.csv'
).set_index('Year')
freight_based_activity = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/freight_based_activity.csv'
).set_index('Year')
data_dict = {
'All_Passenger': {
'energy': {
'deliv': passenger_based_energy_use
},
'activity': passenger_based_activity
},
'All_Freight': {
'energy': {
'deliv': freight_based_energy_use
},
'activity': freight_based_activity
}
}
print(freight_based_energy_use)
return data_dict
def main(self, breakout, save_breakout, calculate_lmdi): # base_year=None,
"""potentially refactor later
Args:
_base_year ([type], optional): [description]. Defaults to None.
"""
# if not base_year:
# _base_year = self.base_year
# else:
# _base_year = _base_year
data_dict = self.collect_data()
if self.level_of_aggregation == 'all_transportation':
freight_activity, freight_energy = self.get_nested_lmdi(
level_of_aggregation='All_Freight',
breakout=breakout,
save_breakout=save_breakout,
calculate_lmdi=False,
raw_data=data_dict)
passenger_activity, passenger_energy = self.get_nested_lmdi(
level_of_aggregation='All_Passenger',
breakout=breakout,
save_breakout=save_breakout,
calculate_lmdi=False,
raw_data=data_dict)
all_transportation_activity = freight_activity[['All_Freight']].merge(passenger_activity[['All_Passenger']], \
left_index=True, right_index=True, how='outer')
all_transportation_energy = freight_energy[['All_Freight']].merge(passenger_energy[['All_Passenger']], \
left_index=True, right_index=True, how='outer')
results = self.call_lmdi(unit_conversion_factor=1,
lmdi_models=self.lmdi_models)
elif self.level_of_aggregation == 'personal_vehicles_aggregate':
# THIS CASE NEEDS A DIFFERENT FORMAT (NOT SAME NESTED DICTIONARY STRUCTURE), need to come back
categories = ['Passenger Car', 'Light Truck', 'Motorcycles']
results = None
else:
results_dict, results = self.get_nested_lmdi(
level_of_aggregation=self.level_of_aggregation,
breakout=breakout,
save_breakout=save_breakout,
calculate_lmdi=calculate_lmdi,
raw_data=data_dict)
results.to_csv(f"{self.output_directory}/transportation_results2.csv")
print('RESULTS:\n', results)
return results
def compare_aggregates(self, parameter_list):
"""purpose
Parameters
----------
Returns
-------
"""
total_fuel_tbtu_published_mer = self.mer_table_25_dec_2019[
'Total Energy Consumed by the Transportation Sector'] # j
sum_of_modes = self.total_transportation[
'Energy_Consumption_Total'] # F
difference = sum_of_modes - total_fuel_tbtu_published_mer
pct_difference = difference / total_fuel_tbtu_published_mer
return pct_difference
def industrial_projections(self):
"""Gather Industrial projections data
activity:
- Value added --> Gross Domestic Product (Total Industrial only)
- Gross Output
- Value Added
energy: Energy Consumption Trillion Btu
"""
# Industrial : Value of Shipments : Agriculture, Mining, and Construction, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_NMFG_VOS_NA_USA_BLNY09DLR.A'
# Industrial : Value of Shipments : Manufacturing, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_MANF_VOS_NA_USA_BLNY09DLR.A'
# Industrial : Value of Shipments : Total, AEO2019, AEO2020 --> 'AEO.2020.AEO2019REF.ECI_NA_IDAL_NA_VOS_NA_USA_BLNY09DLR.A'
industrial_eia = GetEIAData('industrial')
data_dict = {'Manufacturing': {'Food, Beverages, & Tobacco': {'Food Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_FDP_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Food Products'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_FDP_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Food Products')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_FDP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Food Products')}},
# 'Beverages and Tobacco Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_BVTP_NA_NA_NA_BLNY09DLR.A'}}
},
# 'Textile Mills and Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_TMP_NA_NA_NA_BLNY09DLR.A'}},
'Wood Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_WDP_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Wood Products'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_WDP_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Wood Products')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_SHP_IDAL_WDP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Wood Products')}},
'Paper': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PPM_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Paper'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PPM_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Paper')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_PPM_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Paper')}} ,
# 'Printing': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': 'AEO.2020.REF2020.ECI_VOS_MANF_PRN_NA_NA_NA_BLNY09DLR.A'}},
# 'Petroleum & Coal Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='', id_type='series'),
# 'purchased_electricity': industrial_eia.eia_api(id_='', id_type='series')},
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PCL_NA_NA_NA_BLNY09DLR.A', id_type='series')}},
'Chemicals': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BCH_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Chemicals'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BCH_HAP_PRC_NA_TRLBTU.A', id_type='series', new_name='Chemicals')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_CHE_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Chemicals')}},
'Plastics & Rubber Products': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PLI_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Plastics & Rubber Products'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_PLI_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Plastics & Rubber Products')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PRP_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Plastics & Rubber Products')}},
# 'Nonmetallic Mineral Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_SCG_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, ,
# 'Primary Metals': {'energy': {'total_energy': {'iron_and_steel': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_ISM_TOT_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals'),
# 'aluminum': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AAP_TOT_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals')},
# 'purchased_electricity': {'iron_and_steel': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_ISM_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals'),
# 'aluminum': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AAP_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Primary Metals')}},
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_PMM_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Primary Metals')}},
# 'Fabricated Metal Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_FABM_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, ,
# 'Machinery': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_MCHI_NA_NA_NA_BLNY09DLR.A', id_type='series')}}, ,
# 'Computer & Electronic Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_CMPEL_NA_NA_NA_BLNY09DLR.A', id_type='series')}},
# 'Electrical Equipment': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_EEI_NA_NA_NA_BLNY09DLR.A', id_type='series')}}},
# 'Transportation Equipment': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_IDAL_TEQ_NA_NA_NA_BLNY09DLR.A', id_type='series')}},
# 'Furniture & Related Products': {'energy': {'total_energy': , 'purchased_electricity': },
# 'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_FRP_NA_NA_NA_BLNY09DLR.A', id_type='series')}},
'Miscellaneous': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BMF_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Miscellaneous'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_BMF_ELC_NA_NA_TRLBTU.A', id_type='series', new_name='Miscellaneous')}, # Energy data from "Balance of Manufacturing"
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_MANF_MISC_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Miscellaneous')}}},
'Nonmanufacturing': {'Agriculture/Forestry/Fishing/Hunting': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AGG_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_AGG_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_AFF_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Agriculture/Forestry/Fishing/Hunting')}}, # here elec is purchased electricity, Note: try to find total elec
'Mining': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Mining'),
'purchased_electricity': {'excluding_oil_shale': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_PEO_NA_NA_TRLBTU.A', id_type='series', new_name='Mining'),
'including_oil_shale': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_MING_PES_NA_NA_TRLBTU.A', id_type='series', new_name='Mining')}},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_MING_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Mining')}},
'Construction': {'energy': {'total_energy': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_CNS_NA_NA_NA_TRLBTU.A', id_type='series', new_name='Construction'),
'purchased_electricity': industrial_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_IDAL_CNS_PRC_NA_NA_TRLBTU.A', id_type='series', new_name='Construction')},
'activity': {'value_of_shipments': industrial_eia.eia_api(id_='AEO.2020.REF2020.ECI_VOS_NMFG_CNS_NA_NA_NA_BLNY09DLR.A', id_type='series', new_name='Construction')}}}}
energy_use_industrial_electricity_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Industry')
energy_use_industrial_total_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_TOT_NA_NA_QBTU.A', id_type='series', new_name='Industry')
energy_use_industrial_delivered_energy_us = industrial_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_IDAL_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Industry')
return data_dict
def electricity_projections():
"""Gather Commercial projections data
activity: Million kWh
energy: Energy Consumption Trillion Btu
"""
electricity_eia = GetEIAData('electricity')
data_dict = {'Elec Power Sector':
{'Electricity Only':
{'Fossil Fuels':
{'Coal': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_STC_NA_NA_QBTU.A', id_type='series', new_name='Coal')},
'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_CL_NA_USA_BLNKWH.A', id_type='series', new_name='Coal')},
'Petroleum': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_LFL_NA_NA_QBTU.A', id_type='series', new_name='Petroleum')},
'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_PET_NA_USA_BLNKWH.A', id_type='series', new_name='Petroleum')},
'Natural Gas': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NG_NA_NA_QBTU.A', id_type='series', new_name='Natural Gas')},
'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_NG_NA_USA_BLNKWH.A', id_type='series', new_name='Natural Gas')}},
'Nuclear': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NUC_NA_NA_QBTU.A', id_type='series', new_name='Nuclear')},
'activity': electricity_eia.eia_api(id_='AEO.2020.AEO2019REF.GEN_NA_ELEP_POW_NUP_NA_USA_BLNKWH.A', id_type='series', new_name='Nuclear')},
# 'Hydro Electric': {'energy': '',
# 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_HYD_CNV_NA_BLNKWH.A', id_type='series')},
'Renewable':
{'Wood': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_ELEP_NA_WBM_NA_NA_QBTU.A', id_type='series', new_name='Wood')},
'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_WBM_NA_NA_BLNKWH.A', id_type='series', new_name='Wood')}, # This is wood and other biomass
'Waste': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_ENU_ELEP_NA_NBMSW_NA_NA_QBTU.A', id_type='series', new_name='Waste'),
'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_BGM_NA_NA_BLNKWH.A', id_type='series', new_name='Waste')}}, # This is Biogenic Municipal Waste
# 'Solar': {'Solar Photovoltaic': {'energy': electricity_eia.eia_api(id_='', id_type='series'),
# 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_SLR_PHTVL_NA_BLNKWH.A', id_type='series')},
# 'Solar Thermal': {'energy': electricity_eia.eia_api(id_='', id_type='series'),
# 'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_SLR_THERM_NA_BLNKWH.A', id_type='series')}},
'Wind': {'energy': {'primary': electricity_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_ELEP_NA_OFW_NA_NA_QBTU.A', id_type='series', new_name='Wind')},
'activity': electricity_eia.eia_api(id_='AEO.2020.REF2020.GEN_NA_ELEP_NA_WND_NA_NA_BLNKWH.A', id_type='series', new_name='Wind')}}}}}
return data_dict
def transportation_projections(self):
"""Gather Transportation projections data
activity:
- Passenger-miles [P-M] (Passenger)
- Ton-miles [T-M] (Freight)
energy: Energy Consumption Trillion Btu
"""
# Commercial Carriers? --> domestic air carriers: AEO.2020.AEO2019REF.CNSM_NA_TRN_AIR_DAC_NA_NA_TRLBTU.A
# international air carriers: AEO.2020.AEO2019REF.CNSM_NA_TRN_AIR_IAC_NA_NA_TRLBTU.A
# Transportation Energy Use : Highway : Commercial Light Trucks, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_CML_NA_NA_TRLBTU.A
# Transportation Energy Use : Highway : Freight Trucks : Large, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_LGT26KLBS_NA_TRLBTU.A
# Transportation Energy Use : Highway : Freight Trucks : Light Medium, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_LITEMED_NA_TRLBTU.A
# Transportation Energy Use : Highway : Freight Trucks : Medium, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_MD10T26KLB_NA_TRLBTU.A
# Transportation Energy Use : Highway : Freight Trucks, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_HWY_FGHT_NA_NA_TRLBTU.A
# Transportation Energy Use : Non-Highway : Rail : Freight, Reference, AEO2020: AEO.2020.REF2020.CNSM_NA_TRN_RAIL_FGT_NA_NA_TRLBTU.A
transportation_eia = GetEIAData('transportation')
transportation_categories = {'All_Passenger':
{'Highway':
{'Passenger Cars and Trucks':
{'Light Trucks – LWB Vehicles':
{'Light Trucks': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_LDV_LTRT_NA_TRLBTU.A', id_type='series', new_name='Light Trucks')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_CML_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Light Trucks')},
'Light-Duty Vehicles': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_LDV_LTRT_NA_TRLBTU.A', id_type='series', new_name='Light-Duty Vehicles')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_LDV_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Light-Duty Vehicles')}}},
'Buses': {'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020._TRV_TRN_NA_BST_NA_NA_BPM.A', id_type='series', new_name='Buses'),
'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_HWY_BUS_NA_NA_TRLBTU.A', id_type='series', new_name='Buses')}}},
'Rail': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_RAIL_PSG_PSG_NA_TRLBTU.A', id_type='series', new_name='Rail')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020._TRV_TRN_NA_RLP_NA_NA_BPM.A', id_type='series', new_name='Rail')},
'Air': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_IAC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_AIR_NA_NA_BLNSEATMLS.A', id_type='series', new_name='Air')}},
'All_Freight':
{'Highway':
{'Freight-Trucks': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_FGHT_USE_NA_NA_QBTU.A', id_type='series', new_name='Freight-Trucks')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_FGHT_NA_NA_BLNVEHMLS.A', id_type='series', new_name='Freight-Trucks')}},
'Rail': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_NA_TRN_RAIL_FGT_NA_NA_TRLBTU.A', id_type='series', new_name='Rail')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.KEI_TRV_TRN_NA_RAIL_NA_NA_BLNTNMLS.A', id_type='series', new_name='Rail')},
'Air': {'energy': {'primary': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_FTC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')},
'activity': transportation_eia.eia_api(id_='AEO.2020.REF2020.CNSM_NA_TRN_AIR_FTC_NA_NA_TRLBTU.A', id_type='series', new_name='Air')}}}
energy_use_transportation_electricity_us = transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_ELC_NA_NA_QBTU.A', id_type='series', new_name='Transportation')
energy_use_transportation_total_us = transportation_eia.eia_api(id_='YOUR_API_KEY_HERE&series_id=AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_TOT_NA_NA_QBTU.A', id_type='series', new_name='Transportation')
energy_use_transportation_delivered_energy_us = transportation_eia.eia_api(id_='AEO.2020.AEO2019REF.CNSM_ENU_TRN_NA_DELE_NA_NA_QBTU.A', id_type='series', new_name='Transportation')
data_dict = transportation_categories
return data_dict
class IndustrialIndicators(CalculateLMDI):
"""Some of the specific steps to download and process the census data on construction energy costs are
explained in the following paragraphs. The top-level census bureau website for the Economic Census is:
https://www.census.gov/programs-surveys/economic-census.html. Scroll down the page until the
words “2017 Data Tables” are found. After clicking on that link, the user will end up at
https://www.census.gov/programs-surveys/economic-census/news-updates/updates/2017-datatables.html. The “2017 Data Table pages” now include direct links into data.census.gov and large ftp
downloads. After clicking on pages, the webpage https://www.census.gov/programssurveys/economic-census/data/tables.html comes up. Scroll down this page until the entry
“Construction (NAICS Sector 23)” is found. After selecting this entry, the user is then automatically
transferred to: https://www.census.gov/data/tables/2017/econ/economic-census/naics-sector23.html.
"""
def __init__(self, directory, output_directory, level_of_aggregation=None, lmdi_model='multiplicative', base_year=1985, end_year=2018):
self.sub_categories_list = {'Manufacturing': {'Food, Beverages, & Tobacco': None, 'Textile Mills and Products': None,
'Apparel & Leather': None, 'Wood Products': None, 'Paper': None,
'Printing & Allied Support': None, 'Petroleum & Coal Products': None, 'Chemicals': None,
'Plastics & Rubber Products': None, 'Nonmetallic Mineral Products': None, 'Primary Metals': None,
'Fabricated Metal Products': None, 'Machinery': None, 'Computer & Electronic Products': None,
'Electical Equip. & Appliances': None, 'Transportation Equipment': None,
'Furniture & Related Products': None, 'Miscellaneous': None},
'Nonmanufacturing': {'Agriculture, Forestry & Fishing': None,
'Mining': {'Petroleum and Natural Gas': None,
'Other Mining': None,
'Petroleum drilling and Mining Services': None},
'Construction': None}}
self.ind_eia = GetEIAData('industry')
self.conversion_factors = self.ind_eia.conversion_factors()
self.MER_Nov19_Table24 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.AER10_Table21d = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.AER11_Table21d_MER0816 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.mer_dataT0204 = self.ind_eia.eia_api(id_='711252') # 'http://api.eia.gov/category/?api_key=YOUR_API_KEY_HERE&category_id=711252'
self.BEA_Output_data = [0] # Chain-type Quantity Indexes for Value Added by Industry from Bureau of Economic Analysis
self.energy_types = ['elec', 'fuels', 'deliv', 'source', 'source_adj']
super().__init__(sector='industry', level_of_aggregation=level_of_aggregation, lmdi_models=lmdi_model, categories_dict=self.sub_categories_list, \
energy_types=self.energy_types, directory=directory, output_directory=output_directory, base_year=base_year)
def reconcile_physical_units(self, ):
"""Convert physical units to Btu. (Prior to 2005, the data on energy consumption fuels to produce electricity were supplied in physical units (e.g. mcf of natural gas, tons of coal, etc))
Data Source: EIA's Annual Energy Review (AER)"""
pass
def manufacturing(self):
# categories = self.sub_categories_list['Manufacturing']
# manufacturing_data = Manufacturing().manufacturing()
# print('manufacturing_data: \n', manufacturing_data)
manufacturing_data = None
return manufacturing_data
def non_manufacturing(self):
"""Primary Data Sources: Economic Census (previously the Census of Manufactures, Census of Agriculture, and Census of Mining)
Prior to 1985, primary data source is the National Energy Accounts (NEA)
http://www.nass.usda.gov/Statistics_by_Subject/index.php
"""
# categories = self.sub_categories_list['Nonmanufacturing']
non_manufacturing_data = NonManufacturing().nonmanufacturing_data()
print('non_manufacturing_data: \n', non_manufacturing_data)
return non_manufacturing_data
def collect_data(self):
man = self.manufacturing()
non_man = self.non_manufacturing()
data_dict = {'Manufacturing': man, 'Nonmanufacturing': non_man}
return data_dict
def total_industrial_util_adj_lmdi(self):
util_adj_categories = ['Fuels', 'Delivered Electricity', 'Source Electricity', 'Total Source'] # This case is quite different from the others
return util_adj_categories
def main(self, breakout, save_breakout, calculate_lmdi):
unit_conversion_factor = 1
data_dict = self.collect_data()
results_dict, formatted_results = self.get_nested_lmdi(level_of_aggregation=self.level_of_aggregation, breakout=breakout, save_breakout=save_breakout, calculate_lmdi=calculate_lmdi, raw_data=data_dict, account_for_weather=False)
return formatted_results
class TransportationIndicators(CalculateLMDI):
"""
Class to calculate Energy Intensity indicators
for the U.S. Transportation Sector
"""
def __init__(self,
directory,
output_directory,
level_of_aggregation,
lmdi_model='multiplicative',
tedb_date='04302020',
base_year=1985,
end_year=2018):
self.transit_eia = GetEIAData('transportation')
self.mer_table25_dec_2019 = \
self.transit_eia.eia_api(id_='711272', id_type='category')
self.mer_table_43_nov2019 = \
self.transit_eia.eia_api(id_='711272', id_type='category')
self.aer_2010_table_65 = \
self.transit_eia.eia_api(id_='711272', id_type='category')
self.tedb_date = tedb_date
self.energy_types = ['deliv']
self.sub_categories_list = \
{'All_Transportation':
{'All_Passenger':
{'Highway':
{'Passenger Cars and Trucks':
{'Passenger Car – SWB Vehicles':
{'Passenger Car': None,
'SWB Vehicles': None},
'Light Trucks – LWB Vehicles':
{'Light Trucks': None,
'LWB Vehicles': None},
'Motorcycles': None},
'Buses':
{'Urban Bus': None,
'Intercity Bus': None,
'School Bus': None},
'Paratransit':
None},
'Rail':
{'Urban Rail':
{'Commuter Rail': None,
'Heavy Rail': None,
'Light Rail': None},
'Intercity Rail': None},
'Air':
{'Commercial Carriers': None,
'General Aviation': None}},
'All_Freight':
{'Highway':
{'Single-Unit Truck': None,
'Combination Truck': None},
'Rail': None,
'Air': None,
'Waterborne': None,
'Pipeline':
{'Oil Pipeline': None,
'Natural Gas Pipeline': None}}}}
super().__init__(sector='transportation',
level_of_aggregation=level_of_aggregation,
lmdi_models=lmdi_model,
categories_dict=self.sub_categories_list,
energy_types=self.energy_types,
directory=directory,
output_directory=output_directory,
base_year=base_year,
end_year=end_year,
unit_conversion_factor=1000000)
def import_tedb_data(self,
table_number,
skip_footer=None,
skiprows=None,
sheet_name=None,
usecols=None,
index_col=None):
try:
file_url = f'https://tedb.ornl.gov/wp-content/uploads/2020/04/Table{table_number}_{self.tedb_date}.xlsx'
xls = pd.read_excel(file_url,
skipfooter=skip_footer,
skiprows=skiprows,
sheet_name=sheet_name,
usecols=usecols,
index_col=index_col)
return xls
except urllib.error.HTTPError as e:
print('error with table', table_number, e)
raise e
# def adjust_truck_freight(self, parameter_list):
# """purpose
# Parameters
# ----------
# Returns
# -------
# DOES THIS WORK ? dataframes
# """
# gross_output = pd.read_excel('./EnergyIntensityIndicators/Industry/Data/BLS_BEA_Data.xlsx', sheet_name='BLS_Data_011920', index_col=0)
# gross_output = gross_output.transpose() # Note: Gross output in million 2005 dollars from BLS database for their employment projections input-output model,
# # PNNL spreadsheet: BLS_output_data.xlsx in folder BLS_Industry_Data)
# vehicle_miles_fhwa_tvm1 = pd.read_excel('https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx', header=4, index_col=0)
# old_methodology_2007_extrapolated = gross_output.iloc[2007] / gross_output.iloc[2006] * vehicle_miles_fhwa_tvm1.iloc[2006, :]
# old_series_scaled_to_new = vehicle_miles_fhwa_tvm1 * vehicle_miles_fhwa_tvm1.iloc[2007, :] / old_methodology_2007_extrapolated
# return old_series_scaled_to_new
def passenger_based_activity(self):
"""
Time series for the activity measures for passenger transportation
1970-76: Oak Ridge National Laboratory. 1993. Transportation Energy Data Book, Edition 13. ORNL-6743. Oak Ridge, Tennessee, Table 3.30, p. 3-46.
1977-2017 American Public Transportation Association. 2019 Public Transportation Fact Book. Appendix A, Table 3, Pt. A
Note: Transit bus does not include trolley bus because APTA did not not report electricity consumption for trolley buses separately for all years.
https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/
Note: Series not continuous for transit bus between 2006 and 2007; estimation procedures changed for bus systems outside urban areas
Note: Series not continuous between 1983 and 1984.
"""
# Historical data from PNNL
passenger_based_activity = \
pd.read_csv(
'./EnergyIntensityIndicators/Transportation/passenger_based_activity.csv'
).set_index('Year').rename(
columns={'Motor-cycles': 'Motorcycles',
'Light Truck': 'Light Trucks',
'Transit Bus': 'Urban Bus',
'Para-Transit': 'Paratransit',
'All Carriers (pass miles), millions': 'Commercial Carriers'}
)
# # Passenger cars and light trucks
# fha_table_vm1 = pd.read_excel(
# 'https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx',
# header=4, index_col=0
# )
# # Bus / Transit
# apta_table3 = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='3', skiprows=7, skipfooter=42, index_col=0
# ) # 1997-2017 apta_table3
# # Bus / Intercity: see revised_intercity_bus_estimates
# # Bus / School: see revised_intercity_bus_estimates
# # Paratransit
# paratransit_activity_1984_2017 = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='3', skiprows=7, skipfooter=42, usecols='A,G', index_col=0
# ) # 1997-2017 apta_table3
# # Commercial Air Carrier
# commercial_air_carrier = pd.read_excel(
# 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_02_01312021.xlsx',
# skiprows=9, skip_footer=42, index_col=0, use_cols='B:D'
# )
# # Urban Rail (Commuter)
# urban_rail_commuter = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='3', skiprows=7, skipfooter=42, index_col=0,
# use_cols='A,L') # 1997-2017 apta_table3
# # Urban Rail (Light, Heavy)
# urban_rail_light_heavy = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='3', skiprows=7, skipfooter=42, usecols='A,O,P', index_col=0
# ) # 1997-2017 apta_table3
# # Intercity Rail (Amtrak) Note: Amtrak data is compiled by fiscal year rather than calendar year
# intercity_rail_2017 = pd.read_excel(
# 'https://www.bts.gov/sites/bts.dot.gov/files/table_01_40_091820.xlsx',
# skiprows=1, skipfooter=25, index_col=0
# )
# intercity_rail_2017 = intercity_rail_2017.transpose()
# intercity_rail_2017 = intercity_rail_2017[['Intercity/Amtraki']] # not sure how the i superscript shows up
# 'Bureau of Transportation Statistics, National Transportation Statistics, Table 1-40: U.S. Passenger-Miles
return passenger_based_activity
def passenger_based_energy_use(self):
"""Calculate the fuel consumption in Btu for passenger transportation
"""
passenger_based_energy_use = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/passenger_based_energy_use.csv'
).set_index('Year').rename(
columns={
'Light Truck': 'Light Trucks',
'Para-transit': 'Paratransit',
'Intercity Rail (Amtrak)': 'Intercity Rail'
})
passenger_based_energy_use['Paratransit'] = \
passenger_based_energy_use['Paratransit'].fillna(0.000000001)
return passenger_based_energy_use
def fuel_heat_content(self, gross=True):
"""Assumed Btu content of the various types of petroleum products.
This dataframe is not time variant (no need to update it with subsquent indicator updates)
Parameters
----------
gross : bool
if True use gross fuel heat content, if False use net
"""
fuel_heat_content = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/Data/fuel_heat_content.csv'
)
if gross:
fuel_heat_content_df = fuel_heat_content[[
'Fuel', 'Gross Content Fmt'
]]
else:
fuel_heat_content_df = fuel_heat_content[['Fuel', 'Net Content']]
return fuel_heat_content_df
# def fuel_consump(self, parameter_list):
# """
# Time series of fuel consumption for the major transportation
# subsectors. Data are generally in
# millions gallons or barrels of petroleum.
# Parameters
# ----------
# Returns
# -------
# """
# fha_table_vm1 = pd.read_excel(
# 'https://www.fhwa.dot.gov/policyinformation/statistics/2018/xls/vm1.xlsx',
# header=4, index_col=0
# ) # 2007-2017 FHA Highway Statistics Table VM-1
# apta_table59 = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='59', skiprows=2, skipfooter=24, index_col=0
# ) # 1997-2017 apta_table3
# apta_table60 = pd.read_excel(
# 'https://www.apta.com/wp-content/uploads/2020-APTA-Fact-Book-Appendix-A.xlsx',
# sheet_name='60', skiprows=2, skipfooter=24, index_col=0
# ) # 1997-2017 apta_table3
# # swb_vehciles_all_fuel = fha_table_vm1
# # motorcycles_all_fuel_1970_2017 = self.import_tedb_data(table_number='A_02') # Alternative: 2017 data from Highway Statistics, Table VM-1.
# # lwb_vehicles_all_fuel = fha_table_vm1 # 2007-2017 FHA Highway Statistics Table VM-1
# # bus_urban_diesel = apta_table59 #[] #Table 59 in Appendix A, Bus Fuel Consumption (downloaded Excel file)
# # from: https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/
# # #Excel file: 2019-APTA-Fact-Book-Appendix-A.xlsx
# # bus_urban_cng = apta_table59 #[] #same as bus_urban_diesel
# # bus_urban_lng = apta_table59 #[] #same as bus_urban_diesel
# # bus_urban_other = apta_table59 #[] #same as bus_urban_diesel
# # bus_urban_lpg = apta_table59 #[] #same as bus_urban_diesel
# # paratransit_diesel = apta_table60 #[] # http://www.apta.com/resources/statistics/Pages/transitstats.aspx
# # # https://www.apta.com/research-technical-resources/transit-statistics/public-transportation-fact-book/
# # Table 60 in Appendix A, Demand Response Fuel Consumption
# # general_aviation = self.import_tedb_data(table_number='A_08')
# # commercial_air_carrier = self.import_tedb_data(table_number='A_09')
# # rail_commuter = self.import_tedb_data(table_number='A_13')
# # rail_urban = self.import_tedb_data(table_number='A_15')
# # class_I_freight_distillate_fuel_oil = self.import_tedb_data(table_number='A_13')
# # pipeline_natural_gas_million_cu_ft = self.import_tedb_data(table_number='A_12')
# fuel_consump_df = []
# return fuel_consump_df
def freight_based_energy_use(self, freight_based_activity):
"""Calculate the fuel consumption in Btu for freight transportation
Need FuelConsump, Fuel Heat Content
"""
# Historical data from PNNL
freight_based_energy_use = pd.read_csv(
'./EnergyIntensityIndicators/Transportation/' +
'freight_based_energy_use.csv').set_index('Year')
# This was used by PNNL to negate oil pipeline data (no reliable data source)
freight_based_energy_use['Oil Pipeline'] = \
freight_based_activity['Oil Pipeline'].multiply(0.000001)
# natural_gas_delivered_to_end_users = \
# self.aer_2010_table_65 # Column AH, million cu. ft.
# natural_gas_delivered_lease_plant_pipeline_fuel = \
# self.mer_table_43_nov2019 # Column M - column D - column I
# natural_gas_delivered_lease_plant_pipeline_fuel.at[0] = 0.000022395
# natural_gas_consumption_million_tons = \
# natural_gas_delivered_to_end_users.multiply(
# natural_gas_delivered_lease_plant_pipeline_fuel, axis=1)
# avg_length_natural_gas_shipment_miles = 620
# freight_based_energy_use['Natural Gas Pipeline'] = \
# natural_gas_consumption_million_tons.multiply(
# avg_length_natural_gas_shipment_miles
# )
return freight_based_energy_use
def freight_based_activity(self):
"""Time series for the activity measures for passenger transportation
"""
# Historical data, from PNNL
freight_based_activity = \
pd.read_csv('./EnergyIntensityIndicators/Transportation/' +
'freight_based_activity.csv').set_index('Year')
freight_based_activity['Single-Unit Truck'] = \
freight_based_activity['Single-Unit Truck'].multiply(3) # Assumes 3-ton avg. load
freight_based_activity['Oil Pipeline'] = \
freight_based_activity['Oil Pipeline'].multiply(0.000001)
# USDOT, Bureau of Transportation Statistics
# https://www.bts.gov/content/energy-intensity-class-i-railroad-freight-service Table 4-25
# class_1_rail = pd.read_excel('https://www.bts.gov/sites/bts.dot.gov/files/table_04_25_112219.xlsx',
# skiprows=1, skipfooter=9, index_col=0)
# class_1_rail = class_1_rail.transpose()
# class_1_rail = class_1_rail[['Revenue freight ton-miles (millions)']]
# air_carrier = \
# pd.read_excel(
# 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_02_01312021.xlsx',
# skiprows=9, skip_footer=42, index_col=1, use_cols='B,H')
# waterborne_vessles = \
# pd.read_excel(
# 'https://tedb.ornl.gov/wp-content/uploads/2021/02/Table10_05_01312021.xlsx',
# skiprows=9, skip_footer=42, index_col=1, use_cols='B,D')
# gas_pipeline = self.mer_table_43_nov2019
return freight_based_activity
# def water_freight_regression(self, intensity, actual_dd):
# """[summary]
# Args:
# intensity ([type]): [description]
# actual_dd ([type]): [description]
# Returns:
# predicted_value ([type]): [description]
# """
# X = np.log(intensity['intensity'])
# Y = intensity['Year']
# reg = linear_model.LinearRegression()
# reg.fit(X, Y)
# coefficients = reg.coef_
# predicted_value = reg.predict(actual_dd) # Predicted value of the intensity based on actual degree days
# return predicted_value
# def detailed_data_school_buses(self, parameter_list):
# """
# Adjusted miles - use number of buses times
# interpolated values for annual miles per bus.
# For years ending 0 and 5, use published data.
# Thus, adjusted values match in years ending 0 and 5 with Column 4,
# and for 1994. After 1994, values taken directly from
# School Bus Fleet Magazine estimates, adjusted for missing values
# Args:
# parameter_list ([type]): [description]
# Returns:
# revised_pass_miles (pd.DataFrame): [description]
# """
# avg_load_factor = 23
# adjusted_vehicle_miles = pd.read_csv(
# './EnergyIntensityIndicators/Transportation/Data/adjusted_data_school_bus.csv'
# )
# revised_pass_miles = adjusted_vehicle_miles.multiply(avg_load_factor * 0.001)
# return revised_pass_miles
def collect_data(self):
"""Method to collect freight and passenger energy and activity data.
This method should be adjusted to call dataframe building methods
rather than reading csvs, when those are ready
Gather Energy Data to use in LMDI Calculation TBtu and Activity Data
to use in LMDI Calculation passenger-miles [P-M], ton-miles [T-M]
Returns
-------
data_dict : dict
"""
passenger_based_energy_use = self.passenger_based_energy_use()
passenger_based_activity = self.passenger_based_activity()
freight_based_activity = self.freight_based_activity()
freight_based_energy_use = \
self.freight_based_energy_use(freight_based_activity)
data_dict = {
'All_Passenger': {
'Highway': {
'Passenger Cars and Trucks': {
'Passenger Car – SWB Vehicles': {
'Passenger Car': {
'energy': {
'deliv':
passenger_based_energy_use[[
'Passenger Car'
]]
},
'activity':
passenger_based_activity[['Passenger Car']]
},
'SWB Vehicles': {
'energy': {
'deliv':
passenger_based_energy_use[[
'SWB Vehicles'
]]
},
'activity':
passenger_based_activity[['SWB Vehicles']]
}
},
'Light Trucks – LWB Vehicles': {
'Light Trucks': {
'energy': {
'deliv':
passenger_based_energy_use[[
'Light Trucks'
]]
},
'activity':
passenger_based_activity[['Light Trucks']]
},
'LWB Vehicles': {
'energy': {
'deliv':
passenger_based_energy_use[[
'LWB Vehicles'
]]
},
'activity':
passenger_based_activity[['LWB Vehicles']]
}
},
'Motorcycles': {
'energy': {
'deliv':
passenger_based_energy_use[['Motorcycles']]
},
'activity':
passenger_based_activity[['Motorcycles']]
}
},
'Buses': {
'Urban Bus': {
'energy': {
'deliv':
passenger_based_energy_use[['Urban Bus']]
},
'activity': passenger_based_activity[['Urban Bus']]
},
'Intercity Bus': {
'energy': {
'deliv':
passenger_based_energy_use[['Intercity Bus']]
},
'activity':
passenger_based_activity[['Intercity Bus']]
},
'School Bus': {
'energy': {
'deliv':
passenger_based_energy_use[['School Bus']]
},
'activity':
passenger_based_activity[['School Bus']]
}
},
'Paratransit': {
'energy': {
'deliv':
passenger_based_energy_use[['Paratransit']]
},
'activity': passenger_based_activity[['Paratransit']]
}
},
'Rail': {
'Urban Rail': {
'Commuter Rail': {
'energy': {
'deliv':
passenger_based_energy_use[['Commuter Rail']]
},
'activity':
passenger_based_activity[['Commuter Rail']]
},
'Heavy Rail': {
'energy': {
'deliv':
passenger_based_energy_use[['Heavy Rail']]
},
'activity':
passenger_based_activity[['Heavy Rail']]
},
'Light Rail': {
'energy': {
'deliv':
passenger_based_energy_use[['Light Rail']]
},
'activity':
passenger_based_activity[['Light Rail']]
}
},
'Intercity Rail': {
'energy': {
'deliv':
passenger_based_energy_use[['Intercity Rail']]
},
'activity':
passenger_based_activity[['Intercity Rail']]
}
},
'Air': {
'Commercial Carriers': {
'energy': {
'deliv': passenger_based_energy_use[['Carrier']]
},
'activity':
passenger_based_activity[['Commercial Carriers']]
},
'General Aviation': {
'energy': {
'deliv':
passenger_based_energy_use[['General Aviation']]
},
'activity':
passenger_based_activity[['General Aviation']]
}
}
},
'All_Freight': {
'Highway': {
'Single-Unit Truck': {
'energy': {
'deliv':
freight_based_energy_use[['Single-Unit Truck']]
},
'activity':
freight_based_activity[['Single-Unit Truck']]
},
'Combination Truck': {
'energy': {
'deliv':
freight_based_energy_use[['Combination Truck']]
},
'activity':
freight_based_activity[['Combination Truck']]
}
},
'Rail': {
'energy': {
'deliv': freight_based_energy_use[['Rail']]
},
'activity': freight_based_activity[['Rail']]
},
'Air': {
'energy': {
'deliv': freight_based_energy_use[['Air']]
},
'activity': freight_based_activity[['Air']]
},
'Waterborne': {
'energy': {
'deliv': freight_based_energy_use[['Waterborne']]
},
'activity': freight_based_activity[['Waterborne']]
},
'Pipeline': {
'Oil Pipeline': {
'energy': {
'deliv': freight_based_energy_use[['Oil Pipeline']]
},
'activity': freight_based_activity[['Oil Pipeline']]
},
'Natural Gas Pipeline': {
'energy': {
'deliv':
freight_based_energy_use[['Natural Gas Pipeline']]
},
'activity':
freight_based_activity[['Natural Gas Pipeline']]
}
}
}
}
data_dict = {'All_Transportation': data_dict}
return data_dict
def main(self, breakout, calculate_lmdi): # base_year=None,
"""Decompose Energy use for the Transportation sector
"""
data_dict = self.collect_data()
if self.level_of_aggregation == 'personal_vehicles_aggregate':
categories = {
'Passenger Car': None,
'Light Truck': None,
'Motorcycles': None
}
results = None
else:
results_dict, results = self.get_nested_lmdi(
level_of_aggregation=self.level_of_aggregation,
breakout=breakout,
calculate_lmdi=calculate_lmdi,
raw_data=data_dict,
lmdi_type='LMDI-I')
return results_dict