def combineInventoriesforFacilitiesinBaseInventory(base_inventory,
inventory_dict,
filter_for_LCI=True,
remove_overlap=True):
"""Combine stewi inventories for all facilities present in base_inventory.
The base_inventory must be in the inventory_dict
:param base_inventory: reference inventory e.g. "TRI"
:param inventory_dict: dictionary of inventories and years,
e.g. {"TRI":"2014","NEI":"2014","RCRAInfo":"2015"}
:param filter_for_LCI: boolean. Passes through to stewi to filter_for_LCI.
See stewi.
:param remove_overlap: boolean. Removes overlap across inventories
based on preferences defined in globals
:return: Flow-By-Facility Combined Format
"""
inventory_acronyms = list(inventory_dict.keys())
facilitymatches = facilitymatcher.get_matches_for_inventories(
inventory_acronyms)
inventories = getInventoriesforFacilityMatches(inventory_dict,
facilitymatches,
filter_for_LCI,
base_inventory)
inventories = addChemicalMatches(inventories)
# Aggregate and remove overlap if requested
if remove_overlap:
inventories = aggregate_and_remove_overlap(inventories)
inventories = addBaseInventoryIDs(inventories, facilitymatches,
base_inventory)
return inventories
def combineFullInventories(inventory_dict, filter_for_LCI=True, remove_overlap=True):
"""Combines full stewi inventories
:param inventory_dict: dictionary of inventories and years,
e.g. {"TRI":"2014","NEI":"2014","RCRAInfo":"2015"}
:param filter_for_LCI: boolean. Passes through to stewi to filter_for_LCI. See stewi.
:param remove_overlap: boolean. Removes overlap across inventories based on preferences
defined in globals
:return: Flow-By-Facility Combined Format
"""
inventory_acronyms = list(inventory_dict.keys())
facilitymatches = facilitymatcher.get_matches_for_inventories(inventory_acronyms)
inventories = getInventoriesforFacilityMatches(inventory_dict, facilitymatches, filter_for_LCI)
inventories = addChemicalMatches(inventories)
# Aggregate and remove overlap if requested
if remove_overlap:
inventories = aggregate_and_remove_overlap(inventories)
# For combined records, preserve record of that in 'FacilityIDs_Combined'
inventories['FacilityIDs_Combined'] = inventories['FacilityID']
# Otherwise take the first ID as the facility ID
inventories['FacilityID'] = \
inventories['FacilityID'].apply(lambda x: get_id_before_underscore(x))
return inventories
def combineInventoriesforFacilityList(base_inventory, inventory_dict, facility_id_list,
filter_for_LCI=True, remove_overlap=True):
"""Combines stewi inventories for all facilities present in a given facility id list
The base_inventory must be in the inventory_dict
:param base_inventory: reference inventory e.g. "TRI"
:param inventory_dict: dictionary of inventories and years,
e.g. {"TRI":"2014","NEI":"2014","RCRAInfo":"2015"}
:param facility_id_list: list of facility ids from base_inventory
e.g. ['99501MPCLS1076O', '99501NCHRG459WB', '99515VNWTR590E1']
:param filter_for_LCI: boolean. Passes through to stewi to filter_for_LCI. See stewi.
:param remove_overlap: boolean.
Removes overlap across inventories based on preferences defined in globals
:return: Flow-By-Facility Combined Format
"""
inventory_acronyms = list(inventory_dict.keys())
facilitymatches = facilitymatcher.get_matches_for_id_list(base_inventory, facility_id_list,
inventory_acronyms)
inventories = getInventoriesforFacilityMatches(inventory_dict, facilitymatches, filter_for_LCI,
base_inventory)
# now remove the records from the base_inventory in the facility list
inventories_index_for_base_but_not_in_facilitylist = inventories[
(inventories['Source'] == base_inventory) &
(~inventories['FacilityID'].isin(facility_id_list))].index
inventories = inventories.drop(inventories_index_for_base_but_not_in_facilitylist, axis=0)
# Add in chemical matches
inventories = addChemicalMatches(inventories)
# Aggregate and remove overlap if requested
if remove_overlap:
inventories = aggregate_and_remove_overlap(inventories)
inventories = addBaseInventoryIDs(inventories, facilitymatches, base_inventory)
return inventories
def reassign_airplane_emissions(df, year, NAICS_level_value):
"""
Reassigns emissions from airplanes to NAICS associated with air
transportation instead of the NAICS assigned to airports
:param df: a dataframe of emissions and mapped faciliites from stewicombo
:param year: year as str
:param NAICS_level_value: desired NAICS aggregation level,
using sector_level_key, should match target_sector_level
:return: df
"""
import stewi
from stewicombo.overlaphandler import remove_default_flow_overlaps
from stewicombo.globals import addChemicalMatches
# subtract emissions for air transportation from airports in NEI
airport_NAICS = '4881'
air_transportation_SCC = '2275020000'
air_transportation_naics = '481111'
log.info('Reassigning emissions from air transportation from airports')
# obtain and prepare SCC dataset
df_airplanes = stewi.getInventory('NEI', year, stewiformat='flowbyprocess')
df_airplanes = df_airplanes[df_airplanes['Process'] ==
air_transportation_SCC]
df_airplanes['Source'] = 'NEI'
df_airplanes = addChemicalMatches(df_airplanes)
df_airplanes = remove_default_flow_overlaps(df_airplanes, SCC=True)
df_airplanes.drop(columns=['Process'], inplace=True)
facility_mapping_air = df[['FacilityID', 'NAICS']]
facility_mapping_air.drop_duplicates(keep='first', inplace=True)
df_airplanes = df_airplanes.merge(facility_mapping_air,
how='left',
on='FacilityID')
df_airplanes['Year'] = year
df_airplanes = df_airplanes[df_airplanes['NAICS'].str[0:len(airport_NAICS)]
== airport_NAICS]
# subtract airplane emissions from airport NAICS at individual facilities
df_planeemissions = df_airplanes[['FacilityID', 'FlowName', 'FlowAmount']]
df_planeemissions.rename(columns={'FlowAmount': 'PlaneEmissions'},
inplace=True)
df = df.merge(df_planeemissions, how='left', on=['FacilityID', 'FlowName'])
df[['PlaneEmissions']] = df[['PlaneEmissions']].fillna(value=0)
df['FlowAmount'] = df['FlowAmount'] - df['PlaneEmissions']
df.drop(columns=['PlaneEmissions'], inplace=True)
# add airplane emissions under air transport NAICS
df_airplanes.loc[:, 'NAICS_lvl'] = \
air_transportation_naics[0:NAICS_level_value]
df = pd.concat([df, df_airplanes], ignore_index=True)
return df
def reassign_airplane_emissions(df, year, NAICS_level_value):
import stewi
from stewicombo.overlaphandler import remove_default_flow_overlaps
from stewicombo.globals import addChemicalMatches
## subtract emissions for air transportation from airports in NEI
airport_NAICS = '4881'
air_transportation_SCC = '2275020000'
air_transportation_naics = '481111'
log.info('Reassigning emissions from air transportation from airports')
# obtain and prepare SCC dataset
df_airplanes = stewi.getInventory('NEI', year, stewiformat='flowbySCC')
df_airplanes = df_airplanes[df_airplanes['SCC'] == air_transportation_SCC]
df_airplanes['Source'] = 'NEI'
df_airplanes = addChemicalMatches(df_airplanes)
df_airplanes = remove_default_flow_overlaps(df_airplanes, SCC=True)
df_airplanes.drop(columns=['SCC'], inplace=True)
facility_mapping_air = df[['FacilityID', 'NAICS']]
facility_mapping_air.drop_duplicates(keep='first', inplace=True)
df_airplanes = df_airplanes.merge(facility_mapping_air,
how='left',
on='FacilityID')
df_airplanes['Year'] = year
df_airplanes = df_airplanes[df_airplanes['NAICS'].str[0:len(airport_NAICS)]
== airport_NAICS]
# subtract airplane emissions from airport NAICS at individual facilities
df_planeemissions = df_airplanes[['FacilityID', 'FlowName', 'FlowAmount']]
df_planeemissions.rename(columns={'FlowAmount': 'PlaneEmissions'},
inplace=True)
df = df.merge(df_planeemissions, how='left', on=['FacilityID', 'FlowName'])
df[['PlaneEmissions']] = df[['PlaneEmissions']].fillna(value=0)
df['FlowAmount'] = df['FlowAmount'] - df['PlaneEmissions']
df.drop(columns=['PlaneEmissions'], inplace=True)
# add airplane emissions under air transport NAICS
df_airplanes.loc[:, 'NAICS_lvl'] = air_transportation_naics[
0:NAICS_level_value]
df = pd.concat([df, df_airplanes], ignore_index=True)
return df
def stewicombo_to_sector(inventory_dict, NAICS_level, geo_scale, compartments):
"""
Returns emissions from stewicombo in fbs format
:param inventory_dict: a dictionary of inventory types and years (e.g.,
{'NEI':'2017', 'TRI':'2017'})
:param NAICS_level: desired NAICS aggregation level, using sector_level_key,
should match target_sector_level
:param geo_scale: desired geographic aggregation level ('national', 'state',
'county'), should match target_geoscale
:param compartments: list of compartments to include (e.g., 'water', 'air',
'soil'), use None to include all compartments
"""
from stewi.globals import output_dir as stw_output_dir
from stewi.globals import weighted_average
import stewi
import stewicombo
import facilitymatcher
from stewicombo.overlaphandler import remove_default_flow_overlaps
from stewicombo.globals import addChemicalMatches
from facilitymatcher import output_dir as fm_output_dir
NAICS_level_value = sector_level_key[NAICS_level]
## run stewicombo to combine inventories, filter for LCI, remove overlap
df = stewicombo.combineFullInventories(inventory_dict,
filter_for_LCI=True,
remove_overlap=True,
compartments=compartments)
df.drop(columns=['SRS_CAS', 'SRS_ID', 'FacilityIDs_Combined'],
inplace=True)
facility_mapping = pd.DataFrame()
# load facility data from stewi output directory, keeping only the facility IDs, and geographic information
inventory_list = list(inventory_dict.keys())
for i in range(len(inventory_dict)):
# define inventory name as inventory type + inventory year (e.g., NEI_2017)
inventory_name = inventory_list[i] + '_' + list(
inventory_dict.values())[i]
facilities = pd.read_csv(stw_output_dir + 'facility/' +
inventory_name + '.csv',
usecols=['FacilityID', 'State', 'County'],
dtype={'FacilityID': str})
if len(facilities[facilities.duplicated(subset='FacilityID',
keep=False)]) > 0:
log.info('Duplicate facilities in ' + inventory_name +
' - keeping first listed')
facilities.drop_duplicates(subset='FacilityID',
keep='first',
inplace=True)
facility_mapping = facility_mapping.append(facilities)
# Apply FIPS to facility locations
facility_mapping = apply_county_FIPS(facility_mapping)
## merge dataframes to assign facility information based on facility IDs
df = pd.merge(df, facility_mapping, how='left', on='FacilityID')
## Access NAICS From facility matcher and assign based on FRS_ID
all_NAICS = facilitymatcher.get_FRS_NAICSInfo_for_facility_list(
frs_id_list=None, inventories_of_interest_list=inventory_list)
all_NAICS = all_NAICS.loc[all_NAICS['PRIMARY_INDICATOR'] == 'PRIMARY']
all_NAICS.drop(columns=['PRIMARY_INDICATOR'], inplace=True)
all_NAICS = naics_expansion(all_NAICS)
if len(all_NAICS[all_NAICS.duplicated(subset=['FRS_ID', 'Source'],
keep=False)]) > 0:
log.info('Duplicate primary NAICS reported - keeping first')
all_NAICS.drop_duplicates(subset=['FRS_ID', 'Source'],
keep='first',
inplace=True)
df = pd.merge(df, all_NAICS, how='left', on=['FRS_ID', 'Source'])
# add levelized NAICS code prior to aggregation
df['NAICS_lvl'] = df['NAICS'].str[0:NAICS_level_value]
## subtract emissions for air transportation from airports in NEI
airport_NAICS = '4881'
air_transportation_SCC = '2275020000'
air_transportation_naics = '481111'
if 'NEI' in inventory_list:
log.info('Reassigning emissions from air transportation from airports')
# obtain and prepare SCC dataset
df_airplanes = stewi.getInventory('NEI',
inventory_dict['NEI'],
stewiformat='flowbySCC')
df_airplanes = df_airplanes[df_airplanes['SCC'] ==
air_transportation_SCC]
df_airplanes['Source'] = 'NEI'
df_airplanes = addChemicalMatches(df_airplanes)
df_airplanes = remove_default_flow_overlaps(df_airplanes, SCC=True)
df_airplanes.drop(columns=['SCC'], inplace=True)
facility_mapping_air = df[['FacilityID', 'NAICS']]
facility_mapping_air.drop_duplicates(keep='first', inplace=True)
df_airplanes = df_airplanes.merge(facility_mapping_air,
how='left',
on='FacilityID')
df_airplanes['Year'] = inventory_dict['NEI']
df_airplanes = df_airplanes[
df_airplanes['NAICS'].str[0:len(airport_NAICS)] == airport_NAICS]
# subtract airplane emissions from airport NAICS at individual facilities
df_planeemissions = df_airplanes[[
'FacilityID', 'FlowName', 'FlowAmount'
]]
df_planeemissions.rename(columns={'FlowAmount': 'PlaneEmissions'},
inplace=True)
df = df.merge(df_planeemissions,
how='left',
on=['FacilityID', 'FlowName'])
df[['PlaneEmissions']] = df[['PlaneEmissions']].fillna(value=0)
df['FlowAmount'] = df['FlowAmount'] - df['PlaneEmissions']
df.drop(columns=['PlaneEmissions'], inplace=True)
# add airplane emissions under air transport NAICS
df_airplanes.loc[:, 'NAICS_lvl'] = air_transportation_naics[
0:NAICS_level_value]
df = pd.concat([df, df_airplanes], ignore_index=True)
# update location to appropriate geoscale prior to aggregating
df.dropna(subset=['Location'], inplace=True)
df['Location'] = df['Location'].astype(str)
df = update_geoscale(df, geo_scale)
# assign grouping variables based on desired geographic aggregation level
grouping_vars = ['NAICS_lvl', 'FlowName', 'Compartment', 'Location']
# aggregate by NAICS code, FlowName, compartment, and geographic level
fbs = df.groupby(grouping_vars).agg({
'FlowAmount': 'sum',
'Year': 'first',
'Unit': 'first'
})
# add reliability score
fbs['DataReliability'] = weighted_average(df, 'ReliabilityScore',
'FlowAmount', grouping_vars)
fbs.reset_index(inplace=True)
# apply flow mapping
fbs = map_elementary_flows(fbs, inventory_list)
# rename columns to match flowbysector format
fbs = fbs.rename(columns={"NAICS_lvl": "SectorProducedBy"})
# add hardcoded data, depending on the source data, some of these fields may need to change
fbs['Class'] = 'Chemicals'
fbs['SectorConsumedBy'] = 'None'
fbs['SectorSourceName'] = 'NAICS_2012_Code'
fbs['FlowType'] = 'ELEMENTARY_FLOW'
fbs = assign_fips_location_system(fbs, list(inventory_dict.values())[0])
# add missing flow by sector fields
fbs = add_missing_flow_by_fields(fbs, flow_by_sector_fields)
# sort dataframe and reset index
fbs = fbs.sort_values(list(
flow_by_sector_fields.keys())).reset_index(drop=True)
return fbs