def test_writing_flows(self):
"""Test that first 100 flows are written to 100 files in JSON_LD archive
"""
flowlist = fedelemflowlist.get_flows(preferred_only=False)
#Test first 100
flowlist = flowlist.iloc[0:100]
try:
flowlist_json_file = outputpath + 'test_FedElemFlowList_first100.zip'
#Remove the test file if it already exists
if os.path.exists(flowlist_json_file):
os.remove(flowlist_json_file)
fedelemflowlist.write_jsonld(flowlist, flowlist_json_file)
except PermissionError:
log.error("No permission to write to file " + flowlist_json_file)
try:
flowlist_json = zipfile.ZipFile(flowlist_json_file)
test_extract_path = outputpath + 'test_FedElemFlowList_first100/'
flowlist_json.extractall(test_extract_path)
flows_path = test_extract_path + 'flows/'
categories_path = test_extract_path + 'categories/'
flow_files = os.listdir(flows_path)
len_extracted_flow_files = len(flow_files)
#Clean up
for f in flow_files:
os.remove(flows_path + f)
category_files = os.listdir(categories_path)
for f in category_files:
os.remove(categories_path + f)
os.removedirs(flows_path)
os.removedirs(categories_path)
os.removedirs(test_extract_path)
except FileNotFoundError:
log.error(flowlist_json_file + ' not found')
self.assertEqual(100, len_extracted_flow_files)
def get(subset=None) -> pd.DataFrame:
"""
Returns a dataframe of inventory based methods.
:param subset: a list of dictionary keys from available inventories, if
none selected all availabile inventories will be generated
:return: df in standard LCIAmethod format
"""
method = df.data_frame(list())
method['Characterization Factor'] = pd.to_numeric(
method['Characterization Factor'])
if subset == None:
list_of_inventories = subsets.get_subsets()
else:
list_of_inventories = subset
alt_units = flowlist.get_alt_conversion()
for inventory in list_of_inventories:
flows = flowlist.get_flows(subset=inventory)
flows.drop([
'Formula', 'Synonyms', 'Class', 'External Reference', 'Preferred',
'AltUnit', 'AltUnitConversionFactor'
],
axis=1,
inplace=True)
flows['Indicator'] = inventory
flows['Indicator unit'] = subsets.get_inventory_unit(inventory)
flows['Characterization Factor'] = 1
# Apply unit conversions where flow unit differs from indicator unit
flows_w_conversion = pd.merge(
flows,
alt_units,
how='left',
left_on=['Flowable', 'Indicator unit', 'Unit'],
right_on=['Flowable', 'AltUnit', 'Unit'])
flows_w_conversion.loc[(flows_w_conversion['AltUnit'] ==
flows_w_conversion['Indicator unit']),
'Characterization Factor'] = flows_w_conversion[
'AltUnitConversionFactor']
flows_w_conversion.drop(
['AltUnit', 'AltUnitConversionFactor', 'InverseConversionFactor'],
axis=1,
inplace=True)
method = pd.concat([method, flows_w_conversion], ignore_index=True)
method['Method'] = 'Inventory'
return method
import fedelemflowlist
import pandas as pd
from fedelemflowlist.globals import outputpath, flow_list_fields
#Set name of mapping file. More than one mapping file can be used
mapping_to_use = ['TRACI2.1']
if __name__ == '__main__':
mapping = fedelemflowlist.get_flowmapping(mapping_to_use)
# Get Flow UUIDs for flows used in selected mapping
mapping_flow_uuids = pd.DataFrame(pd.unique(mapping['TargetFlowUUID']),
columns=["Flow UUID"])
# Get all flows
all_flows = fedelemflowlist.get_flows()
all_UUIDs = all_flows['Flow UUID']
# Subset all flows to get just those used in selected mapping
flows_used_in_mapping = pd.merge(all_flows, mapping_flow_uuids)
flows_used_UUIDs = flows_used_in_mapping['Flow UUID']
# Flows not in mappings
flows_notused_UUIDs = set(all_UUIDs) - set(flows_used_UUIDs)
len(flows_notused_UUIDs)
flows_notused = all_flows[all_flows['Flow UUID'].isin(flows_notused_UUIDs)]
#unique flowables
flowable_fields = ['Class', 'Flowable', 'CAS No', 'Formula', 'Synonyms']
flowables_notused = flows_notused[flowable_fields]
def modify_list(df):
modifications = [
'Caprolactam (See Modification)',
'Hydrogen sulfide (See Modification)',
'Methyl ethyl ketone (2-Butanone) (See Modification)',
]
df = df[~df['HAP'].isin(modifications)]
return df
if __name__ == '__main__':
df = extract_table()
df = modify_list(df)
df = modify_CAS(df)
fl = fedelemflowlist.get_flows()
fl = fl[fl['Context'] == 'emission/air']
fl.drop(
columns=['Context', 'Flow UUID', 'AltUnit', 'AltUnitConversionFactor'],
inplace=True)
merged_df = df.merge(fl, how='left', on='CAS No')
merged_df.loc[merged_df['CAS No'] == '--0', 'CAS No'] = None
# for flows unmatched by CAS, remove extraneous data within flow name
# and match on flowable (case insensitive)
no_match = merged_df.loc[merged_df['Flowable'].isnull()]
no_match[['f1', 'f2']] = no_match['HAP'].str.split("\(", expand=True)
no_match['f3'] = no_match['f1'].str.strip().str[-1:].str.isnumeric()
no_match.loc[no_match['f3'], 'f1'] = no_match['f1'].str[:-2]
no_match['HAP2'] = no_match['f1'].str.strip()
"""
Writes preferred flows to a JSON-LD archive in the output folder
"""
import fedelemflowlist
from fedelemflowlist.globals import outputpath,flow_list_specs
if __name__ == '__main__':
all_flows = fedelemflowlist.get_flows(preferred_only=False)
fedelemflowlist.write_jsonld(all_flows,outputpath+'FedElemFlowList_'+flow_list_specs['list_version']+'_all.zip')
all_flows.to_excel(outputpath+'FedElemFlowList_'+flow_list_specs['list_version']+'_all.xlsx',index=False)
Write csvs of the new flows and expired flows in csv format, write new flows to JSON-LD
Currently compares only by UUID.
"""
import pandas as pd
import fedelemflowlist
from fedelemflowlist.globals import outputpath, flow_list_specs
# get the current list
# Enter name of old version here. Must be in output folder
old_version_parquet = 'FedElemFlowListMaster1.0.parquet'
old_version = '1.0'
if __name__ == '__main__':
current_list = fedelemflowlist.get_flows()
# get UUIDs
current_UUIDs = current_list['Flow UUID']
# load old version from output folder
list_file = outputpath + old_version_parquet
old_list = pd.read_parquet(list_file, engine="pyarrow")
old_list_UUIDs = old_list['Flow UUID']
new_UUIDS = list(set(current_UUIDs) - set(old_list_UUIDs))
new_flows = current_list[current_list['Flow UUID'].isin(new_UUIDS)]
new_flows.to_csv(outputpath + 'new_flows' + old_version + 'to' +
flow_list_specs['list_version'] + '.csv',
index=False)
def setUp(self):
"""Get flowlist used for all tests
"""
self.flowmappings = fedelemflowlist.get_flowmapping()
self.flowlist = self.flowlist = fedelemflowlist.get_flows()
def get_elci_emissions(yaml_load):
"""Generate plant level emissions from eLCI package."""
try:
import electricitylci
import electricitylci.model_config as config
except ImportError:
log.error("Must install ElectricityLCI package in order to generate "
"plant level air emissions.")
import fedelemflowlist as fl
config.model_specs = config.build_model_class('ELCI_1')
config.model_specs.include_upstream_processes = False
config.model_specs.regional_aggregation = 'US'
config.model_specs.include_renewable_generation = False
config.model_specs.include_netl_water = True
config.model_specs.stewicombo_file = 'ELCI_1'
if 'DMR' not in config.model_specs.inventories_of_interest.keys():
config.model_specs.inventories_of_interest['DMR'] = 2016
emissions = electricitylci.get_generation_process_df()
emissions = emissions.loc[emissions['stage_code'] == 'Power plant']
# FlowUUID is not null (will remove RCRA flows)
emissions = emissions.loc[~emissions['FlowUUID'].isna()]
column_dict = {'Balancing Authority Name': 'Location',
'FuelCategory': 'SectorProducedBy',
'FlowName': 'Flowable',
'Compartment': 'Compartment',
'FlowUUID': 'FlowUUID',
'FlowAmount': 'FlowAmount',
'Unit': 'Unit',
'Year': 'Year',
'source_string': 'MetaSources'}
col_list = [c for c in column_dict.keys() if c in emissions]
emissions_df = emissions[col_list]
emissions_df = emissions_df.rename(columns=column_dict)
# Update Compartment from FEDEFL
fedefl = fl.get_flows()[['Flow UUID', 'Context']]
emissions_df = emissions_df.merge(fedefl, how='left',
left_on='FlowUUID',
right_on='Flow UUID')
emissions_df.drop(columns=['Flow UUID', 'Compartment'], inplace=True)
# Assign other fields
emissions_df['LocationSystem'] = 'BAA'
emissions_df['FlowType'] = 'ELEMENTARY_FLOW'
emissions_df['Class'] = 'Chemicals'
# Update SectorProducedBy
mapping = get_activitytosector_mapping('eLCI')
mapping = mapping[['Activity', 'Sector']]
emissions_df_mapped = emissions_df.merge(mapping, how='left',
left_on='SectorProducedBy',
right_on='Activity')
emissions_df_mapped.drop(columns=['SectorProducedBy', 'Activity'],
inplace=True)
emissions_df_mapped.rename(columns={'Sector': 'SectorProducedBy'},
inplace=True)
emissions_df_mapped.dropna(subset=['SectorProducedBy'],
inplace=True)
return emissions_df_mapped
"""Writes preferred flows to a JSON-LD archive in the output folder."""
import fedelemflowlist
from fedelemflowlist.globals import outputpath, flow_list_specs
if __name__ == '__main__':
preferred_flows = fedelemflowlist.get_flows(preferred_only=True)
fedelemflowlist.write_jsonld(
preferred_flows, outputpath + 'FedElemFlowList_' +
flow_list_specs['list_version'] + '_preferred.zip')
def setUp(self):
"""Get flowlist used for all tests."""
self.flowlist = fedelemflowlist.get_flows()
