def test_writing_flows(self): """Test that first 100 flows are written to 100 files in JSON_LD archive """ flowlist = fedelemflowlist.get_flows(preferred_only=False) #Test first 100 flowlist = flowlist.iloc[0:100] try: flowlist_json_file = outputpath + 'test_FedElemFlowList_first100.zip' #Remove the test file if it already exists if os.path.exists(flowlist_json_file): os.remove(flowlist_json_file) fedelemflowlist.write_jsonld(flowlist, flowlist_json_file) except PermissionError: log.error("No permission to write to file " + flowlist_json_file) try: flowlist_json = zipfile.ZipFile(flowlist_json_file) test_extract_path = outputpath + 'test_FedElemFlowList_first100/' flowlist_json.extractall(test_extract_path) flows_path = test_extract_path + 'flows/' categories_path = test_extract_path + 'categories/' flow_files = os.listdir(flows_path) len_extracted_flow_files = len(flow_files) #Clean up for f in flow_files: os.remove(flows_path + f) category_files = os.listdir(categories_path) for f in category_files: os.remove(categories_path + f) os.removedirs(flows_path) os.removedirs(categories_path) os.removedirs(test_extract_path) except FileNotFoundError: log.error(flowlist_json_file + ' not found') self.assertEqual(100, len_extracted_flow_files)
def get(subset=None) -> pd.DataFrame: """ Returns a dataframe of inventory based methods. :param subset: a list of dictionary keys from available inventories, if none selected all availabile inventories will be generated :return: df in standard LCIAmethod format """ method = df.data_frame(list()) method['Characterization Factor'] = pd.to_numeric( method['Characterization Factor']) if subset == None: list_of_inventories = subsets.get_subsets() else: list_of_inventories = subset alt_units = flowlist.get_alt_conversion() for inventory in list_of_inventories: flows = flowlist.get_flows(subset=inventory) flows.drop([ 'Formula', 'Synonyms', 'Class', 'External Reference', 'Preferred', 'AltUnit', 'AltUnitConversionFactor' ], axis=1, inplace=True) flows['Indicator'] = inventory flows['Indicator unit'] = subsets.get_inventory_unit(inventory) flows['Characterization Factor'] = 1 # Apply unit conversions where flow unit differs from indicator unit flows_w_conversion = pd.merge( flows, alt_units, how='left', left_on=['Flowable', 'Indicator unit', 'Unit'], right_on=['Flowable', 'AltUnit', 'Unit']) flows_w_conversion.loc[(flows_w_conversion['AltUnit'] == flows_w_conversion['Indicator unit']), 'Characterization Factor'] = flows_w_conversion[ 'AltUnitConversionFactor'] flows_w_conversion.drop( ['AltUnit', 'AltUnitConversionFactor', 'InverseConversionFactor'], axis=1, inplace=True) method = pd.concat([method, flows_w_conversion], ignore_index=True) method['Method'] = 'Inventory' return method
import fedelemflowlist import pandas as pd from fedelemflowlist.globals import outputpath, flow_list_fields #Set name of mapping file. More than one mapping file can be used mapping_to_use = ['TRACI2.1'] if __name__ == '__main__': mapping = fedelemflowlist.get_flowmapping(mapping_to_use) # Get Flow UUIDs for flows used in selected mapping mapping_flow_uuids = pd.DataFrame(pd.unique(mapping['TargetFlowUUID']), columns=["Flow UUID"]) # Get all flows all_flows = fedelemflowlist.get_flows() all_UUIDs = all_flows['Flow UUID'] # Subset all flows to get just those used in selected mapping flows_used_in_mapping = pd.merge(all_flows, mapping_flow_uuids) flows_used_UUIDs = flows_used_in_mapping['Flow UUID'] # Flows not in mappings flows_notused_UUIDs = set(all_UUIDs) - set(flows_used_UUIDs) len(flows_notused_UUIDs) flows_notused = all_flows[all_flows['Flow UUID'].isin(flows_notused_UUIDs)] #unique flowables flowable_fields = ['Class', 'Flowable', 'CAS No', 'Formula', 'Synonyms'] flowables_notused = flows_notused[flowable_fields]
def modify_list(df): modifications = [ 'Caprolactam (See Modification)', 'Hydrogen sulfide (See Modification)', 'Methyl ethyl ketone (2-Butanone) (See Modification)', ] df = df[~df['HAP'].isin(modifications)] return df if __name__ == '__main__': df = extract_table() df = modify_list(df) df = modify_CAS(df) fl = fedelemflowlist.get_flows() fl = fl[fl['Context'] == 'emission/air'] fl.drop( columns=['Context', 'Flow UUID', 'AltUnit', 'AltUnitConversionFactor'], inplace=True) merged_df = df.merge(fl, how='left', on='CAS No') merged_df.loc[merged_df['CAS No'] == '--0', 'CAS No'] = None # for flows unmatched by CAS, remove extraneous data within flow name # and match on flowable (case insensitive) no_match = merged_df.loc[merged_df['Flowable'].isnull()] no_match[['f1', 'f2']] = no_match['HAP'].str.split("\(", expand=True) no_match['f3'] = no_match['f1'].str.strip().str[-1:].str.isnumeric() no_match.loc[no_match['f3'], 'f1'] = no_match['f1'].str[:-2] no_match['HAP2'] = no_match['f1'].str.strip()
""" Writes preferred flows to a JSON-LD archive in the output folder """ import fedelemflowlist from fedelemflowlist.globals import outputpath,flow_list_specs if __name__ == '__main__': all_flows = fedelemflowlist.get_flows(preferred_only=False) fedelemflowlist.write_jsonld(all_flows,outputpath+'FedElemFlowList_'+flow_list_specs['list_version']+'_all.zip') all_flows.to_excel(outputpath+'FedElemFlowList_'+flow_list_specs['list_version']+'_all.xlsx',index=False)
Write csvs of the new flows and expired flows in csv format, write new flows to JSON-LD Currently compares only by UUID. """ import pandas as pd import fedelemflowlist from fedelemflowlist.globals import outputpath, flow_list_specs # get the current list # Enter name of old version here. Must be in output folder old_version_parquet = 'FedElemFlowListMaster1.0.parquet' old_version = '1.0' if __name__ == '__main__': current_list = fedelemflowlist.get_flows() # get UUIDs current_UUIDs = current_list['Flow UUID'] # load old version from output folder list_file = outputpath + old_version_parquet old_list = pd.read_parquet(list_file, engine="pyarrow") old_list_UUIDs = old_list['Flow UUID'] new_UUIDS = list(set(current_UUIDs) - set(old_list_UUIDs)) new_flows = current_list[current_list['Flow UUID'].isin(new_UUIDS)] new_flows.to_csv(outputpath + 'new_flows' + old_version + 'to' + flow_list_specs['list_version'] + '.csv', index=False)
def setUp(self): """Get flowlist used for all tests """ self.flowmappings = fedelemflowlist.get_flowmapping() self.flowlist = self.flowlist = fedelemflowlist.get_flows()
def get_elci_emissions(yaml_load): """Generate plant level emissions from eLCI package.""" try: import electricitylci import electricitylci.model_config as config except ImportError: log.error("Must install ElectricityLCI package in order to generate " "plant level air emissions.") import fedelemflowlist as fl config.model_specs = config.build_model_class('ELCI_1') config.model_specs.include_upstream_processes = False config.model_specs.regional_aggregation = 'US' config.model_specs.include_renewable_generation = False config.model_specs.include_netl_water = True config.model_specs.stewicombo_file = 'ELCI_1' if 'DMR' not in config.model_specs.inventories_of_interest.keys(): config.model_specs.inventories_of_interest['DMR'] = 2016 emissions = electricitylci.get_generation_process_df() emissions = emissions.loc[emissions['stage_code'] == 'Power plant'] # FlowUUID is not null (will remove RCRA flows) emissions = emissions.loc[~emissions['FlowUUID'].isna()] column_dict = {'Balancing Authority Name': 'Location', 'FuelCategory': 'SectorProducedBy', 'FlowName': 'Flowable', 'Compartment': 'Compartment', 'FlowUUID': 'FlowUUID', 'FlowAmount': 'FlowAmount', 'Unit': 'Unit', 'Year': 'Year', 'source_string': 'MetaSources'} col_list = [c for c in column_dict.keys() if c in emissions] emissions_df = emissions[col_list] emissions_df = emissions_df.rename(columns=column_dict) # Update Compartment from FEDEFL fedefl = fl.get_flows()[['Flow UUID', 'Context']] emissions_df = emissions_df.merge(fedefl, how='left', left_on='FlowUUID', right_on='Flow UUID') emissions_df.drop(columns=['Flow UUID', 'Compartment'], inplace=True) # Assign other fields emissions_df['LocationSystem'] = 'BAA' emissions_df['FlowType'] = 'ELEMENTARY_FLOW' emissions_df['Class'] = 'Chemicals' # Update SectorProducedBy mapping = get_activitytosector_mapping('eLCI') mapping = mapping[['Activity', 'Sector']] emissions_df_mapped = emissions_df.merge(mapping, how='left', left_on='SectorProducedBy', right_on='Activity') emissions_df_mapped.drop(columns=['SectorProducedBy', 'Activity'], inplace=True) emissions_df_mapped.rename(columns={'Sector': 'SectorProducedBy'}, inplace=True) emissions_df_mapped.dropna(subset=['SectorProducedBy'], inplace=True) return emissions_df_mapped
"""Writes preferred flows to a JSON-LD archive in the output folder.""" import fedelemflowlist from fedelemflowlist.globals import outputpath, flow_list_specs if __name__ == '__main__': preferred_flows = fedelemflowlist.get_flows(preferred_only=True) fedelemflowlist.write_jsonld( preferred_flows, outputpath + 'FedElemFlowList_' + flow_list_specs['list_version'] + '_preferred.zip')
def setUp(self): """Get flowlist used for all tests.""" self.flowlist = fedelemflowlist.get_flows()