def return_fba_method_meta(sourcename, **kwargs):
"""
Return meta for a FlowByActivity method
:param sourcename: string, the FlowByActivity sourcename
:param kwargs: requires "year" defined
:return: meta object
"""
from flowsa.bibliography import load_source_dict
# load info from either a FBA method yaml or the literature yaml
fba = load_source_dict(sourcename)
# initiate empty dictionary
fba_dict = {}
# add year if creating an FBA metafile
if 'year' in kwargs:
fba_dict['data_year'] = kwargs['year']
try:
# loop through the FBA yaml and add info
for k, v in fba.items():
# include bib_id because this ifno pulled
# when generating a method bib
if k in ('author', 'source_name', 'source_url',
'original_data_download_date', 'date_accessed', 'bib_id'):
fba_dict[k] = str(v)
except:
log.warning('No metadata found for %s', sourcename)
fba_dict['meta_data'] = f'No metadata found for {sourcename}'
return fba_dict
def map_flows(fba,
from_fba_source,
flow_type='ELEMENTARY_FLOW',
ignore_source_name=False,
**kwargs):
"""
Applies mapping via esupy from fedelemflowlist or material
flow list to convert flows to standardized list of flows
:param fba: df flow-by-activity or flow-by-sector
:param from_fba_source: str Source name of fba list to look for mappings
:param flow_type: str either 'ELEMENTARY_FLOW', 'TECHNOSPHERE_FLOW',
or 'WASTE_FLOW'
:param ignore_source_name: bool, passed to apply_flow_mapping
:param kwargs: optional - keep_unmapped_rows: False if want
unmapped rows dropped, True if want to retain and keep_fba_columns:
boolean, True or False, indicate if want to maintain
'FlowName' and 'Compartment' columns in returned df
:return: df, with flows mapped using federal elementary flow list or
material flow list
"""
# prior to mapping elementary flows, ensure all data
# are in an annual format
fba = convert_units_to_annual(fba)
keep_unmapped_rows = False
# if need to maintain FBA columns, create copies of columns
if kwargs != {}:
if ('keep_fba_columns' in kwargs) & \
(kwargs['keep_fba_columns'] is True):
fba['Flowable'] = fba['FlowName']
fba['Context'] = fba['Compartment']
# if keep unmapped rows identified in kwargs, then use
if 'keep_unmapped_rows' in kwargs:
keep_unmapped_rows = kwargs['keep_unmapped_rows']
# else, rename
else:
fba = fba.rename(columns={
'FlowName': 'Flowable',
'Compartment': 'Context'
})
mapped_df = apply_flow_mapping(fba,
from_fba_source,
flow_type=flow_type,
keep_unmapped_rows=keep_unmapped_rows,
ignore_source_name=ignore_source_name)
if mapped_df is None or len(mapped_df) == 0:
# return the original df but with columns renamed so
# can continue working on the FBS
log.warning("Error in flow mapping")
mapped_df = fba.copy()
mapped_df['FlowUUID'] = None
return mapped_df
def getMetadata(source, year=None, category='FlowByActivity'):
"""
Use the esupy package functions to return the metadata for
a FBA or FBS used to generate a FBS
:param source: string, FBA or FBA source name
:param year: string, year of FBA data, for FBS use None
:param category: string, 'FlowBySector' or 'FlowByActivity'
:return: meta object, previously generated FBA or FBS meta
"""
from flowsa.flowbyactivity import set_fba_name
name = set_fba_name(source, year)
meta = read_source_metadata(paths, set_fb_meta(name, category))
if meta is None:
log.warning('No metadata found for %s', source)
meta = {'source_meta': f'No metadata found for {name}'}
return meta
def load_yaml_dict(filename, flowbytype=None):
"""
Load the information in a yaml file, from source_catalog, or FBA,
or FBS files
:return: dictionary containing all information in yaml
"""
if filename == 'source_catalog':
folder = datapath
else:
if flowbytype == 'FBA':
folder = sourceconfigpath
elif flowbytype == 'FBS':
folder = flowbysectormethodpath
else:
raise KeyError('Must specify either \'FBA\' or \'FBS\'')
yaml_path = folder + filename + '.yaml'
try:
with open(yaml_path, 'r') as f:
config = yaml.safe_load(f)
except IOError:
log.error('%s method file not found', flowbytype)
# Allow for .yaml files to recursively inherit other .yaml files. Keys in
# children will overwrite the same key from a parent.
inherits = config.get('inherits_from')
while inherits:
yaml_path = folder + inherits + '.yaml'
with open(yaml_path, 'r') as f:
parent = yaml.safe_load(f)
# Check for common keys and log a warning if any are found
common_keys = [k for k in config if k in parent]
if common_keys:
log.warning(f'Keys {common_keys} from parent file {yaml_path} '
f'were overwritten by child file.')
# Update inheritance information before updating the parent dict
inherits = parent.get('inherits_from')
parent.update(config)
config = parent
return config
def assign_fips_location_system(df, year_of_data):
"""
Add location system based on year of data. County level FIPS
change over the years.
:param df: df with FIPS location system
:param year_of_data: int, year of data pulled
:return: df, with 'LocationSystem' column values
"""
# ensure year integer
year_of_data = int(year_of_data)
if year_of_data >= 2015:
df['LocationSystem'] = 'FIPS_2015'
elif 2013 <= year_of_data < 2015:
df['LocationSystem'] = 'FIPS_2013'
elif 2010 <= year_of_data < 2013:
df['LocationSystem'] = 'FIPS_2010'
elif year_of_data < 2010:
log.warning(
"Missing FIPS codes from crosswalk for %s. "
"Assigning to FIPS_2010", str(year_of_data))
df['LocationSystem'] = 'FIPS_2010'
return df
def main(**kwargs):
"""
Creates a flowbysector dataset
:param kwargs: dictionary of arguments, only argument is
"method_name", the name of method corresponding to flowbysector
method yaml name
:return: parquet, FBS save to local folder
"""
if len(kwargs) == 0:
kwargs = parse_args()
method_name = kwargs['method']
download_FBA_if_missing = kwargs.get('download_FBAs_if_missing')
# assign arguments
vLog.info("Initiating flowbysector creation for %s", method_name)
# call on method
method = load_yaml_dict(method_name, flowbytype='FBS')
# create dictionary of data and allocation datasets
fb = method['source_names']
# Create empty list for storing fbs files
fbs_list = []
for k, v in fb.items():
# pull fba data for allocation
flows = load_source_dataframe(k, v, download_FBA_if_missing)
if v['data_format'] == 'FBA':
# ensure correct datatypes and that all fields exist
flows = clean_df(flows,
flow_by_activity_fields,
fba_fill_na_dict,
drop_description=False)
# clean up fba before mapping, if specified in yaml
if "clean_fba_before_mapping_df_fxn" in v:
vLog.info("Cleaning up %s FlowByActivity", k)
flows = dynamically_import_fxn(
k, v["clean_fba_before_mapping_df_fxn"])(flows)
# map flows to federal flow list or material flow list
flows_mapped, mapping_files = \
map_fbs_flows(flows, k, v, keep_fba_columns=True)
# clean up fba, if specified in yaml
if "clean_fba_df_fxn" in v:
vLog.info("Cleaning up %s FlowByActivity", k)
flows_mapped = dynamically_import_fxn(
k, v["clean_fba_df_fxn"])(flows_mapped)
# if activity_sets are specified in a file, call them here
if 'activity_set_file' in v:
aset_names = pd.read_csv(flowbysectoractivitysetspath +
v['activity_set_file'],
dtype=str)
else:
aset_names = None
# master list of activity names read in from data source
ml_act = []
# create dictionary of allocation datasets for different activities
activities = v['activity_sets']
# subset activity data and allocate to sector
for aset, attr in activities.items():
# subset by named activities
if 'activity_set_file' in v:
names = \
aset_names[aset_names['activity_set'] == aset]['name']
else:
names = attr['names']
# to avoid double counting data from the same source, in
# the event there are values in both the APB and ACB
# columns, if an activity has already been read in and
# allocated, remove that activity from the mapped flows
# regardless of what activity set the data was read in
flows_mapped = flows_mapped[~(
(flows_mapped[fba_activity_fields[0]].isin(ml_act)) |
(flows_mapped[fba_activity_fields[1]].isin(ml_act))
)].reset_index(drop=True)
ml_act.extend(names)
vLog.info("Preparing to handle %s in %s", aset, k)
# subset fba data by activity
flows_subset = flows_mapped[
(flows_mapped[fba_activity_fields[0]].isin(names)) |
(flows_mapped[fba_activity_fields[1]].isin(names)
)].reset_index(drop=True)
# subset by flowname if exists
if 'source_flows' in attr:
flows_subset = flows_subset[flows_subset['FlowName'].isin(
attr['source_flows'])]
if len(flows_subset) == 0:
log.warning(f"no data found for flows in {aset}")
continue
if len(flows_subset[flows_subset['FlowAmount'] != 0]) == 0:
log.warning(f"all flow data for {aset} is 0")
continue
# if activities are sector-like, check sectors are valid
if check_activities_sector_like(k):
flows_subset2 = replace_naics_w_naics_from_another_year(
flows_subset, method['target_sector_source'])
# check impact on df FlowAmounts
vLog.info(
'Calculate FlowAmount difference caused by '
'replacing NAICS Codes with %s, saving '
'difference in Validation log',
method['target_sector_source'],
)
calculate_flowamount_diff_between_dfs(
flows_subset, flows_subset2)
else:
flows_subset2 = flows_subset.copy()
# extract relevant geoscale data or aggregate existing data
flows_subset_geo = subset_df_by_geoscale(
flows_subset2, v['geoscale_to_use'],
attr['allocation_from_scale'])
# if loading data subnational geoscale, check for data loss
if attr['allocation_from_scale'] != 'national':
compare_geographic_totals(flows_subset_geo, flows_mapped,
k, attr, aset, names)
# Add sectors to df activity, depending on level
# of specified sector aggregation
log.info("Adding sectors to %s", k)
flows_subset_wsec = add_sectors_to_flowbyactivity(
flows_subset_geo,
sectorsourcename=method['target_sector_source'],
allocationmethod=attr['allocation_method'])
# clean up fba with sectors, if specified in yaml
if "clean_fba_w_sec_df_fxn" in v:
vLog.info("Cleaning up %s FlowByActivity with sectors", k)
flows_subset_wsec = dynamically_import_fxn(
k, v["clean_fba_w_sec_df_fxn"])(flows_subset_wsec,
attr=attr,
method=method)
# rename SourceName to MetaSources and drop columns
flows_mapped_wsec = flows_subset_wsec.\
rename(columns={'SourceName': 'MetaSources'}).\
drop(columns=['FlowName', 'Compartment'])
# if allocation method is "direct", then no need
# to create alloc ratios, else need to use allocation
# dataframe to create sector allocation ratios
if attr['allocation_method'] == 'direct':
fbs = direct_allocation_method(flows_mapped_wsec, k, names,
method)
# if allocation method for an activity set requires a specific
# function due to the complicated nature
# of the allocation, call on function here
elif attr['allocation_method'] == 'allocation_function':
fbs = function_allocation_method(flows_mapped_wsec, k,
names, attr, fbs_list)
else:
fbs = dataset_allocation_method(flows_mapped_wsec, attr,
names, method, k, v, aset,
aset_names,
download_FBA_if_missing)
# drop rows where flowamount = 0
# (although this includes dropping suppressed data)
fbs = fbs[fbs['FlowAmount'] != 0].reset_index(drop=True)
# define grouping columns dependent on sectors
# being activity-like or not
if check_activities_sector_like(k) is False:
groupingcols = fbs_grouping_fields_w_activities
groupingdict = flow_by_sector_fields_w_activity
else:
groupingcols = fbs_default_grouping_fields
groupingdict = flow_by_sector_fields
# clean df
fbs = clean_df(fbs, groupingdict, fbs_fill_na_dict)
# aggregate df geographically, if necessary
log.info("Aggregating flowbysector to %s level",
method['target_geoscale'])
# determine from scale
if fips_number_key[v['geoscale_to_use']] <\
fips_number_key[attr['allocation_from_scale']]:
from_scale = v['geoscale_to_use']
else:
from_scale = attr['allocation_from_scale']
fbs_geo_agg = agg_by_geoscale(fbs, from_scale,
method['target_geoscale'],
groupingcols)
# aggregate data to every sector level
log.info("Aggregating flowbysector to all sector levels")
fbs_sec_agg = sector_aggregation(fbs_geo_agg, groupingcols)
# add missing naics5/6 when only one naics5/6
# associated with a naics4
fbs_agg = sector_disaggregation(fbs_sec_agg)
# check if any sector information is lost before reaching
# the target sector length, if so,
# allocate values equally to disaggregated sectors
vLog.info(
'Searching for and allocating FlowAmounts for any parent '
'NAICS that were dropped in the subset to '
'%s child NAICS', method['target_sector_level'])
fbs_agg_2 = equally_allocate_parent_to_child_naics(
fbs_agg, method['target_sector_level'])
# compare flowbysector with flowbyactivity
compare_activity_to_sector_flowamounts(flows_mapped_wsec,
fbs_agg_2, aset, k,
method)
# return sector level specified in method yaml
# load the crosswalk linking sector lengths
sector_list = get_sector_list(method['target_sector_level'])
# subset df, necessary because not all of the sectors are
# NAICS and can get duplicate rows
fbs_1 = fbs_agg_2.loc[
(fbs_agg_2[fbs_activity_fields[0]].isin(sector_list)) &
(fbs_agg_2[fbs_activity_fields[1]].isin(sector_list))].\
reset_index(drop=True)
fbs_2 = fbs_agg_2.loc[
(fbs_agg_2[fbs_activity_fields[0]].isin(sector_list)) &
(fbs_agg_2[fbs_activity_fields[1]].isnull())].\
reset_index(drop=True)
fbs_3 = fbs_agg_2.loc[
(fbs_agg_2[fbs_activity_fields[0]].isnull()) &
(fbs_agg_2[fbs_activity_fields[1]].isin(sector_list))].\
reset_index(drop=True)
fbs_sector_subset = pd.concat([fbs_1, fbs_2, fbs_3])
# drop activity columns
fbs_sector_subset = fbs_sector_subset.drop(
['ActivityProducedBy', 'ActivityConsumedBy'],
axis=1,
errors='ignore')
# save comparison of FBA total to FBS total for an activity set
compare_fba_geo_subset_and_fbs_output_totals(
flows_subset_geo, fbs_sector_subset, aset, k, v, attr,
method)
log.info("Completed flowbysector for %s", aset)
fbs_list.append(fbs_sector_subset)
else:
if 'clean_fbs_df_fxn' in v:
flows = dynamically_import_fxn(v["clean_fbs_df_fxn_source"],
v["clean_fbs_df_fxn"])(flows)
flows = update_geoscale(flows, method['target_geoscale'])
# if the loaded flow dt is already in FBS format,
# append directly to list of FBS
log.info("Append %s to FBS list", k)
# ensure correct field datatypes and add any missing fields
flows = clean_df(flows, flow_by_sector_fields, fbs_fill_na_dict)
fbs_list.append(flows)
# create single df of all activities
log.info("Concat data for all activities")
fbss = pd.concat(fbs_list, ignore_index=True, sort=False)
log.info("Clean final dataframe")
# add missing fields, ensure correct data type,
# add missing columns, reorder columns
fbss = clean_df(fbss, flow_by_sector_fields, fbs_fill_na_dict)
# prior to aggregating, replace MetaSources string with all sources
# that share context/flowable/sector values
fbss = harmonize_FBS_columns(fbss)
# aggregate df as activities might have data for
# the same specified sector length
fbss = aggregator(fbss, fbs_default_grouping_fields)
# sort df
log.info("Sort and store dataframe")
# ensure correct data types/order of columns
fbss = clean_df(fbss, flow_by_sector_fields, fbs_fill_na_dict)
fbss = fbss.sort_values(
['SectorProducedBy', 'SectorConsumedBy', 'Flowable',
'Context']).reset_index(drop=True)
# check for negative flow amounts
check_for_negative_flowamounts(fbss)
# tmp reset data quality scores
fbss = reset_fbs_dq_scores(fbss)
# save parquet file
meta = set_fb_meta(method_name, "FlowBySector")
write_df_to_file(fbss, paths, meta)
write_metadata(method_name, method, meta, "FlowBySector")
# rename the log file saved to local directory
rename_log_file(method_name, meta)
log.info(
'See the Validation log for detailed assessment of '
'model results in %s', logoutputpath)
def check_for_missing_sector_data(df, target_sector_level):
"""
Modeled after validation.py check_if_losing_sector_data
Allocates flow amount equally across child NAICS when parent NAICS
is not target_level
:param df: df
:param target_sector_level: str, final sector level of FBS (ex. NAICS_6)
:return: df with missing sector level data
"""
from flowsa.dataclean import replace_NoneType_with_empty_cells
from flowsa.dataclean import replace_strings_with_NoneType
# temporarily replace null values with empty cells
df = replace_NoneType_with_empty_cells(df)
activity_field = "SectorProducedBy"
rows_lost = pd.DataFrame()
cw_load = load_crosswalk('sector_length')
for i in range(3, sector_level_key[target_sector_level]):
# create df of i length
df_subset = df.loc[df[activity_field].apply(lambda x: len(x) == i)]
# import cw and subset to current sector length and
# target sector length
nlength = list(sector_level_key.keys())[list(
sector_level_key.values()).index(i)]
cw = cw_load[[nlength, target_sector_level]].drop_duplicates()
# add column with counts
cw['sector_count'] = cw.groupby(nlength)[nlength].transform('count')
# merge df & replace sector produced columns
df_x = pd.merge(df_subset,
cw,
how='left',
left_on=[activity_field],
right_on=[nlength])
df_x[activity_field] = df_x[target_sector_level]
df_x = df_x.drop(columns=[nlength, target_sector_level])
# calculate new flow amounts, based on sector count,
# allocating equally to the new sector length codes
df_x['FlowAmount'] = df_x['FlowAmount'] / df_x['sector_count']
df_x = df_x.drop(columns=['sector_count'])
# replace null values with empty cells
df_x = replace_NoneType_with_empty_cells(df_x)
# append to df
sector_list = df_subset[activity_field].drop_duplicates()
if len(df_x) != 0:
log.warning(
'Data found at %s digit NAICS to be allocated: '
'{}'.format(' '.join(map(str, sector_list))), str(i))
rows_lost = rows_lost.append(df_x, ignore_index=True, sort=True)
if len(rows_lost) == 0:
log.info('No data loss from NAICS in dataframe')
else:
log.info('Allocating FlowAmounts equally to each %s',
target_sector_level)
# add rows of missing data to the fbs sector subset
df_allocated = pd.concat([df, rows_lost], ignore_index=True, sort=True)
df_allocated = df_allocated.loc[df_allocated[activity_field].apply(
lambda x: len(x) == sector_level_key[target_sector_level])]
df_allocated.reset_index(inplace=True)
# replace empty cells with NoneType (if dtype is object)
df_allocated = replace_strings_with_NoneType(df_allocated)
return df_allocated