def validate_mapping(in_df, out_df, metric):
''' Tests the mapping output to verify results. Stops
'''
uid_cols = md.get_uid_cols(3)
test_results = {'missing entries': [], 'new entries': []}
in_df = md.stdz_col_formats(in_df)
out_df = md.stdz_col_formats(out_df)
test_df = pd.merge(in_df[uid_cols], out_df[uid_cols],
on=uid_cols, how='outer', indicator=True)
if test_df['_merge'].isin(["left_only"]).any():
test_results['missing entries'] = \
test_df.loc[test_df._merge.eq("left_only"),uid_cols].to_dict()
if test_df['_merge'].eq("right_only").any():
test_results['new entries'] = \
test_df.loc[test_df._merge.eq("right_only"), uid_cols].to_dict()
if len(out_df) != len(in_df):
test_results['missing or extra uids'] = "fail"
pt.verify_metric_total(in_df, out_df, metric, "mapping test")
tests.checkTestResults(test_results, 'validate mapping', displayOnly=False)
def get_age_frmat_map(frmat_type):
''' Loads map with indicators for which age groups need splitting
'''
if frmat_type == "im_frmat":
resource = pd.read_csv(
utils.get_path('im_frmat_map', process="mi_dataset"))
elif frmat_type == "frmat":
resource = pd.read_csv(
utils.get_path('frmat_map', process="mi_dataset"))
resource = md.stdz_col_formats(
resource, additional_float_stubs=['age_specific', 'age_split'])
return (resource)
def manage_no_split(df, metric_name, uid_cols, this_dataset):
''' finalize and save dataset where no split is needed
'''
uids_noAge = [c for c in uid_cols if 'age' not in c]
# collapse remaining under5 and 80+ ages
df = md.stdz_col_formats(df)
df[metric_name].fillna(value=0, inplace=True)
final_df = df.copy(deep=True)
# final_df = md.collapse_youngAndOld_ages(df, uids_noAge, metric_name)
#
if metric_name == "pop":
md.complete_prep_step(final_df, this_dataset, is_pop=True)
else:
md.complete_prep_step(final_df, this_dataset, is_pop=False)
return(None)
def disaggregate_acause(df, ds_instance):
''' Description: Returns a dataframe in which metric values have been
distributed across all associated acauses
How it Works: Utilizes the create_metric_weights function to reshape
the df so that acause is in long form. Adds proportions to
each acause by observation, then applies those proportions to
split the input metric value across the attributed acauses.
Finally, collapses to re-combine data to single datapoints by
gbd_cause and acause.
NOTE: this process drops the 'cause' and 'cause_name' columns.
'''
# Ensure that 'age' is not listed as a uid
metric = ds_instance.metric
uids_noAcause = [c for c in md.get_uid_cols(5) if 'acause' not in c]
acause_cols = [a for a in df.columns if 'acause' in a]
all_uids = md.get_uid_cols(5)
needs_split = (df['acause2'].notnull() & ~df['acause2'].isin([""]))
to_split = df.loc[needs_split, :]
no_split = df.loc[~needs_split, :]
# If no split needed, simply return the datafraqme with a renamed acause1
if len(to_split) == 0:
df.rename(columns={'acause1': 'acause'}, inplace=True)
acause_cols.remove('acause1')
df.drop(labels=acause_cols, axis=1, inplace=True)
return (df)
print("disaggregating acause...")
# create weights used for splitting
weight_df = pp.create_metric_weights(df, all_uids, ds_instance)
# calculate proportions based on the weights
proportions_df = pp.add_proportions(weight_df, uids_noAcause)
# adjust by proportions
is_split = to_split.merge(proportions_df)
is_split['split_value'] = is_split[metric]
is_split.loc[:, metric] = is_split['proportion'] * is_split['split_value']
is_split = md.stdz_col_formats(is_split)
#
no_split.rename(columns={'acause1': 'acause'}, inplace=True)
acause_cols.remove('acause1')
no_split.drop(labels=acause_cols, axis=1, inplace=True)
#
output = no_split.append(is_split)
pt.verify_metric_total(df, output, metric, "disaggregate acause")
return (output.loc[:, no_split.columns.tolist()])
def run_split_pop(this_dataset):
''' split population data in combined age groups
'''
print(" splitting population...")
uid_cols = md.get_uid_cols(prep_type_id=4, is_pop=True)
df = this_dataset.load_pop(prep_type_id=1)
# update column names (some old 00_pop files still have registry_id column
# instead of registry_index)
if "registry_id" in df.columns:
df.rename(columns={'registry_id': 'registry_index'}, inplace=True)
# Exit if no population present
if len(df) == 0:
return(None)
# Temporarily reshape and updated dataframe until input is no longer in STATA
uids_noAge = [c for c in uid_cols if 'age' not in c]
df = dft.wide_to_long(df,
stubnames='pop',
i=uids_noAge,
j='age',
drop_others=True)
df = df.loc[df.age != 1, :] # drop 'all ages'
pop_cols = [p for p in df.columns.values if "pop" in p]
df.loc[:, pop_cols].fillna(value=0, inplace=True)
df = md.stdz_col_formats(df)
if not (pt.has_combinedSex(df) |
pt.has_age_unk(df, "pop") |
pt.has_nonStdAge(df)
):
manage_no_split(df, "pop", uid_cols, this_dataset)
else:
# replace missing data with zeroes
uid_cols += ['location_id', 'country_id', 'year']
df = md.add_location_ids(df)
df = modeled_locations.add_country_id(df)
# data with no country_id have no population estimates.
# global estimate should be used to generate weights
df.loc[df['country_id'] == 0, 'location_id'] = 1
# add mean year to faciliate merge with population weights
df = add_year_id(df)
# Split data
manage_split(df, "pop", uid_cols, this_dataset)
return(None)
def main(dataset_id, data_type_id):
''' Disaggregates uids that are mapped to multiple gbd causes, including
garbage codes, Kaposi Sarcoma, and non-melanoma skin cancer
'''
# prep_step 5 = cause_disaggregation
this_dataset = md.MI_Dataset(dataset_id, 5, data_type_id)
input_data = this_dataset.load_input()
metric = this_dataset.metric
uid_cols = md.get_uid_cols(5)
input_data = input_data.loc[
~input_data['age'].isin([26, 3, 4, 5, 6, 91, 92, 93, 94]), :]
# Format and add observation numbers
formatted_input = prep_for_disagg(input_data.copy(), uid_cols, metric)
# Disaggregate
disaggregated_df = core.disaggregate_acause(formatted_input, this_dataset)
# update uid columns to account for reshaped acause
uid_cols = [u for u in uid_cols if 'acause' not in u] + ['acause']
#
kaposi_df = core.redist_kaposi(disaggregated_df, metric, uid_cols)
if data_type_id == 2:
adjusted_df = core.redist_nmsc_gc(kaposi_df, metric)
else:
adjusted_df = kaposi_df
final_df = core.map_remaining_garbage(adjusted_df, data_type_id)
# run test functions and save output
pt.verify_metric_total(input_data, adjusted_df, metric,
"cause disaggregation module")
# collapse to incorperate newly-split data
output_uids = md.get_uid_cols(6)
final_df = md.stdz_col_formats(final_df)
final_df = dft.collapse(final_df,
by_cols=output_uids,
func='sum',
combine_cols=metric)
# save
md.complete_prep_step(final_df, this_dataset)
print("Acause disaggregated")
def main(dataset_id, data_type_id):
''' Maps data with the following steps:
1) imports input data
2) runs mapping function
3) expands icd codes to fill garbage acause(n) where necessary
4) applies sex and cause restrictions
'''
this_dataset = md.MI_Dataset(dataset_id, 3, data_type_id)
metric_name = this_dataset.metric
uid_cols = md.get_uid_cols(3)
input_data = this_dataset.load_input().reset_index(drop=True)
input_data.loc[input_data['im_frmat'].isnull() & input_data['frmat'].eq(9),
'im_frmat'] = 9
df = md.stdz_col_formats(input_data)
# Ensure that there is no "all age" data.
df = df.loc[df['age'] != 1, :]
# Map data and apply restrictions
mapped_df = map_data(df, this_dataset)
restricted_df = cm.restrict_causes(mapped_df,
cause_col='gbd_cause',
data_type_id=data_type_id,
restrict_age=False)
md.complete_prep_step(restricted_df, this_dataset)
print("mapping complete.\n")
def run(dataset_id, data_type_id, uid):
''' Preps data for recalculation then recalculates as necessary
'''
this_dataset = md.MI_Dataset(dataset_id, 2, data_type_id)
dataset_name = this_dataset.name
metric = this_dataset.metric
input_file = run_sr.get_sr_file(this_dataset, "sr_input")
# Exit if output already exists
output_file = run_sr.get_sr_file(this_dataset, 'split_output', uid)
print(output_file)
if os.path.exists(output_file):
print(" output file found for uid " + str(uid))
return(None)
#
negative_data_ok = is_exception(dataset_id, data_type_id)
error_folder = utils.get_path("mi_input", base='j_temp')
subcause_issue_file = '{}/subcause_issue/{}_{}_uid_{}.txt'.format(
error_folder, dataset_name, data_type_id, uid)
exception_file = '{}/negative_data/{}_{}_uid_{}.csv'.format(
error_folder, dataset_name, data_type_id, uid)
for d in [subcause_issue_file, exception_file, error_folder]:
utils.ensure_dir(d)
#
print(" removing subtotals from uid {}...".format(uid))
# add data for the given uid
df = pd.read_hdf(input_file, 'split_{}'.format(uid))
# Create a list of possible codes so that decimal subcauses are only added
# if available
input_cause_list = sorted(df['orig_cause'].unique().tolist())
# create a dictionary for codes in the selected uid and attach the uid's
# data
uid_subset = {}
input_data = {}
# process decimals first and ranges last to ensure that nested causes are
# removed
for c in sorted(df['orig_cause'].unique().tolist()):
uid_subset[c] = {}
input_data[c] = {}
uid_subset[c]['codes'] = []
uid_subset[c]['subcodes'] = []
if "-" not in c and "," not in c:
uid_subset[c]['codes'].append(c)
# add subcodes to 'subcode' key
df.loc[df['orig_cause'].eq(c), 'cause'].dropna().unique().tolist()
for subcode in sorted(df['cause'].where(df['orig_cause'] == c
).dropna().unique().tolist()):
if subcode != c:
uid_subset[c]['subcodes'].append(subcode)
# if none of the subcodes appear in the list, set the cause as a
# subcode of itself (prevents the addition of unused decimal
# causes)
if not len(uid_subset[c]['subcodes']):
uid_subset[c]['subcodes'] = uid_subset[c]['codes']
elif (not any('{}.'.format(sub[:3]) in check
for check in input_cause_list
for sub in uid_subset[c]['subcodes'])):
uid_subset[c]['subcodes'] = uid_subset[c]['codes']
else:
for code in sorted(df['cause'].where(
df['orig_cause'].eq(c)).dropna().unique().tolist()):
uid_subset[c]['codes'].append(code)
uid_subset[c]['subcodes'].append(code)
# create other lists associated with the cause and add the metric data
uid_subset[c]['subcauses_remaining'] = []
uid_subset[c]['codes_removed'] = []
uid_subset[c]['causes_removed'] = []
uid_subset[c]['data'] = df.loc[df['cause'].eq(c),
['age', metric]].set_index('age')
input_data[c]['data'] = uid_subset[c]['data']
input_data[c]['codes'] = uid_subset[c]['codes']
# Determine subcauses and highest number of causes remaining (how many
# subcauses are contained within each cause)
uid_set = set_subcauses(uid_subset, subcause_issue_file)
highest_level = determine_highest_level(uid_set)
# remove lowest level codes from parent causes
if highest_level == 0:
print(' no subcauses present.')
else:
subcauses_removed = True
while subcauses_removed:
uid_set, subcauses_removed = remove_subcauses(
uid_set, uid, exception_file)
# remove duplicates
uid_set = remove_duplicates(uid_set)
# re-set subcauses and num_subcause_remaining
uid_set, highest_level = set_subcauses(
uid_set, subcause_issue_file,)
print(" subcauses removed.")
# Prepare Output
print("saving output...")
output = pd.DataFrame(
columns=['cause', 'codes_remaining', 'codes_removed', 'age', metric])
for c in uid_set:
# format cause information
cause_data = pd.DataFrame(
columns=['cause', 'codes_remaining', 'codes_removed'])
cause_data.loc[0, ['cause']] = c
# if nothing was removed, or there was only a single cause, or all of
# the input codes are still present, set the codes remaining as the
# cause
if (not len(uid_set[c]['codes_removed']) or
("-" not in c and "," not in c) or
set(input_data[c]['codes']) <= set(uid_set[c]['codes'])):
cause_data.loc[0, ['codes_remaining']] = c
else:
cause_data.loc[0, ['codes_remaining']] = ','.join(
convert_to_range(uid_set[c]['codes']))
cause_data.loc[0, ['codes_removed']] = ','.join(
convert_to_range(uid_set[c]['codes_removed']))
# format output data
output_data = uid_set[c]['data']
output_data['age'] = output_data.index
output_data['cause'] = c
orig_data = input_data[c]['data']
orig_data['age'] = orig_data.index
orig_data = orig_data.rename(
columns={metric: 'orig_metric_value'})
orig_data['cause'] = c
# combine and add to output
final = pd.merge(output_data, cause_data, on='cause')
final = pd.merge(final, orig_data, on=['cause', 'age'])
output = output.append(final)
# Create output dataset
output['uniqid'] = uid
# Update encoding (bug fix to work around pandas to_stata issue)
output = md.stdz_col_formats(output, additional_float_stubs='uniqid')
# Export results
output.to_csv(output_file, index=False)
print('\n Done!')
time.sleep(1)
return(None)