def main(dataset_id, data_type_id):
''' Maps data with the following steps:
1) imports input data
2) runs mapping function
3) expands icd codes to fill garbage acause(n) where necessary
4) applies sex and cause restrictions
'''
this_dataset = md.MI_Dataset(dataset_id, 3, data_type_id)
metric_name = this_dataset.metric
uid_cols = md.get_uid_cols(3)
input_data = this_dataset.load_input().reset_index(drop=True)
input_data.loc[input_data['im_frmat_id'].isnull()
& input_data['frmat_id'].eq(9), 'im_frmat_id'] = 9
df = md.stdz_col_formats(input_data)
# Ensure that there is no "all age" data. Remove this line after updating
# the preceeding steps to stop using the old cancer age formats
df = df.loc[df['age'] != 1, :]
# Map data and apply restrictions
mapped_df = map_data(df, this_dataset)
restricted_df = cm.restrict_causes(mapped_df,
cause_col='gbd_cause',
data_type_id=data_type_id,
restrict_age=False)
md.complete_prep_step(restricted_df, this_dataset)
print("mapping complete.\n")
def disaggregate_acause(df, ds_instance):
''' Description: Returns a dataframe in which metric values have been
distributed across all associated acauses
How it Works: Utilizes the create_metric_weights function to reshape
the df so that acause is in long form. Adds proportions to
each acause by observation, then applies those proportions to
split the input metric value across the attributed acauses.
Finally, collapses to re-combine data to single datapoints by
gbd_cause and acause.
NOTE: this process drops the 'cause' and 'cause_name' columns.
'''
# Ensure that 'age' is not listed as a uid
metric = ds_instance.metric
uids_noAcause = [c for c in md.get_uid_cols(5) if 'acause' not in c]
acause_cols = [a for a in df.columns if 'acause' in a]
all_uids = md.get_uid_cols(5)
needs_split = (df['acause2'].notnull() & ~df['acause2'].isin([""]))
to_split = df.loc[needs_split, :]
no_split = df.loc[~needs_split, :]
# If no split needed, simply return the datafraqme with a renamed acause1
if len(to_split) == 0:
df.rename(columns={'acause1': 'acause'}, inplace=True)
acause_cols.remove('acause1')
df.drop(labels=acause_cols, axis=1, inplace=True)
return (df)
print("disaggregating acause...")
# create weights used for splitting
weight_df = pp.create_metric_weights(df, all_uids, ds_instance)
# calculate proportions based on the weights
proportions_df = pp.add_proportions(weight_df, uids_noAcause)
# adjust by proportions
is_split = to_split.merge(proportions_df)
is_split['split_value'] = is_split[metric]
is_split.loc[:, metric] = is_split['proportion'] * is_split['split_value']
is_split = md.stdz_col_formats(is_split)
#
no_split.rename(columns={'acause1': 'acause'}, inplace=True)
acause_cols.remove('acause1')
no_split.drop(labels=acause_cols, axis=1, inplace=True)
#
output = no_split.append(is_split)
pt.verify_metric_total(df, output, metric, "disaggregate acause")
return (output.loc[:, no_split.columns.tolist()])
def main(dataset_id, data_type_id, split_num):
'''
'''
# Load input
metric_dict = {'2': 'cases', '3': 'deaths'}
this_dataset = md.MI_Dataset(dataset_id, 6, data_type_id)
metric_name = this_dataset.metric
rdp_input = manager.get_rdp_file(this_dataset, 'rdp_input')
input_data = pd.read_hdf(rdp_input, 'split_{}'.format(split_num))
# rename sex_id until rdp packages names are updated
input_data.rename(columns={'sex_id': 'sex'}, inplace=True)
# Redistribute data where possible
if not manager.needs_rdp(input_data, this_dataset):
print(" no redistribution needed for ds {} type {} split {}".format(
dataset_id, data_type_id, split_num))
save_worker_output(input_data, this_dataset, split_num)
return (input_data)
else:
print(" redistributing ds {} type {} split {}".format(
dataset_id, data_type_id, split_num))
# Add maps to enable RDP
input_data.rename(columns={'uid': 'split_group'}, inplace=True)
mapped = add_location_hierarchy_info(input_data)
# RDP cannot run without location metadata, and should not run for hiv
# Set aside those data
skip_rdp_mask = cannot_redistribute(mapped)
set_aside = mapped.loc[skip_rdp_mask, input_data.columns.tolist()]
to_redistribute = mapped.loc[~skip_rdp_mask, :]
# Redistribute remaining data
if to_redistribute.any().any():
rdp_results = run_rdp_core(to_redistribute, this_dataset,
split_num)
# Recombine
if set_aside.any().any():
rdp_results = rdp_results.append(set_aside, ignore_index=True)
to_finalize = rdp_results
else:
print(" No data to redistribute. Finalizing.")
to_finalize = input_data.rename(columns={'cause': 'acause'})
output_cols = md.get_uid_cols(7)
# rename sex_id until rdp packages get updated
output_cols = ['sex' if x == 'sex_id' else x for x in output_cols]
to_finalize = cm.correct_causes(to_finalize)
finalized_df = dft.collapse(to_finalize,
by_cols=output_cols,
stub=metric_name)
# Check totals (note: because of data precision, data before and after
# may not be precisely equivalent)
diff = finalized_df[metric_name].sum() - input_data[metric_name].sum()
assert abs(diff/input_data[metric_name].sum()) < 5, \
"Difference from input after rdp is too large"
save_worker_output(finalized_df, this_dataset, split_num)
return (finalized_df)
def update_redistributed_acause(df, ds_instance, split_num):
''' Returns dataframe (df) after merging with maps to update cause information
-- Maps:
decimal cause map : used to revert cause names to decimal form
cause map : used to validate output causes
'''
metric_name = ds_instance.metric
output_uids = md.get_uid_cols(7)
output_uids = ['sex' if x == 'sex_id' else x for x in output_uids]
#
def manage_rdp_remnants(df, temp_folder, split_num, metric):
''' Verifies if any garbage remains after
'''
# Get any codes that didn't merge and save them
rdp_error = ((df['acause'].isnull() | (df['_merge'] == 'left_only'))
& df[ds_instance.metric].isin([np.nan, 0]))
rdp_error_list = sorted(df.loc[rdp_error, 'cause'].unique().tolist())
if len(rdp_error_list):
print("The following causes are not in the cause map:")
print(rdp_error_list)
return (None)
# Convert acause back to cancer cause
code_format_updates = { # not necessary once rdp uses the map in the cancer db
'C0': 'C00',
'C1': 'C01',
'C2': 'C02',
'C3': 'C03',
'C4': 'C04',
'C4A': 'C04',
'C5': 'C05',
'C6': 'C06',
'C7': 'C07',
'C8': 'C08',
'C9': 'C09',
'neo_other': 'neo_other_cancer'
}
for key, value in code_format_updates.items():
df.loc[df['acause'] == key, 'acause'] = value
# Merge with cause map
df.rename(columns={'acause': 'cause'}, inplace=True) # No map
cause_map = cm.load_rdp_cause_map(ds_instance.data_type_id)
df = df.merge(cause_map,
how='left',
on=['coding_system', 'cause'],
indicator=True)
# Check that all data were mapped to cause
manage_rdp_remnants(df, ds_instance.temp_folder, split_num, metric_name)
# Reformat to merge data with original source
df = df.loc[:, output_uids + [metric_name]]
final_df = dft.collapse(df, by_cols=output_uids, stub=metric_name)
return (final_df)
def main(dataset_id, data_type_id):
'''
'''
# prep_step 5 = cause_disaggregation
this_dataset = md.MI_Dataset(dataset_id, 5, data_type_id)
input_data = this_dataset.load_input()
metric = this_dataset.metric
uid_cols = md.get_uid_cols(5)
input_data = input_data.loc[~input_data['age'].isin(
[26, 3, 4, 5, 6, 91, 92, 93, 94]), :]
# Format and add observation numbers
formatted_input = prep_for_disagg(input_data.copy(), uid_cols, metric)
# Disaggregate
disaggregated_df = core.disaggregate_acause(formatted_input, this_dataset)
# update uid columns to account for reshaped acause
uid_cols = [u for u in uid_cols if 'acause' not in u] + ['acause']
#
kaposi_df = core.redist_kaposi(disaggregated_df, metric, uid_cols)
if data_type_id == 2:
adjusted_df = core.redist_nmsc_gc(kaposi_df, metric)
else:
adjusted_df = kaposi_df
final_df = core.map_remaining_garbage(adjusted_df, data_type_id)
# run test functions and save output
pt.verify_metric_total(input_data, adjusted_df,
metric, "cause disaggregation module")
# collapse to incorperate newly-split data
output_uids = md.get_uid_cols(6)
final_df = md.stdz_col_formats(final_df)
final_df = dft.collapse(final_df,
by_cols=output_uids,
func='sum',
combine_cols=metric
)
# save
md.complete_prep_step(final_df, this_dataset)
print("Acause disaggregated")
def run_split_metric(this_dataset):
'''
'''
print(" splitting metric data...")
uid_cols = md.get_uid_cols(4)
metric_name = this_dataset.metric
input_data = this_dataset.load_input()
# save if no splitting is needed
if not (pt.has_combinedSex(input_data) |
pt.has_age_unk(input_data, metric_name) |
pt.has_nonStdAge(input_data)):
manage_no_split(input_data, metric_name, uid_cols, this_dataset)
else:
manage_split(input_data, metric_name, uid_cols, this_dataset)
return(None)
def reformat_input(df, ds_instance):
''' Collapse and reshape input data from standardize_format output
'''
metric_name = ds_instance.metric
uid_cols = md.get_uid_cols(2)
wide_uid_cols = [u for u in uid_cols if 'age' not in u]
uids_noCause = [u for u in uid_cols if 'cause' not in u]
df.loc[df['im_frmat_id'].isnull() & df['frmat_id'].isin([9]), 'im_frmat_id'] = 9
df = md.stdz_col_formats(df)
df = dft.collapse(df, by_cols=wide_uid_cols, func='sum', stub=metric_name)
df = dft.wide_to_long(df, stubnames=metric_name,
i=wide_uid_cols, j='age')
df = df.groupby(uid_cols, as_index=False)[metric_name].sum()
df[metric_name].fillna(value=0, inplace=True)
df = md.stdz_col_formats(df)
df = dft.make_group_id_col(df, uids_noCause, id_col='uniqid')
return(df)
def run_split_pop(this_dataset):
'''
'''
print(" splitting population...")
uid_cols = md.get_uid_cols(prep_type_id=4, is_pop=True)
df = this_dataset.load_pop(prep_type_id=1)
if "registry_id" in df.columns: # some old 00_pop files still have registry_id column instead of registry_index
df.rename(columns={'registry_id': 'registry_index'}, inplace=True)
# Exit if no population present
if len(df) == 0:
return(None)
# Temporarily reshape and updated dataframe until input is no longer in STATA
uids_noAge = [c for c in uid_cols if 'age' not in c]
df = dft.wide_to_long(df,
stubnames='pop',
i=uids_noAge,
j='age',
drop_others=True)
df = df.loc[df.age != 1, :] # drop 'all ages'
pop_cols = [p for p in df.columns.values if "pop" in p]
df.loc[:, pop_cols].fillna(value=0, inplace=True)
df = md.stdz_col_formats(df)
if not (pt.has_combinedSex(df) |
pt.has_age_unk(df, "pop") |
pt.has_nonStdAge(df)
):
manage_no_split(df, "pop", uid_cols, this_dataset)
else:
# replace missing data with zeroes
uid_cols += ['location_id', 'country_id', 'year']
df = md.add_location_ids(df)
df = modeled_locations.add_country_id(df)
# data with no country_id have no population estimates.
# global estimate should be used to generate weights
df.loc[df['country_id'] == 0, 'location_id'] = 1
# add mean year to faciliate merge with population weights
df = gct.add_year_id(df)
# Split data
manage_split(df, "pop", uid_cols, this_dataset)
return(None)
def validate_mapping(in_df, out_df, metric):
''' Tests the mapping output to verify results.
'''
uid_cols = md.get_uid_cols(3)
test_results = {'missing entries': [], 'new entries': []}
in_df = md.stdz_col_formats(in_df)
out_df = md.stdz_col_formats(out_df)
test_df = pd.merge(in_df[uid_cols],
out_df[uid_cols],
on=uid_cols,
how='outer',
indicator=True)
if test_df['_merge'].isin(["left_only"]).any():
test_results['missing entries'] = test_df.loc[
test_df._merge == "left_only", uid_cols].to_dict()
if test_df['_merge'].isin(["right_only"]).any():
test_results['new entries'] = test_df.loc[
test_df._merge == "right_only", uid_cols].to_dict()
if len(out_df) != len(in_df):
test_results['missing or extra uids'] = "fail"
pt.verify_metric_total(in_df, out_df, metric, "mapping test")
tests.checkTestResults(test_results, 'validate mapping', displayOnly=False)
def manage_split(df, metric_name, uid_cols, this_dataset):
''' Converts age and sex categories in the df to those used by the cancer prep process.
1) Adds obs number
2) Splits aggregated ages
3) Combines disaggregated ages
4) Splits unknown age category
5) Splits aggregated/unknown sex category
'''
is_pop = bool(metric_name == "pop")
df[metric_name].fillna(value=0, inplace=True)
split_df = df.copy()
# add observation number by group, without age
uids_noAge = [c for c in uid_cols if 'age' not in c]
split_df = core.add_missing_ages(split_df, uids_noAge, metric_name)
obs_numbers = split_df[uids_noAge].drop_duplicates()
obs_numbers['obs'] = obs_numbers.reset_index().index
split_df = split_df.merge(obs_numbers)
uid_cols.append('obs')
# generate cause_weight
if is_pop:
cause_wgts = pp.gen_pop_wgts("age_wgt", df['location_id'].unique())
else:
# create weights used for splitting
cause_wgts = pp.create_metric_weights(split_df, uid_cols, this_dataset)
# collapse to get one weight per observation. dropping redundant entries beforehand
cause_wgts.drop_duplicates(subset=['obs','age','sex_id'], inplace=True)
cause_wgts = dft.collapse(cause_wgts, by_cols=['obs', 'age', 'sex_id'],
func='sum', combine_cols='wgt')
# split
if pt.has_nonStdAge(split_df):
print(" splitting non-standard age...")
split_df = core.split_age(dataset=split_df,
wgt_df=cause_wgts,
metric=metric_name,
uid_cols=uid_cols)
# collapse remaining under5 and 80+ ages
# split_df = md.collapse_youngAndOld_ages(split_df, uid_cols, metric_name)
# redistribute "unknown age" data according to the current distribution of cases/deaths
if pt.has_age_unk(split_df, metric_name):
print(" splitting unknown age...")
# create weights
split_df = core.split_unknown_age(dataset=split_df,
wgt_df=cause_wgts,
metric=metric_name,
uid_cols=uid_cols)
# check for errors. If more than 3 metric totals are greater than .0001% different, alert user
at.compare_pre_post_split(split_df, df, metric_name)
# split sex = 3 and sex = 9 data
if pt.has_combinedSex(split_df):
print(" splitting sex...")
if metric_name == "pop":
sex_split_prop = pp.gen_pop_wgts(
"sex_wgt", df['location_id'].unique())
else:
sex_split_prop = pp.create_sex_weights(cause_wgts,
uid_vars=['obs', 'age'],
metric=metric_name)
split_df = core.split_sex(split_df,
sex_split_prop,
uid_cols,
metric=metric_name)
# collapse remaining underu5 and 80+ ages
# final_df = md.collapse_youngAndOld_ages(split_df, uid_cols, metric_name)
final_df = split_df.copy(deep=True)
final_df = final_df.loc[~final_df['age'].isin([26]), :]
# collapse to incorperate newly-split data
output_uids = md.get_uid_cols(5, is_pop)
final_df = dft.collapse(final_df,
by_cols=output_uids,
func='sum',
combine_cols=metric_name
)
# save and exit
md.complete_prep_step(final_df, this_dataset, is_pop)
return(None)
def submit_sr(calc_df, this_dataset):
''' Splits data based on subtotal-recalculation requirement and submits
jobs as needed to recalculate subtotals. Then returns a re-combined
dataset with subtotals recalculated
'''
def submission_req(df, uid):
''' Returns boolean indicating whether data are to be submitted,
qualified by whether subtotals are present and whether any
component codes exist that could enable recalculation
'''
uid_test = df[df['uniqid'].eq(uid)]
meets_requirement = bool( has_subtotals(uid_test, 'orig_cause')
and components_present(uid_test) )
return(meets_requirement)
def output_file_func(id):
''' Function fed to get_results relative to the
'''
return(get_sr_file(this_dataset, 'split_output', id[0]))
#
output_uids = md.get_uid_cols(3)
metric_name = this_dataset.metric
job_header = "cnSR_{}_{}".format(dataset_id, data_type_id)
sr_input_file = get_sr_file(this_dataset, "sr_input")
worker_script = utils.get_path("subtotal_recalculation_worker",
process="mi_dataset")
# convert components to string to enable save in hdf file
uniqid_map = calc_df[output_uids + ['uniqid', 'orig_cause']
].copy().drop_duplicates()
submitted_data, unsubmitted_data = cup.split_submission_data(calc_df,
group_id_col='uniqid',
submission_requirement=submission_req,
hdf_file=sr_input_file,
regenerate_hdf=False)
if len(submitted_data) == 0:
final_results = unsubmitted_data
else:
uid_list = submitted_data['uniqid'].unique().tolist()
sr_jobs = cup.generate_prep_workers(worker_script,
list_of_uids=uid_list,
ds_instance=this_dataset,
job_header=job_header,
is_resubmission=is_resubmission)
output_files = cup.get_results(sr_jobs,
output_file_func,
parent_process_name="sr",
noisy_checker=True,
add_resubmission_argument=is_resubmission,
wait_time=5)
# Re-combine compiled results with the set-aside data, before collapsing
# and testing
results = pe.read_files(output_files)
results.rename(columns={'cause':'orig_cause','codes_remaining':'cause'},
inplace=True)
results = md.stdz_col_formats(results, additional_float_stubs='uniqid')
results = results.merge(uniqid_map, how='outer', indicator=True)
assert results['_merge'].isin(["both", "right_only"]).all(), \
"Error merging with uids"
del results['_merge']
# entries with blank "cause" could not be corrected. replace with the
# original aggregate (will be handled by cause recalculation and rdp).
results.loc[results['cause'].eq(""), 'cause'] = results['orig_cause']
# drop causes that were zeroed in subtotal recalculation
results['total'] = results.groupby(output_uids)[metric_name].transform(sum)
results = results.loc[results['total'].ne(0) &
results[metric_name].notnull(), :]
final_results = results.append(unsubmitted_data)
# Re-combine with data that were not split
final_results = dft.collapse(final_results, by_cols=output_uids,
combine_cols=this_dataset.metric)
return(final_results)
def map_data(input_df, this_dataset):
''' Returns the input dataframe with attached cause mapping
Process:
1) Declares mapping order, then applies maps in that order
NOTE: there will be no need to iterate this step once
custom codes are mapped at the input stage
2) Verifies map application
'''
print("Applying maps...")
# Define the order in which maps will be merged with the data
map_order = {1: 'dataset_id', 2: 'country_id', 3: 'coding_system'}
# Define the order in which maps will be merged with the data
merge_cols = ['coding_system', 'cause', 'cause_name']
uid_cols = [c for c in md.get_uid_cols(3) if c not in merge_cols]
# Prepare for merge with cause map
unmapped_df = md.add_location_ids(input_df)
check_cols = [
'cause', 'cause_name', 'registry_index', 'sex_id', 'year_start',
'year_end'
]
check_duplicates(unmapped_df, check_cols)
unmapped_df['dataset_id'] = this_dataset.dataset_id
# Generate inputs for iterative mapping
cause_map = cm.load_cause_map(this_dataset.data_type_id)
map_cols = ['coding_system', 'cause', 'cause_name', 'gbd_cause'] +\
[a for a in cause_map.columns if 'acause' in a]
mapped_df = pd.DataFrame()
# Apply maps for CUSTOM coding systems.
# Note that this should run once with only 'coding_system' used to merge
for o in map_order.values():
subset_cols = list(
set(map_cols +
[o])) # Ensures that each column appears only once in the list
subset_entries = (cause_map.coding_system.str.contains("CUSTOM")
& ~pd.isnull(cause_map[o]) & (cause_map[o] != 0))
map_subset = cause_map.loc[subset_entries, subset_cols]
if len(map_subset) > 0:
# Note: index is reset and then set to preserve index values of the unmapped_df dataframe
# in the newly_mapped dataframe
newly_mapped = unmapped_df.reset_index().merge(
map_subset, how='inner',
on=merge_cols + [o]).set_index('index')
mapped_df = mapped_df.append(newly_mapped)
unmapped_df.drop(list(newly_mapped.index), axis=0, inplace=True)
cause_map.drop(list(map_subset.index), axis=0, inplace=True)
# Apply maps by non-CUSTOM coding_system
final_map = cause_map[map_cols]
# capitalizes ICCC3 codes in cancer_map to match the causes
final_map.loc[final_map['coding_system'] == "ICCC3",
'cause'] = final_map.loc[final_map['coding_system'] ==
"ICCC3", 'cause'].str.title()
unmapped_df['test'] = unmapped_df.index
newly_mapped = unmapped_df.reset_index().merge(
final_map, how='inner', on=merge_cols).set_index('index')
check_duplicates(newly_mapped, check_cols)
mapped_df = mapped_df.append(newly_mapped)
unmapped_df.drop(list(newly_mapped.index), axis=0, inplace=True)
# Test output
assert len(unmapped_df) == 0, \
"ERROR: could not map all causes. Missing the following: \ncause: {} \ncause_name: {}".format(
unmapped_df.cause.unique(), unmapped_df.cause_name.unique()
)
mt.validate_mapping(input_df, mapped_df, this_dataset.metric)
# special consideration for Germ cell tumors, trophoblastic tumors, and neoplasms of gonads (ICCC code: X)
# dropping both sexes for now until we have proportions set up
mapped_df.loc[(mapped_df['acause1'] == "gender_specific") &
(mapped_df['sex_id'] == 2),
['gbd_cause', 'acause1']] = "neo_ovarian"
mapped_df.loc[(mapped_df['acause1'] == "gender_specific") &
(mapped_df['sex_id'] == 1),
['gbd_cause', 'acause1']] = "neo_testicular"
# Replace non-icd causes with "CUSTOM"
# NOTE: this section should be moved to a different script or deleted altogether
mapped_df.at[~mapped_df.coding_system.isin(["ICD10", "ICD9_detail"]),
"coding_system"] = "CUSTOM"
mapped_df.drop(labels=['country_id', 'dataset_id', 'location_id'],
axis=1,
inplace=True)
# Return
return (mapped_df)