def build_meds_frame(ta, frame_name, in_path, overwrite = True): print "*** CREATING DRUG HISTORY FRAME at " + frame_name if (frame_name in ta.get_frame_names()): if (not overwrite): return ta.get_frame(frame_name) else: ta.drop_frames([frame_name]) add_udf_files(ta, [clean_drugs, derived_features, history_utilities, uphs_fields, uphs_schema]) data_fields = [ PATID, VISID, ADM_DATE, MED_ORDER_NAMEs, DISCHARGE_MED_ORDER_NAMEs ] data_frame = loader.load_frame(ta, in_path, frame_name, data_fields) print "Filtering out rows with missing or junk patient ids and/or visit ids" data_frame.filter(lambda row: row[PATID] != None and row[VISID] != None and [PATID] != 'None' and row[VISID] != 'None') data_defaults = {MED_ORDER_NAMEs : "", DISCHARGE_MED_ORDER_NAMEs: ""} print "Imputing missing drug fields with empty lists" impute_with_constants(data_frame, data_defaults) data_frame.add_columns(lambda row: row[MED_ORDER_NAMEs] + ", " + row[DISCHARGE_MED_ORDER_NAMEs], (UNCLEAN_COMBINED_MEDLIST, str)) data_frame.drop_columns([MED_ORDER_NAMEs, DISCHARGE_MED_ORDER_NAMEs]) data_frame.add_columns(lambda row: drug_cleaner.to_clean_doc(row[UNCLEAN_COMBINED_MEDLIST], medlist_delimiter), (COMBINED_MEDLIST, str)) data_frame.drop_columns([UNCLEAN_COMBINED_MEDLIST]) return data_frame
def create_crossval_labeled_patids(ta, frame_name, in_path, overwrite = True): fields = [PATID] print "****** Assigning train/test labels for cross validation... loading data" if (frame_name in ta.get_frame_names()): if (not overwrite): return ta.get_frame(frame_name) else: ta.drop_frames(frame_name) frame = load_frame(ta, in_path, frame_name, fields) print "****** Assigning train/test labels for cross validation... identiyfing patient population" add_udf_files(ta, [uphs_fields, filters]) frame.filter(lambda row: patid_filter(row)) frame.drop_duplicates() print "****** Assigning train/test labels for cross validation... assigning labels" frame.assign_sample(sample_percentages = [0.9, 0.1], sample_labels = [TRAIN_LABEL, TEST_LABEL], random_seed = 1776, output_column = CROSS_VALIDATION_CLASS) return frame
def create_ground_truth(ta, frame_name, in_path, overwrite = False): add_udf_files(ta, [uphs_fields, history_utilities, ground_truth_utilities, derived_features, filters]) if (frame_name in ta.get_frame_names()): if (not overwrite): return ta.get_frame(frame_name) else: ta.drop_frames([frame_name]) fields = [PATID, VISID, ADMIT_TYPE, ADM_DATE, DISCHARGE_DATE] base_frame_name = '__' + frame_name + '_uphs_demo_visit_types' jsonized_frame_name = '__' + frame_name + '_jsonized_visits' json_col_name = 'visit_history' print "*** CREATE GROUND TRUTH FRAME AT FRAME: " + frame_name print "****** loading data from raw json" base_frame = load_frame(ta, in_path, base_frame_name, fields) print "****** filtering out bad rows with missing information" base_frame.filter(lambda row: patid_filter(row) and visid_filter(row) and adm_date_filter(row) and emergency_admit_filter(row)) print "****** creating visit history atomic records" jsonized_frame = history_features.jsonize(ta, base_frame, jsonized_frame_name, json_col_name, key_col_name = PATID, overwrite = overwrite) print "****** collecting visit history" # historized_frame gets the output frame name since all subsequent changes are in-place and mutate the frame.. # it will be the frame that goes out as the ground truth frame ground_truth_history_delimiter = '|' historized_frame = history_features.historize(ta, jsonized_frame, frame_name, json_col_name, PATID, delimiter = ground_truth_history_delimiter) print "****** calculating READMIT_30 and READMIT_90 scores" LABELED_VISIT_JSON = 'LABELED_VISIT_JSON' # each patient gets a list of visits and their readmit scores historized_frame.add_columns(lambda row: per_visit_readmit_scores(row[json_col_name]), (LABELED_VISIT_JSON, str)) historized_frame.drop_columns([json_col_name]) # per patient list of visit/label readmit scores collapsed to per patient-visit readmit records historized_frame.flatten_column(LABELED_VISIT_JSON, delimiter = ground_truth_history_delimiter) print "****** placing READMIT_30 and READMIT_90 data into desired tabular format" # expanding json records per visit into multiple columns historized_frame.add_columns(lambda row: labeled_visit_to_columns(row[LABELED_VISIT_JSON]), labeled_visit_schema) historized_frame.drop_columns([LABELED_VISIT_JSON]) ta.drop_frames([base_frame, jsonized_frame]) return historized_frame
def get_basic_features(ta, frame_name, in_path, overwrite = True): if (frame_name in ta.get_frame_names()): if (not overwrite): return ta.get_frame(frame_name) else: ta.drop_frames(frame_name) frame = loader.load_frame(ta, in_path, frame_name, data_columns) add_udf_files(ta, [uphs_fields, filters]) frame.filter(lambda row: patid_filter(row) and visid_filter(row)) impute_with_constants(frame, default_values) return frame