def get_recent_lab_values(empi, date): p = loader.get_patient_by_EMPI(empi) lab_latest = {} if 'Lab' in p.keys(): for lab in p['Lab']: if lab['Seq_Date_Time'] and extract_data.parse_date(lab['Seq_Date_Time']) < date: lab_date = extract_data.parse_date(lab['Seq_Date_Time']) if lab['Group_Id'] in lab_latest: recorded_test_date = lab_latest[lab['Group_Id']][0] if lab_date > recorded_test_date: # keep most recent test value lab_latest[lab['Group_Id']] = (lab_date, lab['Result']) else: lab_latest[lab['Group_Id']] = (lab_date, lab['Result']) return lab_latest
def main(): empi = "FAKE_EMPI_385" # testing a single patient symptoms_regexes = getSymptomsRegexes() person = loader.get_patient_by_EMPI(empi) operation_date = build_graphs.get_operation_date(person) note_types = ['Car', 'Lno'] person_pos_history = {} person_neg_history = {} sec_per_day = 24 * 60 * 60 for note_type in note_types: print 'Examining ' + note_type + ' Notes for Patient ' + empi date_key = extract_data.get_date_key(note_type) if note_type in person.keys() and date_key != None: for i in range(len(person[note_type])): print '\tNote' + str(i) doc = person[note_type][i] date = extract_data.parse_date(doc[date_key]) if date != None: delta_days = (date - operation_date).total_seconds() / sec_per_day for sym in symptoms_regexes: normal, neg_pre, neg_suff = [bool(x.search(doc['free_text'])) for x in symptoms_regexes[sym]] if neg_pre or neg_suff: if sym in person_neg_history: person_neg_history[sym].append(delta_days) else: person_neg_history[sym] = [delta_days] print '\t\tNegative,' + sym + ',' + str(delta_days) elif normal: if sym in person_pos_history: person_pos_history[sym].append(delta_days) else: person_pos_history[sym] = [delta_days] print '\t\tPositive,' + sym + ',' + str(delta_days) return person_pos_history, person_neg_history
def get_sent_vector(self, empi): patient = loader.get_patient_by_EMPI(empi) operation_date = extract_data.get_operation_date(patient) diagnoses = get_diagnoses(empi) date_key = extract_data.get_date_key(self.note_type) notes = [] if self.note_type in patient.keys() and date_key != None: # Get sorted list of notes before procedure time_idx_pairs = [] for i in range(len(patient[self.note_type])): doc = patient[self.note_type][i] date = extract_data.parse_date(doc[date_key]) if date != None and date < operation_date: time_idx_pairs.append((operation_date - date, i)) time_idx_pairs.sort() for time, idx in time_idx_pairs[:self.max_notes]: doc = patient[self.note_type][idx] notes.append(doc['free_text']) # ensure that notes vector length is equal to max_notes if len(notes) < self.max_notes: delta = self.max_notes - len(notes) for i in range(delta): notes.append('') # Turn notes into Doc Vectors vectors = map(self.get_sent_vector_from_doc, notes) return np.array(vectors).flatten()
def get_sent_vector(self, empi): patient = loader.get_patient_by_EMPI(empi) operation_date = extract_data.get_operation_date(patient) diagnoses = get_diagnoses(empi) date_key = extract_data.get_date_key(self.note_type) notes = [] if self.note_type in patient.keys() and date_key != None: # Get sorted list of notes before procedure time_idx_pairs = [] for i in range(len(patient[self.note_type])): doc = patient[self.note_type][i] date = extract_data.parse_date(doc[date_key]) if date != None and date < operation_date: time_idx_pairs.append((operation_date - date, i)) time_idx_pairs.sort() for time,idx in time_idx_pairs[:self.max_notes]: doc = patient[self.note_type][idx] notes.append(doc['free_text']) # ensure that notes vector length is equal to max_notes if len(notes) < self.max_notes: delta = self.max_notes - len(notes) for i in range(delta): notes.append('') # Turn notes into Doc Vectors vectors = map(self.get_sent_vector_from_doc, notes) return np.array(vectors).flatten()
def get_diagnoses(empi): """Given an empi, will the return the diagnosis timeline T for that patient. T is just an array of tuples of the form (diagnosis date, Code_Type, code, diagnosis name), sorted by date. Note that a given date may, and often does, have several diagnoses. Also, a diagnosis can be repeatedly reported on every visit.""" p = loader.get_patient_by_EMPI(empi) diagnoses = [] if 'Dia' in p.keys(): for dia in p['Dia']: diagnoses.append((extract_data.parse_date(dia['Date']), dia['Code_Type'], dia['Code'], dia['Diagnosis_Name'])) diagnoses.sort() return diagnoses
def get_concatenated_notes(self, empi): person = loader.get_patient_by_EMPI(empi) operation_date = build_graphs.get_operation_date(person) date_key = extract_data.get_date_key(self.type) notes = [] sec_per_month = 24 * 60 * 60 * (365.0 / 12) if self.type in person.keys() and date_key != None: for i in range(len(person[self.type])): doc = person[self.type][i] date = extract_data.parse_date(doc[date_key]) if date != None and date < operation_date: if self.look_back_months and (operation_date - date).total_seconds() > (self.look_back_months * sec_per_month): continue notes.append(doc['free_text']) return '\n\n'.join(notes)
def get_lab_history_before_date(empi, date, time_thresholds_months): """Given an empi and a date, will return a summarized history of the labs for that patient before the date. Specifically, will return a dictionary where the key is a lab group id and the value is a list of size len(time_threshold_months) where each index represents whether the lab was mostly high or low in the threshold times set it time_thresholds_months. For example, if we have 'BUN' => ['H', None, 'L'], then this indicates a transition from low (L) to high (H) leading up to the indicated date.""" p = loader.get_patient_by_EMPI(empi) lab_history_counts = {} """ lab_history_counts is 2-D array first dimension = time period second dimension = counts of 'H', 'L', and None example = [[15, 1, 2], ...] means in the past 1 month, 'H' was most (15 times) """ seconds_in_month = 365 * 24 * 60 * 60 / 12 values = ['H', 'L', None] if 'Lab' in p.keys(): for lab in p['Lab']: if lab['Seq_Date_Time'] and extract_data.parse_date(lab['Seq_Date_Time']) < date: lab_date = extract_data.parse_date(lab['Seq_Date_Time']) value = lab['Abnormal_Flag'] if lab['Abnormal_Flag'] in ['H', 'L'] else None value_index = values.index(value) time_index = 0 while time_index < len(time_thresholds_months) and (date - lab_date).total_seconds() > (time_thresholds_months[time_index] * seconds_in_month): time_index += 1 if time_index >= len(time_thresholds_months): continue if lab['Group_Id'] not in lab_history_counts: lab_history_counts[lab['Group_Id']] = np.zeros([len(time_thresholds_months), len(values)]) lab_history_counts[lab['Group_Id']][time_index][value_index] += 1 lab_history = {} for lab_name in lab_history_counts: lab_history[lab_name] = [None] * len(time_thresholds_months) for i in range(len(time_thresholds_months)): lab_history[lab_name][i] = values[lab_history_counts[lab_name][i].argmax()] return lab_history
def get_labs_before_date(empi, date): """Given an empi and a date, will return the labs for that patient before that date. Specifically, will return four dictionaries where the key is always the lab group id and the values are the total counts, low counts, high counts, and latest (date, low/high) tuple for that test respectively. Note that low and high mean the test value was below or above the norm respectively.""" p = loader.get_patient_by_EMPI(empi) lab_counts = {} lab_lows = {} lab_highs = {} lab_latest = {} if 'Lab' in p.keys(): for lab in p['Lab']: if lab['Seq_Date_Time'] and extract_data.parse_date(lab['Seq_Date_Time']) < date: if lab['Group_Id'] in lab_counts: lab_counts[lab['Group_Id']] += 1 else: lab_counts[lab['Group_Id']] = 1 lab_date = extract_data.parse_date(lab['Seq_Date_Time']) if lab['Group_Id'] in lab_latest: recorded_test_date = lab_latest[lab['Group_Id']][0] if lab_date > recorded_test_date: # keep most recent test value lab_latest[lab['Group_Id']] = (lab_date, lab['Abnormal_Flag']) else: lab_latest[lab['Group_Id']] = (lab_date, lab['Abnormal_Flag']) if lab['Abnormal_Flag']: if lab['Abnormal_Flag'] == 'L': if lab['Group_Id'] in lab_lows: lab_lows[lab['Group_Id']] += 1 else: lab_lows[lab['Group_Id']] = 1 elif lab['Abnormal_Flag'] == 'H': if lab['Group_Id'] in lab_highs: lab_highs[lab['Group_Id']] += 1 else: lab_highs[lab['Group_Id']] = 1 return lab_counts, lab_lows, lab_highs, lab_latest
def get_encounters(empi): """Given an empi, returns a list of encounters for that patient sorted by Admit Date (since Discharge Date is not always recorded).""" p = loader.get_patient_by_EMPI(empi) encounters = [] if 'Enc' in p.keys(): for enc in p['Enc']: extra_diagnoses = 0 for i in range(1, 10): if enc['Diagnosis_' + str(i)]: extra_diagnoses += 1 if enc['Admit_Date']: encounters.append((extract_data.parse_date(enc['Admit_Date']), str(enc['Inpatient_Outpatient']), extract_data.parse_date(enc['Discharge_Date']), int(enc['LOS_Days']) if enc['LOS_Days'] else 0, extra_diagnoses)) encounters.sort(key = lambda x: x[0]) # just sort on Admit_Date return encounters
def select_doc(self, doc, operation_date, doc_type): """ description: function that returns is specific doc should be used inputs: dict of the doc, datetime of the procedure, string of doc type output: boolean """ doc_date_text = doc[extract_data.get_date_key(doc_type)] doc_date = extract_data.parse_date(doc_date_text) if doc_date is None: return False time_diff = (doc_date - operation_date).days if self.time_horizon != None: return time_diff <= 0 and abs(time_diff) <= abs(self.time_horizon) else: return time_diff <= 0
def select_doc(self, doc, operation_date, doc_type): """ description: function that returns is specific doc should be used inputs: dict of the doc, datetime of the procedure, string of doc type output: boolean """ doc_date_text = doc[extract_data.get_date_key(doc_type)] doc_date = extract_data.parse_date(doc_date_text) if doc_date is None: return False time_diff = (doc_date - operation_date).days if self.time_horizon != None: return time_diff <= 0 and abs(time_diff) <= abs(self.time_horizon) else: return time_diff <= 0
def parse_value(self, doc, operation_date, doc_type): """ description: function that returns the desired value from a document inputs: dict of the doc, datetime of the procedure, string of doc type ouput: value of any type """ note = doc['free_text'].lower() doc_date_text = doc[extract_data.get_date_key(doc_type)] doc_date = extract_data.parse_date(doc_date_text) delta_days = (doc_date - operation_date).days values = [] for pattern in self.patterns: values += [ x for x in re.findall(pattern, note) if len(x) > 0 and not x in [".", " "] ] if values != []: pass # print values if len(values) > 0 and not self.method in ['found', 'count', 'other']: val_before = values values = [float(x) for x in values if unicode(x).isnumeric()] try: if len(values) == 0: return None else: return (delta_days, sum(values) / len(values)) except: print "\n" * 5 print values print "\n" * 5 raise elif self.method == 'other': #returns entire value list return (delta_days, values) elif self.method == 'found': return (delta_days, 1) elif self.method == 'count': return [(delta_days, len(values))] return None
def get_latest_concatenated_notes(self, empi): person = loader.get_patient_by_EMPI(empi) operation_date = build_graphs.get_operation_date(person) date_key = extract_data.get_date_key(self.type) notes = [] if self.type in person.keys() and date_key != None: time_key_pairs = [] for i in range(len(person[self.type])): doc = person[self.type][i] date = extract_data.parse_date(doc[date_key]) if date != None and date < operation_date: time_key_pairs.append((operation_date - date, i)) time_key_pairs.sort() for time,key in time_key_pairs[:self.max_notes]: doc = person[self.type][key] notes.append(doc['free_text']) # ensure that notes vector length is equal to max_notes if len(notes) < self.max_notes: delta = self.max_notes - len(notes) for i in range(delta): notes.append('') return np.array(notes)
def parse_value(self, doc, operation_date, doc_type): """ description: function that returns the desired value from a document inputs: dict of the doc, datetime of the procedure, string of doc type ouput: value of any type """ note = doc['free_text'].lower() doc_date_text = doc[extract_data.get_date_key(doc_type)] doc_date = extract_data.parse_date(doc_date_text) delta_days = (doc_date - operation_date).days values = [] for pattern in self.patterns: values += [x for x in re.findall(pattern, note) if len(x) > 0 and not x in [".", " "]] if values != []: pass# print values if len(values) > 0 and not self.method in ['found', 'count', 'other']: val_before = values values = [float(x) for x in values if unicode(x).isnumeric()] try: if len(values) == 0: return None else: return (delta_days, sum(values)/len(values)) except: print "\n"*5 print values print "\n"*5 raise elif self.method == 'other': #returns entire value list return (delta_days, values ) elif self.method == 'found': return (delta_days, 1) elif self.method == 'count': return [(delta_days, len(values))] return None