class FeatureMatrix:
def __init__(self, variable, num_data_points, params=None):
# Process arguments.
self._var = variable
self._num_rows = num_data_points
if params is None:
self._params = {}
else:
self._params = params
# Initialize FeatureMatrixFactory.
self._factory = FeatureMatrixFactory()
# Initialize DB connection.
self._connection = DBUtil.connection()
def _query_patient_episodes(self,
query,
pat_id_col=None,
index_time_col=None):
# Initialize DB cursor.
cursor = self._connection.cursor()
# Fetch and return results.
log.info('query: %s' % str(query))
if isinstance(query, basestring):
cursor.execute(query)
else:
log.info('query.params: %s' % str(query.params))
cursor.execute(str(query), query.params)
# Parse arguments.
if pat_id_col is None:
pat_id_col = 'pat_id'
if index_time_col is None:
index_time_col = 'index_time'
self._factory.setPatientEpisodeInput(cursor, pat_id_col,
index_time_col)
num_episodes = self._factory.processPatientEpisodeInput()
return num_episodes
def _querystr_patient_episodes(self,
querystr,
pat_id_col=None,
index_time_col=None):
# Initialize DB cursor.
cursor = self._connection.cursor()
# Fetch and return results.
log.info('query: %s' % querystr)
cursor.execute(querystr)
# Parse arguments.
if pat_id_col is None:
pat_id_col = 'pat_id'
if index_time_col is None:
index_time_col = 'index_time'
self._factory.setPatientEpisodeInput(cursor, pat_id_col,
index_time_col)
num_episodes = self._factory.processPatientEpisodeInput()
return num_episodes
def _add_features(self, index_time_col=None):
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
self._add_time_features(index_time_col)
self._add_demographic_features()
self._add_treatment_team_features()
self._add_comorbidity_features()
self._add_flowsheet_features()
self._add_lab_component_features()
else:
# elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
self._add_time_features(index_time_col)
self._add_demographic_features()
self._add_comorbidity_features()
self._add_lab_component_features()
if LocalEnv.DATASET_SOURCE_NAME == 'UCSF':
self._add_treatment_team_features()
self._add_flowsheet_features()
def _add_time_features(self, index_time_col=None):
log.info('Adding admit date features...')
# Add admission date.
ADMIT_DX_CATEGORY_ID = 2
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
self._factory.addClinicalItemFeaturesByCategory([ADMIT_DX_CATEGORY_ID], \
dayBins=[], label='AdmitDxDate', features='pre')
else:
#elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
self._factory.addClinicalItemFeaturesByCategory_UMich([ADMIT_DX_CATEGORY_ID], \
dayBins=[], label='AdmitDxDate', features='pre', tableName='encounters')
# Add time cycle features.
log.info('Adding time cycle features...')
if index_time_col is None:
index_time_col = 'index_time'
self._factory.addTimeCycleFeatures(index_time_col, 'month')
self._factory.addTimeCycleFeatures(index_time_col, 'hour')
def _add_demographic_features(self):
log.info('Adding demographic features...')
# Add birth and death.
self._add_lifespan_features()
# Add sex features.
self._add_sex_features()
# Add race features.
self._add_race_features()
def _add_lifespan_features(self):
log.info('Adding lifespan features...')
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE': # TODO
self._factory.addClinicalItemFeatures(['Birth'],
dayBins=[],
features="pre")
self._factory.addClinicalItemFeatures(['Death'],
dayBins=[],
features="post")
else:
#elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
self._factory.addClinicalItemFeatures_UMich(
['Birth'],
dayBins=[],
features="pre",
clinicalItemType=None,
clinicalItemTime='Birth',
tableName='pt_info')
def _add_sex_features(self):
log.info('Adding sex features...')
SEX_FEATURES = ["Male", "Female"]
for feature in SEX_FEATURES:
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE': # TODO
self._factory.addClinicalItemFeatures([feature],
dayBins=[],
features="pre")
else:
#elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
self._factory.addClinicalItemFeatures_UMich(
[feature],
dayBins=[],
features="pre",
clinicalItemType='GenderName',
clinicalItemTime=None,
tableName="demographics")
def _add_race_features(self):
log.info('Adding race features...')
for feature in self._factory.queryAllRaces():
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE': # TODO
self._factory.addClinicalItemFeatures([feature],
dayBins=[],
features="pre")
else:
#elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
self._factory.addClinicalItemFeatures_UMich(
[feature],
dayBins=[],
features="pre",
clinicalItemType='RaceName',
clinicalItemTime=None,
tableName='demographics')
def _add_treatment_team_features(self):
log.info('Adding treatment team features...')
self._factory.addTreatmentTeamFeatures(features="pre")
def _add_comorbidity_features(self):
log.info('Adding comorbidity features...')
self._factory.addCharlsonComorbidityFeatures(features='pre')
def _add_flowsheet_features(self):
log.info('Adding flowsheet features...')
# Look at flowsheet results from the previous days
FLOW_PRE_TIME_DELTAS = [datetime.timedelta(-3)]
# Don't look into the future, otherwise cheating the prediction
FLOW_POST_TIME_DELTA = datetime.timedelta(0)
# Add flowsheet features for a variety of generally useful vitals.
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
BASIC_FLOWSHEET_FEATURES = [
"BP_High_Systolic", "BP_Low_Diastolic", "FiO2",
"Glasgow Coma Scale Score", "Pulse", "Resp", "Temp", "Urine"
]
elif LocalEnv.DATASET_SOURCE_NAME == 'UCSF':
BASIC_FLOWSHEET_FEATURES = [
'SBP', 'DBP', 'FiO2', 'Pulse', 'Resp', 'Temp', 'o2flow'
]
for pre_time_delta in FLOW_PRE_TIME_DELTAS:
log.info('\t\tpreTimeDelta: %s' % pre_time_delta)
self._factory.addFlowsheetFeatures(BASIC_FLOWSHEET_FEATURES, \
pre_time_delta, FLOW_POST_TIME_DELTA)
def _add_lab_component_features(self):
# Look for lab data 90 days before each episode, but never after self.
# Look at lab results from the previous days
LAB_PRE_TIME_DELTAS = [datetime.timedelta(-14)]
# Don't look into the future, otherwise cheating the prediction
LAB_POST_TIME_DELTA = datetime.timedelta(0)
# Add result features for a variety of generally useful components.
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
BASIC_LAB_COMPONENTS = [
'WBC', # White Blood Cell
'HCT', # Hematocrit
'PLT', # Platelet Count
'NA', # Sodium, Whole Blood
'K', # Potassium, Whole Blood
'CO2', # CO2, Serum/Plasma
'BUN', # Blood Urea Nitrogen
'CR', # Creatinine
'TBIL', # Total Bilirubin
'ALB', # Albumin
'CA', # Calcium
'LAC', # Lactic Acid
'ESR', # Erythrocyte Sedimentation Rate
'CRP', # C-Reactive Protein
'TNI', # Troponin I
'PHA', # Arterial pH
'PO2A', # Arterial pO2
'PCO2A', # Arterial pCO2
'PHV', # Venous pH
'PO2V', # Venous pO2
'PCO2V' # Venous pCO2
]
elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
BASIC_LAB_COMPONENTS = [
'WBC', # White Blood Cell
'HCT', # Hematocrit
'PLT', # Platelet Count
'SOD', # Sodium, Whole Blood
'POT', # Potassium, Whole Blood
'CO2', # CO2, Serum/Plasma
'UN', # Blood Urea Nitrogen
'CREAT', # Creatinine
'TBIL', # Total Bilirubin
'ALB', # Albumin
'CAL', # Calcium
'LACTA', # Lactic Acid; LACTA & LACTV are more frequent
"WEST", # Erythrocyte Sedimentation Rate
'CRP', # C-Reactive Protein
'TROP', # Troponin I
'pHA', # Arterial pH
'PO2AA', # Arterial pO2
'PCOAA2', # Arterial pCO2
'pHV', # Venous pH
'pO2V', # Venous pO2
'pCO2V', # Venous pCO2
]
elif LocalEnv.DATASET_SOURCE_NAME == 'UCSF':
BASIC_LAB_COMPONENTS = [
'WBC', # White Blood Cell
'HCT', # Hematocrit
'PLT', # Platelet Count
'NAWB', # Sodium, Whole Blood
'K', # Potassium, Whole Blood
'CO2', # CO2, Serum/Plasma
'BUN', # Blood Urea Nitrogen
'CREAT', # Creatinine
'TBILI', # Total Bilirubin
'ALB', # Albumin
'CA', # Calcium
'LACTWB', # Lactic Acid; LACTA & LACTV are more frequent
"ESR", # Erythrocyte Sedimentation Rate
'CRP', # C-Reactive Protein
'TRPI', # Troponin I
'PH37', # Arterial pH
'PO2', # Arterial pO2
'PCO2' # Arterial pCO2
# 'pHV', # Venous pH
# 'pO2V', # Venous pO2
# 'pCO2V', # Venous pCO2
]
log.info('Adding lab component features...')
for component in BASIC_LAB_COMPONENTS:
log.info('\t%s' % component)
for preTimeDelta in LAB_PRE_TIME_DELTAS:
log.info('\t\t%s' % preTimeDelta)
self._factory.addLabResultFeatures([component], False,
preTimeDelta,
LAB_POST_TIME_DELTA)
def _build_matrix(self, header=None, dest_path=None):
log.info('Building matrix...')
self._factory.buildFeatureMatrix(header, dest_path)
def write_matrix(self, dest_path, header=None):
log.info('Writing matrix file...')
fm_io = FeatureMatrixIO()
# Get old matrix file.
source_path = self._factory.getMatrixFileName()
# Write to new matrix filee.
matrix_file = open(dest_path, 'w')
# for line in header:
# matrix_file.write('# %s\n' % line)
for line in open(source_path, 'r'):
if line[0] != '#':
matrix_file.write(line)
# Delete old matrix file.
os.remove(source_path)
def _build_matrix_header(self, params=None):
# params['include_lab_suffix_summary'] = True/False
# params['include_clinical_item_suffix_summary'] = True/False
# params['data_overview'] = str description.
# params['field_summary'] = str description
header = list()
file_summary = self._build_file_summary(params['matrix_path'], \
params['matrix_module'])
header.extend(file_summary)
header.extend([''])
if params.get('data_overview'):
header.extend(params['data_overview'])
header.extend([''])
if params.get('field_summary'):
header.extend(params['field_summary'])
header.extend([''])
if params.get('include_clinical_item_suffix_summary'):
ci_suffix_summary = self._build_clinical_item_suffix_summary()
header.extend(ci_suffix_summary)
header.extend([''])
if params.get('include_lab_suffix_summary'):
lab_suffix_summary = self._build_flowsheet_and_lab_result_suffix_summary(
)
header.extend(lab_suffix_summary)
header.extend([''])
return header
def _build_file_summary(self, matrix_path, matrix_module):
summary = list()
# <file_name.tab>
matrix_name = matrix_path.split('/')[-1]
summary.append(matrix_name)
# Created: <timestamp>
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
summary.append('Created: %s' % timestamp)
# Source: __name__
module_name = matrix_module.split('/')[-1]
summary.append('Source: %s' % module_name)
# Command: Pipeline()
class_name = module_name.split('.')[0]
args = [self._var, str(self._num_rows)]
for key, value in self._params:
args.append('%s=%s' % (key, value))
command = '%s(%s)' % (class_name, ', '.join(args))
summary.append('Command: %s' % command)
return summary
def _build_clinical_item_suffix_summary(self):
summary = list()
# [clinical_item] fields may have the following suffixes:
summary.append(
' [clinical_item] fields may have the following suffixes:')
# ___.pre - how many times has this occurred before order_time?
summary.append(
' ___.pre - how many times has this occurred before order_time?'
)
# ___.pre.Xd - how many times has this occurred within X days before index_time?
summary.append(
' ___.pre.Xd - how many times has this occurred within X days before index_time?'
)
# ___.preTimeDays - how many days before order_time was last occurrence?
summary.append(
' ___.preTimeDays - how many days before order_time was last occurrence?'
)
return summary
def _build_flowsheet_and_lab_result_suffix_summary(self):
summary = list()
# [flowsheet] and [lab_result] fields may have the following suffixes:
summary.append(
' [flowsheet] and [lab_result] fields may have the following suffixes:'
)
# ___.X_Y.count - # of result values between X and Y days of index_time.
summary.append(
' ___.X_Y.count - # of result values between X and Y days of index_time.'
)
# ___.X_Y.countInRange - # of result values in normal range.
summary.append(
' ___.X_Y.countInRange - # of result values in normal range.')
# ___.X_Y.min - minimum result value.
summary.append(' ___.X_Y.min - minimum result value.')
# ___.X_Y.max - maximum result value.
summary.append(' ___.X_Y.max - maximum result value.')
# ___.X_Y.median - median result value.
summary.append(' ___.X_Y.median - median result value.')
# ___.X_Y.std - standard deviation of result values.
summary.append(
' ___.X_Y.std - standard deviation of result values.')
# ___.X_Y.first - first result value.
summary.append(' ___.X_Y.first - first result value.')
# ___.X_Y.last - last result value.
summary.append(' ___.X_Y.last - last result value.')
# ___.X_Y.diff - difference between penultimate and proximate values.
summary.append(
' ___.X_Y.diff - difference between penultimate and proximate values.'
)
# ___.X_Y.slope - slope between penultimate and proximate values.
summary.append(
' ___.X_Y.slope - slope between penultimate and proximate values.'
)
# ___.X_Y.proximate - closest result value to order_time.
summary.append(
' ___.X_Y.proximate - closest result value to order_time.')
# ___.X_Y.firstTimeDays - time between first and order_time.
summary.append(
' ___.X_Y.firstTimeDays - time between first and order_time.')
# ___.X_Y.lastTimeDays - time between last and order_time.
summary.append(
' ___.X_Y.lastTimeDays - time between last and order_time.')
# ___.X_Y.proximateTimeDays - time between proximate and order_time.
summary.append(
' ___.X_Y.proximateTimeDays - time between proximate and order_time.'
)
return summary
class LabNormalityPredictionPipeline(SupervisedLearningPipeline):
def __init__(self,
lab_panel,
num_episodes,
use_cache=None,
random_state=None,
isLabPanel=True,
timeLimit=None,
notUsePatIds=None,
holdOut=False,
pat_batch_ind=None,
includeLastNormality=True):
self.notUsePatIds = notUsePatIds
self.pat_batch_ind = pat_batch_ind
self.usedPatIds = []
SupervisedLearningPipeline.__init__(
self,
lab_panel,
num_episodes,
use_cache,
random_state,
isLabPanel,
timeLimit,
holdOut,
isLabNormalityPredictionPipeline=True)
# TODO: naming of lab_panel
self._factory = FeatureMatrixFactory()
self._build_raw_feature_matrix()
if self._isLabPanel:
self.ylabel = 'all_components_normal'
else:
self.ylabel = 'component_normal'
self.includeLastNormality = includeLastNormality
if self.includeLastNormality:
fm_io = FeatureMatrixIO()
df = fm_io.read_file_to_data_frame('data/' + lab_panel +
'/%s-normality-matrix-raw.tab' %
lab_panel)
df = df.sort_values(['pat_id',
'order_time']).reset_index(drop=True)
df['last_normality'] = df['order_proc_id'].apply(
lambda x: float('nan'))
for i in range(1, df.shape[0]):
if df.ix[i, 'pat_id'] == df.ix[i - 1, 'pat_id']:
df.ix[i, 'last_normality'] = df.ix[i - 1, self.ylabel]
df.to_csv('data/' + lab_panel +
'/%s-normality-matrix-raw.tab' % lab_panel,
index=False,
sep='\t')
data_lab_folder = self._fetch_data_dir_path(
inspect.getfile(inspect.currentframe()))
feat2imputed_dict_path = data_lab_folder + '/feat2imputed_dict.pkl'
if holdOut:
'''
For holdOut evaluation data, produce the raw matrix, pick
features according to the saved feat2imputed_dict.
'''
self.feat2imputed_dict = pickle.load(
open(feat2imputed_dict_path, 'r'))
self._build_processed_feature_matrix_holdout()
self._analyze_predictors_on_holdout()
else:
'''
For training/validation data, record the pat_ids,
selected features and their imputed value correspondingly.
'''
pickle.dump(self.usedPatIds,
open('data/used_patient_set_%s.pkl' % self._var, 'w'),
pickle.HIGHEST_PROTOCOL)
self._build_processed_feature_matrix()
self._build_baseline_results() # TODO: prototype in SLPP
# return
# TODO: find better place to put the dict.pkl
pickle.dump(self.feat2imputed_dict,
open(feat2imputed_dict_path, 'w'),
pickle.HIGHEST_PROTOCOL)
self._train_and_analyze_predictors()
def _build_model_dump_path(self, algorithm):
template = '%s' + '-normality-%s-model.pkl' % algorithm
pipeline_file_name = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_model_dump_path(self, template, \
pipeline_file_name)
def _build_raw_matrix_path(self):
raw_matrix_filename = '%s-normality-matrix-raw.tab' % self._var #
raw_matrix_filepath = os.path.join('data', self._var,
raw_matrix_filename) # TODO
if not os.path.exists('data'):
os.mkdir('data')
if not os.path.exists(os.path.join('data', self._var)):
os.mkdir(os.path.join('data', self._var))
return raw_matrix_filepath
if not self._holdOut:
template = '%s-normality-matrix-raw.tab'
else:
template = '%s-normality-matrix-%d-episodes-raw-holdout.tab'
pipeline_file_name = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_matrix_path(self, template, \
pipeline_file_name)
def _build_raw_feature_matrix(self):
raw_matrix_path = self._build_raw_matrix_path()
matrix_class = LabNormalityMatrix
SupervisedLearningPipeline._build_raw_feature_matrix(self, matrix_class, \
raw_matrix_path)
if not self._holdOut:
fm_io = FeatureMatrixIO()
matrix = fm_io.read_file_to_data_frame(raw_matrix_path)
self.usedPatIds = set(matrix['pat_id'].values)
def _build_baseline_results(self):
if not self._holdOut:
template = '%s-normality-matrix-raw.tab'
else:
template = '%s-normality-matrix-%d-episodes-raw-holdout.tab'
pipeline_file_name = inspect.getfile(inspect.currentframe())
# raw_matrix_path = SupervisedLearningPipeline._build_matrix_path(self, template, \
# pipeline_file_name)
raw_matrix_path = self._build_raw_matrix_path()
# Another direct call to the _factory instance
self._factory.obtain_baseline_results(
raw_matrix_path=raw_matrix_path,
random_state=self._random_state,
isLabPanel=self._isLabPanel,
isHoldOut=self._holdOut) #TODO: file name
def _build_processed_matrix_path(self):
processed_matrix_filename = '%s-normality-matrix-processed.tab' % self._var #
processed_matrix_path = os.path.join('data', self._var,
processed_matrix_filename) # TODO
return processed_matrix_path
if not self._holdOut:
template = '%s-normality-matrix-processed.tab'
else:
template = '%s-normality-matrix-%d-episodes-processed-holdout.tab'
pipeline_file_path = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_matrix_path(self, template, \
pipeline_file_path)
def _build_processed_feature_matrix_holdout(self):
fm_io = FeatureMatrixIO()
raw_matrix = fm_io.read_file_to_data_frame(
self._build_raw_matrix_path())
# if outcome_label in self.feat2imputed_dict:
# self.feat2imputed_dict.pop(outcome_label)
#
# processed_matrix = raw_matrix[self.feat2imputed_dict.keys()+[outcome_label]].copy()
'''
TODO: feat2imputed_dict includes the outcome label
'''
processed_matrix = raw_matrix[self.feat2imputed_dict.keys()].copy()
# TODO: tmp solution!
tmp_path = self._build_processed_matrix_path().replace(
"2000", "10000").replace("-holdout", "")
fm_io1 = FeatureMatrixIO()
processed_matrix_previous = fm_io1.read_file_to_data_frame(tmp_path)
processed_matrix = processed_matrix[processed_matrix_previous.columns]
# TODO: tmp solution!
for feat in self.feat2imputed_dict.keys():
processed_matrix[feat] = processed_matrix[feat].fillna(
self.feat2imputed_dict[feat])
fm_io.write_data_frame_to_file(processed_matrix, \
self._build_processed_matrix_path(), None)
def _build_processed_feature_matrix(self):
# Define parameters for processing steps.
params = {}
raw_matrix_path = self._build_raw_matrix_path()
processed_matrix_path = self._build_processed_matrix_path()
features_to_add = {}
imputation_strategies = { #'sxu_new_imputation'
}
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
features_to_remove = [
'pat_id', 'order_time', 'order_proc_id', 'Birth.pre',
'Male.preTimeDays', 'Female.preTimeDays',
'RaceWhiteHispanicLatino.preTimeDays',
'RaceWhiteNonHispanicLatino.preTimeDays',
'RaceHispanicLatino.preTimeDays', 'RaceAsian.preTimeDays',
'RaceBlack.preTimeDays', 'RacePacificIslander.preTimeDays',
'RaceNativeAmerican.preTimeDays', 'RaceOther.preTimeDays',
'RaceUnknown.preTimeDays', 'Death.post', 'Death.postTimeDays'
]
if self._isLabPanel:
features_to_remove += [
'proc_code', 'num_components', 'num_normal_components',
'abnormal_panel'
]
outcome_label = 'all_components_normal' #
else:
features_to_remove += ['base_name']
outcome_label = 'component_normal' # TODO: danger, previous version might not consistent!
else:
features_to_remove = [
'pat_id',
'order_time',
'order_proc_id',
'Birth.pre',
'Male.preTimeDays',
'Female.preTimeDays',
# 'Caucasian.preTimeDays',
# 'Hispanic.preTimeDays',
# 'Native Hawaiian and Other Pacific Islander.preTimeDays'
]
RACE_FEATURES = self._factory.queryAllRaces()
features_to_remove += [x + '.preTimeDays' for x in RACE_FEATURES]
if self._isLabPanel:
features_to_remove += [
'proc_code', 'num_normal_components', 'num_components'
]
outcome_label = 'all_components_normal'
else:
features_to_remove += ['base_name']
outcome_label = 'component_normal' #
features_to_keep = [
# Keep the # of times it's been ordered in past, even if low info.
'%s.pre' % self._var
]
if self.includeLastNormality:
features_to_keep.append('last_normality')
selection_problem = FeatureSelector.CLASSIFICATION
selection_algorithm = FeatureSelector.RECURSIVE_ELIMINATION
percent_features_to_select = 0.05
matrix_class = LabNormalityMatrix
pipeline_file_path = inspect.getfile(inspect.currentframe())
random_state = self._random_state
data_overview = [
# Overview:
'Overview',
# The outcome label is ___.
'The outcome label is %s.' % outcome_label,
# %s is a boolean indicator which summarizes whether all components
'%s is a boolean indicator which summarizes whether all components '
% outcome_label,
# in the lab panel order represented by a given row are normal.
'in the lab panel order represented by a given row are normal.',
# Each row represents a unique lab panel order.
'Each row represents a unique lab panel order.',
# Each row contains fields summarizing the patient's demographics,
"Each row contains fields summarizing the patient's demographics",
# inpatient admit date, prior vitals, and prior lab results.
'inpatient admit date, prior vitals, and prior lab results.',
# Most cells in matrix represent a count statistic for an event's
"Most cells in matrix represent a count statistic for an event's",
# occurrence or a difference between an event's time and index_time.
"occurrence or a difference between an event's time and index_time.",
]
# Bundle parameters into single object to be unpacked in SLP.
params['raw_matrix_path'] = raw_matrix_path
params['processed_matrix_path'] = processed_matrix_path
params['features_to_add'] = features_to_add
params['features_to_keep'] = features_to_keep
params['imputation_strategies'] = imputation_strategies
params['features_to_remove'] = features_to_remove
params['outcome_label'] = outcome_label
params['selection_problem'] = selection_problem
params['selection_algorithm'] = selection_algorithm
params['percent_features_to_select'] = percent_features_to_select
params['matrix_class'] = matrix_class
params['pipeline_file_path'] = pipeline_file_path
params['data_overview'] = data_overview
params['random_state'] = random_state
# Defer processing logic to SupervisedLearningPipeline.
SupervisedLearningPipeline._build_processed_feature_matrix(
self, params)
'''
For testing the model on the holdout set, should remember features
to select from the raw matrix of the holdout data.
'''
final_features = self._X_train.columns.values
if not self.feat2imputed_dict:
'''
The dict was not created during imputation.
Probably because the processed matrix was loaded from previous session.
Take the 'best guess' for the imputed value as the most common one in
any column.
'''
for feat in final_features:
most_freq_val = self._X_train[feat].value_counts().idxmax()
self.feat2imputed_dict[feat] = most_freq_val
'''
TODO: useless?!
'''
# curr_keys = self.feat2imputed_dict.keys()
#
# '''
# Only need to impute the selected features for the holdOut set.
# '''
# for one_key in curr_keys:
# if one_key not in final_features:
# self.feat2imputed_dict.pop(one_key)
def _analyze_predictors_on_holdout(self):
fm_io = FeatureMatrixIO()
algorithms_to_test = list()
algorithms_to_test.extend(SupervisedClassifier.SUPPORTED_ALGORITHMS)
pipeline_file_name = inspect.getfile(inspect.currentframe())
data_dir = SupervisedLearningPipeline._fetch_data_dir_path(
self, pipeline_file_name)
# for algorithm in SupervisedClassifier.SUPPORTED_ALGORITHMS:
# algorithms_to_test.append('bifurcated-%s' % algorithm)
log.debug('algorithms_to_test: %s' % algorithms_to_test)
for algorithm in algorithms_to_test:
log.info('analyzing %s...' % algorithm)
# If report_dir does not exist, make it.
report_dir = '/'.join([data_dir, algorithm])
pipeline_prefix = '%s-normality-prediction-%s' % (self._var,
algorithm)
predictor_path = self._build_model_dump_path(algorithm)
if os.path.exists(
predictor_path) and 'bifurcated' not in algorithm:
log.debug('Loading model from disk...')
# TODO(sbala): Fix loblib.load so that it works for bifurcated
# supervised classifiers.
self._predictor = joblib.load(predictor_path)
# self._features = self._X_train.columns
status = SupervisedClassifier.TRAINED
SupervisedLearningPipeline._analyze_predictor_holdoutset(
self, report_dir, pipeline_prefix)
def _train_and_analyze_predictors(self):
log.info('Training and analyzing predictors...')
problem = SupervisedLearningPipeline.CLASSIFICATION
meta_report = None
fm_io = FeatureMatrixIO()
# Build paths for output.
pipeline_file_name = inspect.getfile(inspect.currentframe())
data_dir = SupervisedLearningPipeline._fetch_data_dir_path(
self, pipeline_file_name)
# Test BifurcatedSupervisedClassifier and SupervisedClassifier.
algorithms_to_test = list()
algorithms_to_test.extend(SupervisedClassifier.SUPPORTED_ALGORITHMS)
# for algorithm in SupervisedClassifier.SUPPORTED_ALGORITHMS:
# algorithms_to_test.append('bifurcated-%s' % algorithm)
log.debug('algorithms_to_test: %s' % algorithms_to_test)
# Train and analyse algorithms.
for algorithm in algorithms_to_test:
log.info('Training and analyzing %s...' % algorithm)
# If report_dir does not exist, make it.
report_dir = '/'.join([data_dir, algorithm])
if not os.path.exists(report_dir):
os.makedirs(report_dir)
# Define hyperparams.
hyperparams = {}
hyperparams['algorithm'] = algorithm
hyperparams[
'hyperparam_strategy'] = SupervisedClassifier.EXHAUSTIVE_SEARCH
hyperparams['max_iter'] = 1024
hyperparams['random_state'] = self._random_state
# If bifurcated algorithm, define bifurcator.
if 'bifurcated' in algorithm:
# bifrucator = LAB.pre == 0
hyperparams['bifurcator'] = '%s.pre' % self._var
hyperparams[
'bifurcation_strategy'] = BifurcatedSupervisedClassifier.EQUAL
hyperparams['bifurcation_value'] = 0
hyperparams['bifurcated'] = True
# Train classifier.
predictor_path = self._build_model_dump_path(algorithm)
if os.path.exists(
predictor_path) and 'bifurcated' not in algorithm:
log.debug('Loading model from disk...')
# TODO(sbala): Fix loblib.load so that it works for bifurcated
# supervised classifiers.
self._predictor = joblib.load(predictor_path)
self._features = self._X_train.columns
status = SupervisedClassifier.TRAINED
else:
status = SupervisedLearningPipeline._train_predictor(
self, problem, [0, 1], hyperparams)
# If failed to train, write an error report.
y_train_counts = self._y_train[
self._y_train.columns[0]].value_counts()
y_test_counts = self._y_test[
self._y_test.columns[0]].value_counts()
if status == SupervisedClassifier.INSUFFICIENT_SAMPLES:
# Skip all analysis and reporting.
# This will be true for all algorithms, so just return.
# Build error report.
algorithm_report = DataFrame(
{
'lab_panel': [self._var],
'algorithm': [algorithm],
'error': [status],
'y_train.value_counts()': [y_train_counts.to_dict()],
'y_test.value_counts()': [y_test_counts.to_dict()]
},
columns=[
'lab_panel', 'algorithm', 'error',
'y_train.value_counts()', 'y_test.value_counts()'
])
header = [
'LabNormalityPredictionPipeline("%s", 10000)' % self._var
]
# Write error report.
fm_io.write_data_frame_to_file(algorithm_report, \
'/'.join([report_dir, '%s-normality-prediction-report.tab' % (self._var)]), \
header)
# If successfully trained, append to a meta report.
elif status == SupervisedClassifier.TRAINED:
pipeline_prefix = '%s-normality-prediction-%s' % (self._var,
algorithm)
SupervisedLearningPipeline._analyze_predictor(
self, report_dir, pipeline_prefix)
SupervisedLearningPipeline._analyze_predictor_traindata(
self, report_dir, pipeline_prefix)
# continue # Do not generate stats results here...
if meta_report is None:
meta_report = fm_io.read_file_to_data_frame('/'.join(
[report_dir,
'%s-report.tab' % pipeline_prefix]))
else:
algorithm_report = fm_io.read_file_to_data_frame('/'.join(
[report_dir,
'%s-report.tab' % pipeline_prefix]))
log.debug('algorithm_report: %s' % algorithm_report)
meta_report = meta_report.append(algorithm_report)
# Write predictor to disk.
predictor = SupervisedLearningPipeline.predictor(self)
predictor_path = self._build_model_dump_path(algorithm)
joblib.dump(predictor, predictor_path)
# After building per-algorithm reports, write to meta report.
# Note that if there were insufficient samples to build any of the
# algorithms, then meta_report will still be None.
if meta_report is not None:
header = [
'LabNormalityPredictionPipeline("%s", 10000)' % self._var
]
fm_io.write_data_frame_to_file(meta_report, \
'/'.join([data_dir, '%s-normality-prediction-report.tab' % self._var]), header)
class LabNormalityPredictionPipeline(SupervisedLearningPipeline):
def __init__(self, lab_panel, num_episodes, use_cache=None, random_state=None, isLabPanel=True,
notUsePatIds=[], pat_batch_ind=None):
self.notUsePatIds = notUsePatIds
self.pat_batch_ind = pat_batch_ind
self.usedPatIds = []
SupervisedLearningPipeline.__init__(self, lab_panel, num_episodes, use_cache, random_state, isLabPanel)
self._factory = FeatureMatrixFactory()
self._build_raw_feature_matrix()
#self._build_processed_feature_matrix()
#self._train_and_analyze_predictors()
def _build_model_dump_path(self, algorithm):
template = '%s' + '-normality-%s-model.pkl' % algorithm
pipeline_file_name = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_model_dump_path(self, template, \
pipeline_file_name)
def _build_raw_matrix_path(self):
if not self.pat_batch_ind:
template = '%s-normality-matrix-%d-episodes-raw.tab'
else:
template = '%s-normality-matrix-%d-episodes-raw-'+str(self.pat_batch_ind)+'.tab'
pipeline_file_name = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_matrix_path(self, template, \
pipeline_file_name)
def _build_raw_feature_matrix(self):
raw_matrix_path = self._build_raw_matrix_path()
matrix_class = LabNormalityMatrix
SupervisedLearningPipeline._build_raw_feature_matrix(self, matrix_class, \
raw_matrix_path)
def _build_processed_matrix_path(self):
template = '%s-normality-matrix-%d-episodes-processed.tab'
pipeline_file_path = inspect.getfile(inspect.currentframe())
return SupervisedLearningPipeline._build_matrix_path(self, template, \
pipeline_file_path)
def _build_processed_feature_matrix(self):
# Define parameters for processing steps.
params = {}
raw_matrix_path = self._build_raw_matrix_path()
processed_matrix_path = self._build_processed_matrix_path()
features_to_add = {}
imputation_strategies = {
}
if LocalEnv.DATASET_SOURCE_NAME == 'STRIDE':
features_to_remove = [
'pat_id', 'order_time', 'order_proc_id',
'proc_code', 'abnormal_panel',
'num_normal_components', 'Birth.pre',
'Male.preTimeDays', 'Female.preTimeDays',
'RaceWhiteHispanicLatino.preTimeDays',
'RaceWhiteNonHispanicLatino.preTimeDays',
'RaceHispanicLatino.preTimeDays',
'RaceAsian.preTimeDays',
'RaceBlack.preTimeDays',
'RacePacificIslander.preTimeDays',
'RaceNativeAmerican.preTimeDays',
'RaceOther.preTimeDays',
'RaceUnknown.preTimeDays',
'Death.post',
'Death.postTimeDays',
'num_components'
]
outcome_label = 'all_components_normal' # TODO: for component...
elif LocalEnv.DATASET_SOURCE_NAME == 'UMich':
features_to_remove = [
'pat_id', 'order_time', 'order_proc_id',
'Birth.pre',
'Male.preTimeDays', 'Female.preTimeDays',
# 'Caucasian.preTimeDays',
# 'Hispanic.preTimeDays',
# 'Native Hawaiian and Other Pacific Islander.preTimeDays'
]
RACE_FEATURES = self._factory.queryAllRaces()
features_to_remove += [x + '.preTimeDays' for x in RACE_FEATURES]
if self._isLabPanel:
features_to_remove += ['proc_code', 'num_normal_components', 'num_components']
else:
features_to_remove += ['base_name']
outcome_label = 'abnormal_lab'
features_to_keep = [
# Keep the # of times it's been ordered in past, even if low info.
'%s.pre' % self._var
]
selection_problem = FeatureSelector.CLASSIFICATION
selection_algorithm = FeatureSelector.RECURSIVE_ELIMINATION
percent_features_to_select = 0.05
matrix_class = LabNormalityMatrix
pipeline_file_path = inspect.getfile(inspect.currentframe())
random_state = self._random_state
data_overview = [
# Overview:
'Overview',
# The outcome label is ___.
'The outcome label is %s.' % outcome_label,
# %s is a boolean indicator which summarizes whether all components
'%s is a boolean indicator which summarizes whether all components ' % outcome_label,
# in the lab panel order represented by a given row are normal.
'in the lab panel order represented by a given row are normal.',
# Each row represents a unique lab panel order.
'Each row represents a unique lab panel order.',
# Each row contains fields summarizing the patient's demographics,
"Each row contains fields summarizing the patient's demographics",
# inpatient admit date, prior vitals, and prior lab results.
'inpatient admit date, prior vitals, and prior lab results.',
# Most cells in matrix represent a count statistic for an event's
"Most cells in matrix represent a count statistic for an event's",
# occurrence or a difference between an event's time and index_time.
"occurrence or a difference between an event's time and index_time.",
]
# Bundle parameters into single object to be unpacked in SLP.
params['raw_matrix_path'] = raw_matrix_path
params['processed_matrix_path'] = processed_matrix_path
params['features_to_add'] = features_to_add
params['features_to_keep'] = features_to_keep
params['imputation_strategies'] = imputation_strategies
params['features_to_remove'] = features_to_remove
params['outcome_label'] = outcome_label
params['selection_problem'] = selection_problem
params['selection_algorithm'] = selection_algorithm
params['percent_features_to_select'] = percent_features_to_select
params['matrix_class'] = matrix_class
params['pipeline_file_path'] = pipeline_file_path
params['data_overview'] = data_overview
params['random_state'] = random_state
# Defer processing logic to SupervisedLearningPipeline.
SupervisedLearningPipeline._build_processed_feature_matrix(self, params)
def _train_and_analyze_predictors(self):
log.info('Training and analyzing predictors...')
problem = SupervisedLearningPipeline.CLASSIFICATION
meta_report = None
fm_io = FeatureMatrixIO()
# Build paths for output.
pipeline_file_name = inspect.getfile(inspect.currentframe())
data_dir = SupervisedLearningPipeline._fetch_data_dir_path(self, pipeline_file_name)
# Test BifurcatedSupervisedClassifier and SupervisedClassifier.
algorithms_to_test = list()
algorithms_to_test.extend(SupervisedClassifier.SUPPORTED_ALGORITHMS)
# for algorithm in SupervisedClassifier.SUPPORTED_ALGORITHMS:
# algorithms_to_test.append('bifurcated-%s' % algorithm)
log.debug('algorithms_to_test: %s' % algorithms_to_test)
# Train and analyse algorithms.
for algorithm in algorithms_to_test:
log.info('Training and analyzing %s...' % algorithm)
# If report_dir does not exist, make it.
report_dir = '/'.join([data_dir, algorithm])
if not os.path.exists(report_dir):
os.makedirs(report_dir)
# Define hyperparams.
hyperparams = {}
hyperparams['algorithm'] = algorithm
hyperparams['hyperparam_strategy'] = SupervisedClassifier.EXHAUSTIVE_SEARCH
hyperparams['max_iter'] = 1024
hyperparams['random_state'] = self._random_state
# If bifurcated algorithm, define bifurcator.
if 'bifurcated' in algorithm:
# bifrucator = LAB.pre == 0
hyperparams['bifurcator'] = '%s.pre' % self._var
hyperparams['bifurcation_strategy'] = BifurcatedSupervisedClassifier.EQUAL
hyperparams['bifurcation_value'] = 0
hyperparams['bifurcated'] = True
# Train classifier.
predictor_path = self._build_model_dump_path(algorithm)
if os.path.exists(predictor_path) and 'bifurcated' not in algorithm:
log.debug('Loading model from disk...')
# TODO(sbala): Fix loblib.load so that it works for bifurcated
# supervised classifiers.
self._predictor = joblib.load(predictor_path)
self._features = self._X_train.columns
status = SupervisedClassifier.TRAINED
else:
status = SupervisedLearningPipeline._train_predictor(self, problem, [0, 1], hyperparams)
# If failed to train, write an error report.
y_train_counts = self._y_train[self._y_train.columns[0]].value_counts()
y_test_counts = self._y_test[self._y_test.columns[0]].value_counts()
if status == SupervisedClassifier.INSUFFICIENT_SAMPLES:
# Skip all analysis and reporting.
# This will be true for all algorithms, so just return.
# Build error report.
algorithm_report = DataFrame(
{
'lab_panel': [self._var],
'algorithm': [algorithm],
'error': [status],
'y_train.value_counts()': [y_train_counts.to_dict()],
'y_test.value_counts()': [y_test_counts.to_dict()]
},
columns=[
'lab_panel', 'algorithm', 'error',
'y_train.value_counts()', 'y_test.value_counts()'
]
)
header = ['LabNormalityPredictionPipeline("%s", 10000)' % self._var]
# Write error report.
fm_io.write_data_frame_to_file(algorithm_report, \
'/'.join([report_dir, '%s-normality-prediction-report.tab' % (self._var)]), \
header)
# If successfully trained, append to a meta report.
elif status == SupervisedClassifier.TRAINED:
pipeline_prefix = '%s-normality-prediction-%s' % (self._var, algorithm)
SupervisedLearningPipeline._analyze_predictor(self, report_dir, pipeline_prefix)
SupervisedLearningPipeline._analyze_predictor_traindata(self, report_dir, pipeline_prefix)
if meta_report is None:
meta_report = fm_io.read_file_to_data_frame('/'.join([report_dir, '%s-report.tab' % pipeline_prefix]))
else:
algorithm_report = fm_io.read_file_to_data_frame('/'.join([report_dir, '%s-report.tab' % pipeline_prefix]))
log.debug('algorithm_report: %s' % algorithm_report)
meta_report = meta_report.append(algorithm_report)
# Write predictor to disk.
predictor = SupervisedLearningPipeline.predictor(self)
predictor_path = self._build_model_dump_path(algorithm)
joblib.dump(predictor, predictor_path)
# After building per-algorithm reports, write to meta report.
# Note that if there were insufficient samples to build any of the
# algorithms, then meta_report will still be None.
if meta_report is not None:
header = ['LabNormalityPredictionPipeline("%s", 10000)' % self._var]
fm_io.write_data_frame_to_file(meta_report, \
'/'.join([data_dir, '%s-normality-prediction-report.tab' % self._var]), header)