def main(args):
try: method = args[1].lower()
except:
method = 'lrfc'
eprint('Using default method = \'{}\''.format(method))
try: data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try: which_half = args[3].lower()
except:
which_half = 'both'
eprint('Using default which_half = {}'.format(which_half))
try: prop_missing = float(args[4])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
# not going to finish in time, so skip nonlinear svm if using full dataset
skip_nonlinear_svm = 'final-100.txt' in data_fn
if skip_nonlinear_svm:
eprint('Skipping SVMs with non-linear kernels')
run(data_fn, method=method, which_half=which_half,
prop_missing=prop_missing, skip_nonlinear_svm=skip_nonlinear_svm)
def main(args):
try:
method = args[1].lower()
except:
method = 'lrfc'
eprint('Using default method = \'{}\''.format(method))
try:
data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try:
which_half = args[3].lower()
except:
which_half = 'both'
eprint('Using default which_half = {}'.format(which_half))
try:
prop_missing = float(args[4])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
# not going to finish in time, so skip nonlinear svm if using full dataset
skip_nonlinear_svm = 'final-100.txt' in data_fn
if skip_nonlinear_svm:
eprint('Skipping SVMs with non-linear kernels')
run(data_fn,
method=method,
which_half=which_half,
prop_missing=prop_missing,
skip_nonlinear_svm=skip_nonlinear_svm)
def get_base_data(data_path, prop_missing):
icd9_descript_path = '{}/{}'.format(DATA_DIR, 'phewas_codes.txt')
# load data
print('Loading data...')
start = time.time()
# get common data
icd9_descript_dict = emr.get_icd9_descript_dict(icd9_descript_path)
X, y, idx_feat_dict, idx_class_dict = emr.get_data(
path=data_path,
icd9_descript_dict=icd9_descript_dict,
prop_missing=prop_missing)
nb_features = len(idx_feat_dict)
nb_classes = len(idx_class_dict)
nb_cases = len(X)
print('Data loaded in {:.5f} s'.format(time.time() - start))
print()
# shuffle indices
perm_indices = np.random.permutation(nb_cases)
try: # try validating shuffled indices
with open(data_path + '_perm_indices.pkl', 'r') as f:
exp_perm_indices = pickle.load(f)
assert np.all(perm_indices == exp_perm_indices)
except:
eprint('file not found ' + data_path + '_perm_indices.pkl')
eprint('not doing perm_indices check')
return X, y, perm_indices, nb_features, nb_classes
def get_base_data(data_path, prop_missing):
icd9_descript_path = '{}/{}'.format(DATA_DIR, 'phewas_codes.txt')
# load data
print('Loading data...')
start = time.time()
# get common data
icd9_descript_dict = emr.get_icd9_descript_dict(icd9_descript_path)
X, y, idx_feat_dict, idx_class_dict = emr.get_data(path=data_path,
icd9_descript_dict=icd9_descript_dict, prop_missing=prop_missing)
nb_features = len(idx_feat_dict)
nb_classes = len(idx_class_dict)
nb_cases = len(X)
print('Data loaded in {:.5f} s'.format(time.time() - start))
print()
# shuffle indices
perm_indices = np.random.permutation(nb_cases)
try: # try validating shuffled indices
with open(data_path + '_perm_indices.pkl', 'r') as f:
exp_perm_indices = pickle.load(f)
assert np.all(perm_indices == exp_perm_indices)
except:
eprint('file not found ' + data_path + '_perm_indices.pkl')
eprint('not doing perm_indices check')
return X, y, perm_indices, nb_features, nb_classes
def main(args):
try:
method = args[1].lower()
except:
method = 'lrfc'
eprint('Using default method = \'{}\''.format(method))
try:
data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try:
prop_missing = float(args[3])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
# not going to finish in time, so skip nonlinear svm if using full dataset
skip_nonlinear_svm = 'final-100.txt' in data_fn
run(data_fn,
method=method,
prop_missing=prop_missing,
skip_nonlinear_svm=skip_nonlinear_svm)
def main(args):
try: method = args[1].lower()
except:
method = 'lrfc'
eprint('Using default method = \'{}\''.format(method))
try: data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try: prop_missing = float(args[3])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
# not going to finish in time, so skip nonlinear svm if using full dataset
skip_nonlinear_svm = 'final-100.txt' in data_fn
run(data_fn, method=method, prop_missing=prop_missing,
skip_nonlinear_svm=skip_nonlinear_svm)
def run(data_fn,
method='lrfc',
which_half='both',
prop_missing=0.0,
k=10,
skip_nonlinear_svm=False,
nb_searches=20):
data_path = '{}/{}'.format(DATA_DIR, data_fn)
def get_results_dir(method, k_idx):
base_folder = 'out/more/{}_{}_{}'.format(method, data_fn, prop_missing)
folder = '{}/{}_idx_partition'.format(base_folder, k_idx)
if not os.path.exists('out'): os.makedirs('out')
if not os.path.exists('out/more'): os.makedirs('out/models')
if not os.path.exists(base_folder): os.makedirs(base_folder)
if not os.path.exists(folder): os.makedirs(folder)
return folder
try: # load saved parameters
get_param_fn = lambda x: '{}/{}_{}_{}_param.pkl'.format(
CACHE_DIR, x, data_fn, prop_missing)
if method == 'lrfc':
with open(get_param_fn('logit'), 'r') as f:
logit_params = pickle.load(f)
with open(get_param_fn('rfc'), 'r') as f:
rfc_params = pickle.load(f)
elif method == 'svm':
with open(get_param_fn('linear-svm'), 'r') as f:
linear_svm_params = pickle.load(f)
if not skip_nonlinear_svm:
with open(get_param_fn('poly-svm'), 'r') as f:
poly_svm_params = pickle.load(f)
with open(get_param_fn('rbf-svm'), 'r') as f:
rbf_svm_params = pickle.load(f)
else:
raise ValueError('unknown method: {}'.format(method))
except:
eprint('Need to do parameter search!')
eprint('Please run `parameter_search.py` with the relevant' +
'command line arguments')
raise
X, y, perm_indices, nb_features, nb_classes = get_base_data(
data_path, prop_missing)
losses = {
'logit': [],
'rfc': [],
'linear-svm': [],
'poly-svm': [],
'rbf-svm': []
}
accs = {
'logit': [],
'rfc': [],
'linear-svm': [],
'poly-svm': [],
'rbf-svm': []
}
runtimes = {
'logit': [],
'rfc': [],
'linear-svm': [],
'poly-svm': [],
'rbf-svm': []
}
if which_half == 'first': loop_seq = range(0, k / 2)
elif which_half == 'last': loop_seq = range(k / 2, k)
elif which_half == 'both': loop_seq = range(0, k)
else:
raise ValueError(
'`which_half` must be \'first\', \'last\' or \'both\'')
for k_idx in loop_seq:
print('-' * 72)
print('Partition k = {}'.format(k_idx))
data_partition_dict = emr.get_k_fold_partition(
X, y, k_idx=k_idx, k=k, perm_indices=perm_indices)
X_train = data_partition_dict['X_train']
y_train = data_partition_dict['y_train']
X_val = data_partition_dict['X_val']
y_val = data_partition_dict['y_val']
X_test = data_partition_dict['X_test']
y_test = data_partition_dict['y_test']
selected_feat_indices = select_feats(X_train + X_val,
y_train + y_val,
nb_features=nb_features)
X_train, y_train = preproc_for_sklearn(X_train, y_train, nb_features)
X_test, y_test = preproc_for_sklearn(X_test, y_test, nb_features)
old_nb_features = len(X_train[0])
X_train = X_train[:, selected_feat_indices]
X_test = X_test[:, selected_feat_indices]
nb_features = len(X_train[0]) # extraneous but for future utility
print('Reduced features from {} to {}'.format(old_nb_features,
nb_features))
if method == 'lrfc':
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
# logistic regression
start = time.time()
logit = LogisticRegression(multi_class='multinomial',
solver='lbfgs',
**logit_params[k_idx])
logit.fit(X_train, y_train)
logit_acc = accuracy_score(y_test, logit.predict(X_test))
logit_y_test_proba = logit.predict_proba(X_test)
logit_loss = log_loss(y_test, logit_y_test_proba)
logit_time = time.time() - start
print(
'Logistic regression / loss: {:.3f} / accuracy: {:.3f} / time: {:.3f} s'
.format(logit_loss, logit_acc, logit_time))
# random forest classifier
start = time.time()
rfc = RandomForestClassifier(**rfc_params[k_idx])
rfc.fit(X_train, y_train)
rfc_acc = accuracy_score(y_test, rfc.predict(X_test))
rfc_y_test_proba = rfc.predict_proba(X_test)
rfc_loss = log_loss(y_test, rfc_y_test_proba)
rfc_time = time.time() - start
print(
'Random forest / loss: {:.3f} / accuracy: {:.3f} / time: {:.3f} s'
.format(rfc_loss, rfc_acc, rfc_time))
save_test_results(
logit_y_test_proba, y_test,
'{}/test_results.txt'.format(get_results_dir('logit', k_idx)))
save_test_results(
rfc_y_test_proba, y_test,
'{}/test_results.txt'.format(get_results_dir('rfc', k_idx)))
# joblib.dump(logit, get_results_dir('logit', k_idx) + '/clf.pkl')
# joblib.dump(rfc, get_results_dir('rfc', k_idx) + '/clf.pkl')
losses['logit'].append(logit_loss)
accs['logit'].append(logit_acc)
runtimes['logit'].append(logit_time)
losses['rfc'].append(rfc_loss)
accs['rfc'].append(rfc_acc)
runtimes['rfc'].append(rfc_time)
elif method == 'svm':
from sklearn.svm import SVC
# linear SVM
start = time.time()
linear_svm = SVC(kernel='linear',
probability=True,
**linear_svm_params[k_idx])
linear_svm.fit(X_train, y_train)
linear_svm_acc = accuracy_score(y_test, linear_svm.predict(X_test))
linear_svm_y_test_proba = linear_svm.predict_proba(X_test)
linear_svm_loss = log_loss(y_test, linear_svm_y_test_proba)
linear_svm_time = time.time() - start
print(
'Linear SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'
.format(linear_svm_acc, linear_svm_loss, linear_svm_time))
save_test_results(
linear_svm_y_test_proba, y_test, '{}/test_results.txt'.format(
get_results_dir('linear-svm', k_idx)))
# joblib.dump(linear_svm, get_results_dir('linear-svm', k_idx) + '/clf.pkl')
losses['linear-svm'].append(linear_svm_loss)
accs['linear-svm'].append(linear_svm_acc)
runtimes['linear-svm'].append(linear_svm_time)
if skip_nonlinear_svm: continue # skip
# polynomial SVM
start = time.time()
poly_svm = SVC(kernel='poly',
probability=True,
**poly_svm_params[k_idx])
poly_svm.fit(X_train, y_train)
poly_svm_acc = accuracy_score(y_test, poly_svm.predict(X_test))
poly_svm_y_test_proba = poly_svm.predict_proba(X_test)
poly_svm_loss = log_loss(y_test, poly_svm_y_test_proba)
poly_svm_time = time.time() - start
print(
'Polynomial SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'
.format(poly_svm_acc, poly_svm_loss, poly_svm_time))
# RBF SVM
start = time.time()
rbf_svm = SVC(kernel='rbf',
probability=True,
**rbf_svm_params[k_idx])
rbf_svm.fit(X_train, y_train)
rbf_svm_acc = accuracy_score(y_test, rbf_svm.predict(X_test))
rbf_svm_y_test_proba = rbf_svm.predict_proba(X_test)
rbf_svm_loss = log_loss(y_test, rbf_svm_y_test_proba)
rbf_svm_time = time.time() - start
print('RBF SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'.
format(rbf_svm_acc, rbf_svm_loss, rbf_svm_time))
save_test_results(
poly_svm_y_test_proba, y_test, '{}/test_results.txt'.format(
get_results_dir('poly-svm', k_idx)))
save_test_results(
rbf_svm_y_test_proba, y_test,
'{}/test_results.txt'.format(get_results_dir('rbf-svm',
k_idx)))
# joblib.dump(poly_svm, get_results_dir('poly-svm', k_idx) + '/clf.pkl')
# joblib.dump(rbf_svm, get_results_dir('rbf-svm', k_idx) + '/clf.pkl')
losses['poly-svm'].append(poly_svm_loss)
accs['poly-svm'].append(poly_svm_acc)
runtimes['poly-svm'].append(poly_svm_time)
losses['rbf-svm'].append(rbf_svm_loss)
accs['rbf-svm'].append(rbf_svm_acc)
runtimes['rbf-svm'].append(rbf_svm_time)
else:
raise ValueError('unknown method: {}'.format(method))
print()
print('#' * 72)
if method == 'lrfc':
print_metrics(losses['logit'], accs['logit'], runtimes['logit'],
'Logistic regression')
print_metrics(losses['rfc'], accs['rfc'], runtimes['rfc'],
'Random forest')
elif method == 'svm':
print_metrics(losses['linear-svm'], accs['linear-svm'],
runtimes['linear-svm'], 'Linear SVM')
if not skip_nonlinear_svm:
print_metrics(losses['poly-svm'], accs['poly-svm'],
runtimes['poly-svm'], 'Polynomial SVM')
print_metrics(losses['rbf-svm'], accs['rbf-svm'],
runtimes['rbf-svm'], 'RBF SVM')
else:
raise ValueError('unknown method: {}'.format(method))
print('#' * 72)
def main(args):
k = 10 # ten partitions for k-fold cross-validation
try: id_string = args[1]
except:
id_string = 'dummy'
eprint('Using default id_string = \'{}\''.format(id_string))
try: data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try: interpret_model = args[3].lower() == 'true' or args[3].lower()[0] == 't'
except:
interpret_model = True
eprint('Using default interpret_model = {}'.format(interpret_model))
try: prop_missing = float(args[4])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
data_path = '{}/{}'.format(DATA_DIR, data_fn)
icd9_descript_path = '{}/{}'.format(DATA_DIR, 'phewas_codes.txt')
model_module = models.deep_mlp
model_id = model_module.__name__.split('.')[2]
data_name = ''.join(data_fn.split('.')[:-1])
if not os.path.exists('out'): os.makedirs('out')
if not os.path.exists('out/more'): os.makedirs('out/more')
out_directory = 'out/more/{}_{}_{}'.format('riddle', data_fn, prop_missing)
if not os.path.exists(out_directory): os.makedirs(out_directory)
start = time.time()
# get common data
icd9_descript_dict = emr.get_icd9_descript_dict(icd9_descript_path)
X, y, idx_feat_dict, idx_class_dict = emr.get_data(path=data_path,
icd9_descript_dict=icd9_descript_dict, prop_missing=prop_missing)
# print/save value-sorted dictionary of classes and features
class_mapping = sorted(idx_class_dict.items(), key=lambda key: key[0])
print('Class mapping:')
print(class_mapping)
print()
with open(out_directory + '/class_mapping.txt', 'w+') as f:
print(class_mapping, file=f)
with open(out_directory + '/feature_mapping.txt', 'w+') as f:
for idx, feat in idx_feat_dict.items():
f.write('{}\t{}\n'.format(idx, feat))
nb_features = len(idx_feat_dict)
nb_classes = len(idx_class_dict)
nb_cases = len(X)
print('Data loaded in {:.5f} seconds'.format(time.time() - start))
# shuffle indices and save them
perm_indices = np.random.permutation(nb_cases)
pickle_object(perm_indices, out_directory + '/perm_indices.pkl')
try: # try validating shuffled indices
with open(data_path + '_perm_indices.pkl', 'r') as f:
exp_perm_indices = pickle.load(f)
assert np.all(perm_indices == exp_perm_indices)
except:
eprint('file not found ' + data_path + '_perm_indices.pkl')
eprint('not doing perm_indices check')
# load saved model parameters
model_params_fn = '{}/{}_{}_{}_param.pkl'.format(CACHE_DIR, 'riddle',
data_fn, prop_missing)
try:
with open(model_params_fn, 'r') as f:
model_params = pickle.load(f)
except:
eprint('Need to do parameter search!')
eprint('Please run `parameter_search.py` with the relevant' +
'command line arguments')
raise
# run pipeline and get metric results
(losses, accs, runtimes), (list_contrib_sums_D,
list_contrib_sums_D2, list_contrib_sums), pairs = \
kfold_run_pipeline(model_module,
model_params, X, y, nb_features=nb_features, nb_classes=nb_classes,
k=k, perm_indices=perm_indices, interpret_model=interpret_model,
out_directory=out_directory, id_string=id_string)
if interpret_model:
total_contrib_sums_D = compute_total_sums(list_contrib_sums_D)
total_contrib_sums_D2 = compute_total_sums(list_contrib_sums_D2)
total_contrib_sums = compute_total_sums(list_contrib_sums)
nb_pairs = len(pairs)
# get descriptions of feature importance
feat_importance_summary = feature_importance.summarize_feature_importance(
total_contrib_sums_D, total_contrib_sums_D2, idx_feat_dict=idx_feat_dict,
idx_class_dict=idx_class_dict, icd9_descript_dict=icd9_descript_dict,
pairs=pairs, nb_cases=nb_cases)
# get frequencies of features per class
feat_class_freq_table = frequency.get_frequency_table(X, y,
idx_feat_dict=idx_feat_dict, idx_class_dict=idx_class_dict)
# get orderings
ordering_summary = ordering.summarize_orderings(total_contrib_sums,
feat_class_freq_table, idx_feat_dict=idx_feat_dict,
idx_class_dict=idx_class_dict, icd9_descript_dict=icd9_descript_dict,
nb_pairs=nb_pairs)
ordering_summary.save_individual_tables(idx_class_dict, out_directory)
ordering_summary.save(out_directory)
# print metrics in a pretty fashion
print_metrics(losses, accs, runtimes, id_string=id_string)
print('This k-fold multipipeline run script took {:.4f} seconds'
.format(time.time() - start))
def run(data_fn, method='lrfc', which_half='both', prop_missing=0.0, k=10,
skip_nonlinear_svm=False, nb_searches=20):
data_path = '{}/{}'.format(DATA_DIR, data_fn)
def get_results_dir(method, k_idx):
base_folder = 'out/more/{}_{}_{}'.format(method, data_fn, prop_missing)
folder = '{}/{}_idx_partition'.format(base_folder, k_idx)
if not os.path.exists('out'): os.makedirs('out')
if not os.path.exists('out/more'): os.makedirs('out/models')
if not os.path.exists(base_folder): os.makedirs(base_folder)
if not os.path.exists(folder): os.makedirs(folder)
return folder
try: # load saved parameters
get_param_fn = lambda x: '{}/{}_{}_{}_param.pkl'.format(CACHE_DIR,
x, data_fn, prop_missing)
if method == 'lrfc':
with open(get_param_fn('logit'), 'r') as f:
logit_params = pickle.load(f)
with open(get_param_fn('rfc'), 'r') as f:
rfc_params = pickle.load(f)
elif method == 'svm':
with open(get_param_fn('linear-svm'), 'r') as f:
linear_svm_params = pickle.load(f)
if not skip_nonlinear_svm:
with open(get_param_fn('poly-svm'), 'r') as f:
poly_svm_params = pickle.load(f)
with open(get_param_fn('rbf-svm'), 'r') as f:
rbf_svm_params = pickle.load(f)
else: raise ValueError('unknown method: {}'.format(method))
except:
eprint('Need to do parameter search!')
eprint('Please run `parameter_search.py` with the relevant' +
'command line arguments')
raise
X, y, perm_indices, nb_features, nb_classes = get_base_data(data_path,
prop_missing)
losses = {'logit':[], 'rfc':[], 'linear-svm':[], 'poly-svm':[], 'rbf-svm':[]}
accs = {'logit':[], 'rfc':[], 'linear-svm':[], 'poly-svm':[], 'rbf-svm':[]}
runtimes = {'logit':[], 'rfc':[], 'linear-svm':[], 'poly-svm':[], 'rbf-svm':[]}
if which_half == 'first': loop_seq = range(0, k / 2)
elif which_half == 'last': loop_seq = range(k / 2, k)
elif which_half == 'both': loop_seq = range(0, k)
else: raise ValueError('`which_half` must be \'first\', \'last\' or \'both\'')
for k_idx in loop_seq:
print('-' * 72)
print('Partition k = {}'.format(k_idx))
data_partition_dict = emr.get_k_fold_partition(X, y, k_idx=k_idx, k=k,
perm_indices=perm_indices)
X_train = data_partition_dict['X_train']
y_train = data_partition_dict['y_train']
X_val = data_partition_dict['X_val']
y_val = data_partition_dict['y_val']
X_test = data_partition_dict['X_test']
y_test = data_partition_dict['y_test']
selected_feat_indices = select_feats(X_train + X_val, y_train + y_val,
nb_features=nb_features)
X_train, y_train = preproc_for_sklearn(X_train, y_train, nb_features)
X_test, y_test = preproc_for_sklearn(X_test, y_test, nb_features)
old_nb_features = len(X_train[0])
X_train = X_train[:, selected_feat_indices]
X_test = X_test[:, selected_feat_indices]
nb_features = len(X_train[0]) # extraneous but for future utility
print('Reduced features from {} to {}'.format(old_nb_features, nb_features))
if method == 'lrfc':
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
# logistic regression
start = time.time()
logit = LogisticRegression(multi_class='multinomial', solver='lbfgs',
**logit_params[k_idx])
logit.fit(X_train, y_train)
logit_acc = accuracy_score(y_test, logit.predict(X_test))
logit_y_test_probas = logit.predict_proba(X_test)
logit_loss = log_loss(y_test, logit_y_test_probas)
logit_time = time.time() - start
print('Logistic regression / loss: {:.3f} / accuracy: {:.3f} / time: {:.3f} s'
.format(logit_loss, logit_acc, logit_time))
# random forest classifier
start = time.time()
rfc = RandomForestClassifier(**rfc_params[k_idx])
rfc.fit(X_train, y_train)
rfc_acc = accuracy_score(y_test, rfc.predict(X_test))
rfc_y_test_probas = rfc.predict_proba(X_test)
rfc_loss = log_loss(y_test, rfc_y_test_probas)
rfc_time = time.time() - start
print('Random forest / loss: {:.3f} / accuracy: {:.3f} / time: {:.3f} s'
.format(rfc_loss, rfc_acc, rfc_time))
save_test_results(logit_y_test_probas, y_test,
'{}/test_results.txt'.format(get_results_dir('logit', k_idx)))
save_test_results(rfc_y_test_probas, y_test,
'{}/test_results.txt'.format(get_results_dir('rfc', k_idx)))
# joblib.dump(logit, get_results_dir('logit', k_idx) + '/clf.pkl')
# joblib.dump(rfc, get_results_dir('rfc', k_idx) + '/clf.pkl')
losses['logit'].append(logit_loss)
accs['logit'].append(logit_acc)
runtimes['logit'].append(logit_time)
losses['rfc'].append(rfc_loss)
accs['rfc'].append(rfc_acc)
runtimes['rfc'].append(rfc_time)
elif method == 'svm':
from sklearn.svm import SVC
# linear SVM
start = time.time()
linear_svm = SVC(kernel='linear', probability=True,
**linear_svm_params[k_idx])
linear_svm.fit(X_train, y_train)
linear_svm_acc = accuracy_score(y_test, linear_svm.predict(X_test))
linear_svm_y_test_probas = linear_svm.predict_proba(X_test)
linear_svm_loss = log_loss(y_test, linear_svm_y_test_probas)
linear_svm_time = time.time() - start
print('Linear SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'
.format(linear_svm_acc, linear_svm_loss, linear_svm_time))
save_test_results(linear_svm_y_test_probas, y_test,
'{}/test_results.txt'.format(get_results_dir('linear-svm', k_idx)))
# joblib.dump(linear_svm, get_results_dir('linear-svm', k_idx) + '/clf.pkl')
losses['linear-svm'].append(linear_svm_loss)
accs['linear-svm'].append(linear_svm_acc)
runtimes['linear-svm'].append(linear_svm_time)
if skip_nonlinear_svm: continue # skip
# polynomial SVM
start = time.time()
poly_svm = SVC(kernel='poly', probability=True,
**poly_svm_params[k_idx])
poly_svm.fit(X_train, y_train)
poly_svm_acc = accuracy_score(y_test, poly_svm.predict(X_test))
poly_svm_y_test_probas = poly_svm.predict_proba(X_test)
poly_svm_loss = log_loss(y_test, poly_svm_y_test_probas)
poly_svm_time = time.time() - start
print('Polynomial SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'
.format(poly_svm_acc, poly_svm_loss, poly_svm_time))
# RBF SVM
start = time.time()
rbf_svm = SVC(kernel='rbf', probability=True,
**rbf_svm_params[k_idx])
rbf_svm.fit(X_train, y_train)
rbf_svm_acc = accuracy_score(y_test, rbf_svm.predict(X_test))
rbf_svm_y_test_probas = rbf_svm.predict_proba(X_test)
rbf_svm_loss = log_loss(y_test, rbf_svm_y_test_probas)
rbf_svm_time = time.time() - start
print('RBF SVM / accuracy: {:.3f} / loss: {:.3f} / time: {:.3f} s'
.format(rbf_svm_acc, rbf_svm_loss, rbf_svm_time))
save_test_results(poly_svm_y_test_probas, y_test,
'{}/test_results.txt'.format(get_results_dir('poly-svm', k_idx)))
save_test_results(rbf_svm_y_test_probas, y_test,
'{}/test_results.txt'.format(get_results_dir('rbf-svm', k_idx)))
# joblib.dump(poly_svm, get_results_dir('poly-svm', k_idx) + '/clf.pkl')
# joblib.dump(rbf_svm, get_results_dir('rbf-svm', k_idx) + '/clf.pkl')
losses['poly-svm'].append(poly_svm_loss)
accs['poly-svm'].append(poly_svm_acc)
runtimes['poly-svm'].append(poly_svm_time)
losses['rbf-svm'].append(rbf_svm_loss)
accs['rbf-svm'].append(rbf_svm_acc)
runtimes['rbf-svm'].append(rbf_svm_time)
else: raise ValueError('unknown method: {}'.format(method))
print()
print('#' * 72)
if method == 'lrfc':
print_metrics(losses['logit'], accs['logit'], runtimes['logit'],
'Logistic regression')
print_metrics(losses['rfc'], accs['rfc'], runtimes['rfc'],
'Random forest')
elif method == 'svm':
print_metrics(losses['linear-svm'], accs['linear-svm'],
runtimes['linear-svm'], 'Linear SVM')
if not skip_nonlinear_svm:
print_metrics(losses['poly-svm'], accs['poly-svm'],
runtimes['poly-svm'], 'Polynomial SVM')
print_metrics(losses['rbf-svm'], accs['rbf-svm'],
runtimes['rbf-svm'], 'RBF SVM')
else: raise ValueError('unknown method: {}'.format(method))
print('#' * 72)
def main(args):
k = 10 # ten partitions for k-fold cross-validation
try:
id_string = args[1]
except:
id_string = 'dummy'
eprint('Using default id_string = \'{}\''.format(id_string))
try:
data_fn = args[2]
except:
data_fn = 'dummy.txt'
eprint('Using default data_fn = \'{}\''.format(data_fn))
try:
interpret_model = args[3].lower() == 'true' or args[3].lower(
)[0] == 't'
except:
interpret_model = True
eprint('Using default interpret_model = {}'.format(interpret_model))
try:
prop_missing = float(args[4])
except:
prop_missing = 0.0
eprint('Using default prop_missing = {}'.format(prop_missing))
data_path = '{}/{}'.format(DATA_DIR, data_fn)
icd9_descript_path = '{}/{}'.format(DATA_DIR, 'phewas_codes.txt')
model_module = models.deep_mlp
model_id = model_module.__name__.split('.')[2]
data_name = ''.join(data_fn.split('.')[:-1])
if not os.path.exists('out'): os.makedirs('out')
if not os.path.exists('out/more'): os.makedirs('out/more')
out_directory = 'out/more/{}_{}_{}'.format('riddle', data_fn, prop_missing)
if not os.path.exists(out_directory): os.makedirs(out_directory)
start = time.time()
# get common data
icd9_descript_dict = emr.get_icd9_descript_dict(icd9_descript_path)
X, y, idx_feat_dict, idx_class_dict = emr.get_data(
path=data_path,
icd9_descript_dict=icd9_descript_dict,
prop_missing=prop_missing)
# print/save value-sorted dictionary of classes and features
class_mapping = sorted(idx_class_dict.items(), key=lambda key: key[0])
print('Class mapping:')
print(class_mapping)
print()
with open(out_directory + '/class_mapping.txt', 'w+') as f:
print(class_mapping, file=f)
with open(out_directory + '/feature_mapping.txt', 'w+') as f:
for idx, feat in idx_feat_dict.items():
f.write('{}\t{}\n'.format(idx, feat))
nb_features = len(idx_feat_dict)
nb_classes = len(idx_class_dict)
nb_cases = len(X)
print('Data loaded in {:.5f} seconds'.format(time.time() - start))
# shuffle indices and save them
perm_indices = np.random.permutation(nb_cases)
pickle_object(perm_indices, out_directory + '/perm_indices.pkl')
try: # try validating shuffled indices
with open(data_path + '_perm_indices.pkl', 'r') as f:
exp_perm_indices = pickle.load(f)
assert np.all(perm_indices == exp_perm_indices)
except:
eprint('file not found ' + data_path + '_perm_indices.pkl')
eprint('not doing perm_indices check')
# load saved model parameters
model_params_fn = '{}/{}_{}_{}_param.pkl'.format(CACHE_DIR, 'riddle',
data_fn, prop_missing)
try:
with open(model_params_fn, 'r') as f:
model_params = pickle.load(f)
except:
eprint('Need to do parameter search!')
eprint('Please run `parameter_search.py` with the relevant' +
'command line arguments')
raise
# run pipeline and get metric results
(losses, accs, runtimes), (list_contrib_sums_D,
list_contrib_sums_D2, list_contrib_sums), pairs = \
kfold_run_pipeline(model_module,
model_params, X, y, nb_features=nb_features, nb_classes=nb_classes,
k=k, perm_indices=perm_indices, interpret_model=interpret_model,
out_directory=out_directory, id_string=id_string)
if interpret_model:
total_contrib_sums_D = compute_total_sums(list_contrib_sums_D)
total_contrib_sums_D2 = compute_total_sums(list_contrib_sums_D2)
total_contrib_sums = compute_total_sums(list_contrib_sums)
nb_pairs = len(pairs)
# get descriptions of feature importance
feat_importance_summary = feature_importance.summarize_feature_importance(
total_contrib_sums_D,
total_contrib_sums_D2,
idx_feat_dict=idx_feat_dict,
idx_class_dict=idx_class_dict,
icd9_descript_dict=icd9_descript_dict,
pairs=pairs,
nb_cases=nb_cases)
# get frequencies of features per class
feat_class_freq_table = frequency.get_frequency_table(
X, y, idx_feat_dict=idx_feat_dict, idx_class_dict=idx_class_dict)
# get orderings
ordering_summary = ordering.summarize_orderings(
total_contrib_sums,
feat_class_freq_table,
idx_feat_dict=idx_feat_dict,
idx_class_dict=idx_class_dict,
icd9_descript_dict=icd9_descript_dict,
nb_pairs=nb_pairs)
ordering_summary.save_individual_tables(idx_class_dict, out_directory)
ordering_summary.save(out_directory)
# print metrics in a pretty fashion
print_metrics(losses, accs, runtimes, id_string=id_string)
print('This k-fold multipipeline run script took {:.4f} seconds'.format(
time.time() - start))
def run(data_fn,
method='lrfc',
prop_missing=0.0,
k=10,
skip_nonlinear_svm=False,
nb_searches=20,
max_nb_samples=10000):
if 'dummy' in data_fn or 'debug' in data_fn: nb_searches = 3
data_path = '{}/{}'.format(DATA_DIR, data_fn)
if not FORCE_RUN: # check if already did param search, if so, skip
did = lambda x: already_done(x, data_fn, prop_missing) # helper
if method == 'riddle' and did(['riddle']):
eprint('Already did parameter search for riddle')
return
elif method == 'lrfc' and did(['logit', 'rfc']):
eprint('Already did parameter search for lrfc')
return
elif method == 'svm' and did(['linear-svm', 'poly-svm', 'rbf-svm']):
eprint('Already did parameter search for svm')
return
params = {
'riddle': {},
'logit': {},
'rfc': {},
'linear-svm': {},
'poly-svm': {},
'rbf-svm': {}
}
X, y, perm_indices, nb_features, nb_classes = get_base_data(
data_path, prop_missing)
for k_idx in range(0, k):
print('-' * 72)
print('Partition k = {}'.format(k_idx))
data_partition_dict = emr.get_k_fold_partition(
X, y, k_idx=k_idx, k=k, perm_indices=perm_indices)
X_train = data_partition_dict['X_train']
y_train = data_partition_dict['y_train']
X_val = data_partition_dict['X_val']
y_val = data_partition_dict['y_val']
# cap number of validation samples
if max_nb_samples != None and len(X_val) > max_nb_samples:
X_val = X_val[0:max_nb_samples]
y_val = y_val[0:max_nb_samples]
if method != 'riddle':
selected_feat_indices = select_feats(X_train + X_val,
y_train + y_val,
nb_features=nb_features)
X_val, y_val = preproc_for_sklearn(X_val,
y_val,
nb_features=nb_features)
X_val = X_val[:, selected_feat_indices]
if method == 'riddle':
start = time.time()
model_module = models.deep_mlp
riddle_param_dist = {
'learning_rate': UniformLogSpace(10, lo=-6, hi=-1)
}
params['riddle'][k_idx] = parameter_tuning.random_search(
model_module,
riddle_param_dist,
X_val,
y_val,
nb_features=nb_features,
nb_classes=nb_classes,
k=3,
process_X_data_func_args={'nb_features': nb_features},
process_y_data_func_args={'nb_classes': nb_classes},
nb_searches=nb_searches)
print('Best parameters for RIDDLE: {} found in {:.3f} s'.format(
params['riddle'][k_idx],
time.time() - start))
elif method == 'lrfc':
# logistic regression
start = time.time()
logit_param_dist = {'C': UniformLogSpace()}
logit_estimator = LogisticRegression(multi_class='multinomial',
solver='lbfgs')
params['logit'][k_idx] = parameter_search(
X_val,
y_val,
estimator=logit_estimator,
search=RandomizedSearchCV,
dist_or_grid=logit_param_dist,
n_iter=nb_searches,
scoring=loss_scorer)
print(
'Best parameters for logistic regression: {} found in {:.3f} s'
.format(params['logit'][k_idx],
time.time() - start))
# random forest classifier
start = time.time()
rfc_param_dist = {'max_features': ['sqrt', 'log2'], \
'max_depth': UniformLogSpace(base=2, lo=2, hi=9)}
rfc_estimator = RandomForestClassifier()
params['rfc'][k_idx] = parameter_search(
X_val,
y_val,
estimator=rfc_estimator,
search=RandomizedSearchCV,
dist_or_grid=rfc_param_dist,
n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for random forest: {} found in {:.3f} s'.
format(params['rfc'][k_idx],
time.time() - start))
elif method == 'svm':
# linear SVM
start = time.time()
linear_svm_param_dist = {'C': UniformLogSpace()}
linear_svm_estimator = SVC(kernel='linear', probability=True)
params['linear-svm'][k_idx] = parameter_search(
X_val,
y_val,
estimator=linear_svm_estimator,
search=RandomizedSearchCV,
dist_or_grid=linear_svm_param_dist,
n_iter=nb_searches,
scoring=loss_scorer)
print(
'Best parameters for linear SVM: {} found in {:.3f} s'.format(
params['linear-svm'][k_idx],
time.time() - start))
if skip_nonlinear_svm: continue # skip
nonlinear_svm_param_dist = {
'C': UniformLogSpace(),
'gamma': UniformLogSpace(base=10, lo=-5, hi=1)
}
# polynomial SVM
start = time.time()
poly_svm_estimator = SVC(kernel='poly', probability=True)
params['poly-svm'][k_idx] = parameter_search(
X_val,
y_val,
estimator=poly_svm_estimator,
search=RandomizedSearchCV,
dist_or_grid=nonlinear_svm_param_dist,
n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for polynomial SVM: {} found in {:.3f} s'.
format(params['poly-svm'][k_idx],
time.time() - start))
# RBF SVM
start = time.time()
rbf_svm_estimator = SVC(kernel='rbf', probability=True)
params['rbf-svm'][k_idx] = parameter_search(
X_val,
y_val,
estimator=rbf_svm_estimator,
search=RandomizedSearchCV,
dist_or_grid=nonlinear_svm_param_dist,
n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for RBF SVM: {} found in {:.3f} s'.format(
params['rbf-svm'][k_idx],
time.time() - start))
else:
raise ValueError('unknown method: {}'.format(method))
# save
for method_name, sub_param_dict in params.items():
if len(sub_param_dict) > 0:
pickle_object(
sub_param_dict,
'{}/{}_{}_{}_param.pkl'.format(CACHE_DIR, method_name, data_fn,
prop_missing))
print('Finished parameter search for method: {}'.format(method))
def run(data_fn, method='lrfc', prop_missing=0.0, k=10,
skip_nonlinear_svm=False, nb_searches=20, max_nb_samples=10000):
if 'dummy' in data_fn or 'debug' in data_fn: nb_searches = 3
data_path = '{}/{}'.format(DATA_DIR, data_fn)
if not FORCE_RUN: # check if already did param search, if so, skip
did = lambda x: already_done(x, data_fn, prop_missing) # helper
if method == 'riddle' and did(['riddle']):
eprint('Already did parameter search for riddle')
return
elif method == 'lrfc' and did(['logit', 'rfc']):
eprint('Already did parameter search for lrfc')
return
elif method == 'svm' and did(['linear-svm', 'poly-svm', 'rbf-svm']):
eprint('Already did parameter search for svm')
return
params = {'riddle': {}, 'logit': {}, 'rfc': {}, 'linear-svm': {},
'poly-svm': {}, 'rbf-svm': {}}
X, y, perm_indices, nb_features, nb_classes = get_base_data(data_path,
prop_missing)
for k_idx in range(0, k):
print('-' * 72)
print('Partition k = {}'.format(k_idx))
data_partition_dict = emr.get_k_fold_partition(X, y, k_idx=k_idx, k=k,
perm_indices=perm_indices)
X_train = data_partition_dict['X_train']
y_train = data_partition_dict['y_train']
X_val = data_partition_dict['X_val']
y_val = data_partition_dict['y_val']
# cap number of validation samples
if max_nb_samples != None and len(X_val)> max_nb_samples:
X_val = X_val[0:max_nb_samples]
y_val = y_val[0:max_nb_samples]
if method != 'riddle':
selected_feat_indices = select_feats(X_train + X_val, y_train + y_val,
nb_features=nb_features)
X_val, y_val = preproc_for_sklearn(X_val, y_val,
nb_features=nb_features)
X_val = X_val[:, selected_feat_indices]
if method == 'riddle':
start = time.time()
model_module = models.deep_mlp
riddle_param_dist = {'learning_rate': UniformLogSpace(10, lo=-6, hi=-1)}
params['riddle'][k_idx] = parameter_tuning.random_search(model_module,
riddle_param_dist, X_val, y_val, nb_features=nb_features,
nb_classes=nb_classes, k=3,
process_X_data_func_args={'nb_features': nb_features},
process_y_data_func_args={'nb_classes': nb_classes},
nb_searches=nb_searches)
print('Best parameters for RIDDLE: {} found in {:.3f} s'
.format(params['riddle'][k_idx], time.time() - start))
elif method == 'lrfc':
# logistic regression
start = time.time()
logit_param_dist = {'C': UniformLogSpace()}
logit_estimator = LogisticRegression(multi_class='multinomial',
solver='lbfgs')
params['logit'][k_idx] = parameter_search(X_val, y_val,
estimator=logit_estimator, search=RandomizedSearchCV,
dist_or_grid=logit_param_dist, n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for logistic regression: {} found in {:.3f} s'
.format(params['logit'][k_idx], time.time() - start))
# random forest classifier
start = time.time()
rfc_param_dist = {'max_features': ['sqrt', 'log2'], \
'max_depth': UniformLogSpace(base=2, lo=2, hi=9)}
rfc_estimator = RandomForestClassifier()
params['rfc'][k_idx] = parameter_search(X_val, y_val,
estimator=rfc_estimator, search=RandomizedSearchCV,
dist_or_grid=rfc_param_dist, n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for random forest: {} found in {:.3f} s'
.format(params['rfc'][k_idx], time.time() - start))
elif method == 'svm':
# linear SVM
start = time.time()
linear_svm_param_dist = {'C': UniformLogSpace()}
linear_svm_estimator = SVC(kernel='linear', probability=True)
params['linear-svm'][k_idx] = parameter_search(X_val, y_val,
estimator=linear_svm_estimator, search=RandomizedSearchCV,
dist_or_grid=linear_svm_param_dist, n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for linear SVM: {} found in {:.3f} s'
.format(params['linear-svm'][k_idx], time.time() - start))
if skip_nonlinear_svm: continue # skip
nonlinear_svm_param_dist = {'C': UniformLogSpace(),
'gamma': UniformLogSpace(base=10, lo=-5, hi=1)}
# polynomial SVM
start = time.time()
poly_svm_estimator = SVC(kernel='poly', probability=True)
params['poly-svm'][k_idx] = parameter_search(X_val, y_val,
estimator=poly_svm_estimator, search=RandomizedSearchCV,
dist_or_grid=nonlinear_svm_param_dist, n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for polynomial SVM: {} found in {:.3f} s'
.format(params['poly-svm'][k_idx], time.time() - start))
# RBF SVM
start = time.time()
rbf_svm_estimator = SVC(kernel='rbf', probability=True)
params['rbf-svm'][k_idx] = parameter_search(X_val, y_val,
estimator=rbf_svm_estimator, search=RandomizedSearchCV,
dist_or_grid=nonlinear_svm_param_dist, n_iter=nb_searches,
scoring=loss_scorer)
print('Best parameters for RBF SVM: {} found in {:.3f} s'
.format(params['rbf-svm'][k_idx], time.time() - start))
else: raise ValueError('unknown method: {}'.format(method))
# save
for method_name, sub_param_dict in params.items():
if len(sub_param_dict) > 0:
pickle_object(sub_param_dict, '{}/{}_{}_{}_param.pkl'.format(
CACHE_DIR, method_name, data_fn, prop_missing))
print('Finished parameter search for method: {}'.format(method))