def getValidationResults(self, results_dict, C, beta, kernel, v,
regularizer):
converged = self.initializeAndTrainMTMKL(self.train_tasks, C, beta,
kernel, v, regularizer)
if self.users_as_tasks:
if not converged:
val_acc = np.nan
val_auc = np.nan
else:
val_acc, val_auc = self.classifier.getAccuracyAucAllTasks(
self.val_tasks)
results_dict['val_acc'] = val_acc
results_dict['val_auc'] = val_auc
else:
accs = []
aucs = []
for t in range(self.n_tasks):
if not converged:
acc = np.nan
auc = np.nan
else:
acc, auc = self.classifier.getAccuracyAucOnOneTask(
self.val_tasks, t)
task_name = self.val_tasks[t]['Name']
results_dict['TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = acc
results_dict['TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = auc
if self.cluster_users or task_name in self.optimize_labels:
accs.append(acc)
aucs.append(auc)
results_dict['val_acc'] = np.mean(accs)
results_dict['val_auc'] = np.mean(aucs)
return results_dict
def run(self): print "\nYou have chosen to test a total of", self.num_settings, "settings for each task" print "There are", self.n_tasks, "tasks, meaning you are training a total of..." print "\t", self.num_settings * self.n_tasks, "neural networks!!" sys.stdout.flush() if self.users_as_tasks: tasks = self.data_df['user_id'].unique() else: tasks = [helper.getFriendlyLabelName(x) for x in self.wanted_labels] i = 0 for t in range(len(tasks)): if self.users_as_tasks: self.runOneTask(tasks[i], self.target_label) else: self.runOneTask(tasks[i], self.wanted_labels[i]) if self.test_run and i > 2: break i += 1 self.val_results_df.to_csv(self.results_path + self.save_prefix + '.csv') if self.users_as_tasks: print "\n\nFINAL RESULTS - Averaging individual models:" print "\tValidation set: Accuracy =", np.nanmean(self.val_results_df['val_acc']), "AUC = ", np.nanmean(self.val_results_df['val_auc']) print "\tTest set: Accuracy =", np.nanmean(self.val_results_df['test_acc']), "AUC = ", np.nanmean(self.val_results_df['test_auc']) print "" print "FINAL RESULTS - Aggregating predictions of individual models" agg_auc = helper.computeAuc(self.cumulative_test_preds, self.cumulative_test_true) agg_acc = helper.getBinaryAccuracy(self.cumulative_test_preds, self.cumulative_test_true) print "\tTest set: Accuracy =", agg_acc, "AUC = ", agg_auc
def getValidationResults(self, results_dict):
self.classifier.trainUntilConverged()
results_dict['num_clusters'] = self.classifier.K
if self.users_as_tasks:
val_acc, val_auc = self.getAccuracyAucOnAllTasks(self.val_tasks)
results_dict['val_acc'] = val_acc
results_dict['val_auc'] = val_auc
else:
accs = []
aucs = []
for t in range(self.n_tasks):
acc, auc = self.getAccuracyAucOnOneTask(self.val_tasks, t)
task_name = self.val_tasks[t]['Name']
results_dict['TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = acc
results_dict['TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = auc
if task_name in self.optimize_labels:
accs.append(acc)
aucs.append(auc)
results_dict['val_acc'] = np.nanmean(accs)
results_dict['val_auc'] = np.nanmean(aucs)
return results_dict
def getSavePrefix(self, file_name, target_label, replace=False):
if '/' in file_name:
slash_loc = file_name.find('/')
file_name = file_name[slash_loc:]
dash_loc = file_name.find('-')
if self.users_as_tasks:
task_name = "tfSTLUsers"
label_name = '-' + helper.getFriendlyLabelName(target_label)
else:
task_name = "tfSTLWellbeing"
label_name = ""
prefix = task_name + file_name[dash_loc:-4] + label_name
if not replace:
while os.path.exists(self.results_path + prefix + '.csv'):
prefix = prefix + '2'
return prefix
def get_preds_for_df(self): X = self.data_df[self.wanted_feats].as_matrix() preds = self.predict(X) assert len(preds) == len(self.data_df) preds_df = copy.deepcopy(self.data_df) for i,wanted_label in enumerate(self.wanted_labels): label_name = helper.getFriendlyLabelName(wanted_label) preds_df['test_pred_'+label_name] = preds test_df = preds_df[preds_df['dataset']=='Test'] test_df = test_df.dropna(subset=[wanted_label], how='any') all_preds = test_df['test_pred_'+label_name].tolist() all_true = test_df[wanted_label].tolist() print "FINAL METRICS ON TEST SET for label", label_name, ":", helper.computeAllMetricsForPreds(all_preds, all_true) print "Predictions have been computed and are stored in dataframe." return preds_df
def getFinalResultsAndSave(self, results_dict):
print "\nRetraining on full training data with the best settings..."
self.drop20 = False
self.initializeAndTrainMTMKL(self.train_tasks,
results_dict['C'],
results_dict['beta'],
results_dict['kernel'],
results_dict['v'],
results_dict['regularizer'],
verbose=True)
print "\nEvaluating results on held-out test set!! ..."
all_preds = []
all_true_y = []
per_task_accs = [np.nan] * self.n_tasks
per_task_aucs = [np.nan] * self.n_tasks
per_task_f1 = [np.nan] * self.n_tasks
per_task_precision = [np.nan] * self.n_tasks
per_task_recall = [np.nan] * self.n_tasks
for t in range(self.n_tasks):
preds = self.classifier.predictOneTask(self.test_tasks, t)
true_y = list(self.test_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
print "no y for task", t, "... skipping"
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t] = t_acc
per_task_aucs[t] = t_auc
per_task_f1[t] = t_f1
per_task_precision[t] = t_precision
per_task_recall[t] = t_recall
print "\nPlotting cool stuff about the final model..."
self.saveImagePlot(self.classifier.eta, 'Etas')
pd.DataFrame(
self.classifier.eta).to_csv(self.etas_path + self.save_prefix +
"-etas.csv")
print "\tHELD OUT TEST METRICS COMPUTED BY APPENDING ALL PREDS"
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
all_preds, all_true_y)
print '\t\tAcc:', acc, 'AUC:', auc, 'F1:', f1, 'Precision:', precision, 'Recall:', recall
print "\n\tHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print '\t\tAcc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
print "\n\tHELD OUT TEST METRICS COMPUTED FOR EACH TASK"
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = self.test_tasks[t]['Name']
task_name = helper.getFriendlyLabelName(task_name)
print "\t\t", task_name, "- Acc:", per_task_accs[
t], "AUC:", per_task_aucs[t], 'F1:', per_task_f1[
t], 'Precision:', per_task_precision[
t], 'Recall:', per_task_recall[t]
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if 'Big5GenderKMeansCluster' in self.file_prefix:
task_column = 'Big5GenderKMeansCluster'
tasks_are_ints = True
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.classifier.predict_01,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=tasks_are_ints)
elif not self.users_as_tasks:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.classifier.predict_01,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
else:
print "Error! Cannot determine what type of model you are training and therefore cannot save predictions."
return
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
def getCrossValidationResults(self,
results_dict,
C,
beta,
kernel,
v,
regularizer,
save_plots=False,
print_per_fold=True):
all_acc = []
all_auc = []
all_f1 = []
all_precision = []
all_recall = []
if not self.users_as_tasks:
per_task_accs = [[] for i in range(self.n_tasks)]
per_task_aucs = [[] for i in range(self.n_tasks)]
per_task_f1 = [[] for i in range(self.n_tasks)]
per_task_precision = [[] for i in range(self.n_tasks)]
per_task_recall = [[] for i in range(self.n_tasks)]
for f in range(self.num_cross_folds):
train_tasks, val_tasks = helper.loadCrossValData(
self.datasets_path,
self.file_prefix,
f,
reshape=False,
fix_y=True)
converged = self.initializeAndTrainMTMKL(train_tasks, C, beta,
kernel, v, regularizer)
if not converged:
all_acc.append(np.nan)
all_auc.append(np.nan)
all_f1.append(np.nan)
all_precision.append(np.nan)
all_recall.append(np.nan)
continue
# Get results!
fold_preds = []
fold_true_y = []
for t in range(self.n_tasks):
preds = self.classifier.predictOneTask(val_tasks, t)
true_y = list(val_tasks[t]['Y'].flatten())
if not self.users_as_tasks:
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t].append(t_acc)
per_task_aucs[t].append(t_auc)
per_task_f1[t].append(t_f1)
per_task_precision[t].append(t_precision)
per_task_recall[t].append(t_recall)
if print_per_fold:
print "Fold", f, "Task", val_tasks[t][
'Name'], "acc", t_acc, "auc", t_auc, "f1", t_f1, "precision", t_precision, "recall", t_recall
fold_preds.extend(preds)
fold_true_y.extend(true_y)
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
fold_preds, fold_true_y)
all_acc.append(acc)
all_auc.append(auc)
all_f1.append(f1)
all_precision.append(precision)
all_recall.append(recall)
if print_per_fold:
print "Fold", f, "acc", acc, "auc", auc, "f1", f1, "precision", precision, "recall", recall
print "accs for all folds", all_acc
print "aucs for all folds", all_auc
# Add results to the dictionary
results_dict['val_acc'] = np.nanmean(all_acc)
results_dict['val_auc'] = np.nanmean(all_auc)
results_dict['val_f1'] = np.nanmean(all_f1)
results_dict['val_precision'] = np.nanmean(all_precision)
results_dict['val_recall'] = np.nanmean(all_recall)
# Add per-task results to the dictionary
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = val_tasks[t]['Name']
results_dict[
'TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_accs[t])
results_dict[
'TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_aucs[t])
results_dict[
'TaskF1-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_f1[t])
results_dict[
'TaskPrecision-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_precision[t])
results_dict[
'TaskRecall-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_recall[t])
return results_dict
def retrainAndPlot(self, setting_dict):
print "\nRETRAINING WITH THE BEST SETTINGS:"
self.net.verbose = True
self.net.setParams(l2_beta=setting_dict['l2_beta'],
initial_learning_rate=setting_dict['learning_rate'],
decay=setting_dict['decay'],
decay_steps=setting_dict['decay_steps'],
decay_rate=setting_dict['decay_rate'],
batch_size=setting_dict['batch_size'],
optimizer=setting_dict['optimizer'],
dropout=setting_dict['dropout'])
self.constructNetwork(setting_dict['hidden_layers'])
self.net.setUpGraph()
self.net.runGraph(self.test_steps, print_test=True)
if self.multilabel:
for label in self.optimize_labels:
friendly_label = helper.getFriendlyLabelName(label)
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '-' +
friendly_label + '.eps',
label=label)
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '-' +
friendly_label + '.png',
label=label)
print "Final validation results for", friendly_label,"... Acc:", \
self.net.training_val_results_per_task['acc'][label][-1], "Auc:", self.net.training_val_results_per_task['auc'][label][-1]
elif self.print_per_task:
for label in self.wanted_labels:
friendly_label = helper.getFriendlyLabelName(label)
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '-' +
friendly_label + '.eps',
label=label)
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '-' +
friendly_label + '.png',
label=label)
print "Final validation results for", friendly_label,"... Acc:", \
self.net.training_val_results_per_task['acc'][label][-1], "Auc:", self.net.training_val_results_per_task['auc'][label][-1]
else:
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '.eps')
self.net.plotValResults(save_path=self.figures_path +
self.val_output_prefix + '.png')
print "Final AUC:", self.net.training_val_results['auc'][-1]
if self.test_csv_filename is not None:
if self.multitask:
task_column = None
if 'Cluster' in self.dataset_name:
print "Guessing the task column is Big5GenderKMeansCluster - if this is incorrect expect errors"
task_column = 'Big5GenderKMeansCluster'
tasks_are_ints = True
if 'User' in self.dataset_name:
print "Guessing the task column is user_id - if this is incorrect expect errors"
task_column = 'user_id'
tasks_are_ints = False
if task_column is not None:
label_name = helper.getFriendlyLabelName(self.dataset_name)
wanted_label = helper.getOfficialLabelName(label_name)
test_preds_df = helper.get_test_predictions_for_df_with_task_column(
self.net.predict,
self.test_csv_filename,
task_column,
self.net.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(
self.net.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=tasks_are_ints)
else:
test_preds_df = helper.get_test_predictions_for_df_with_no_task_column(
self.net.predict,
self.test_csv_filename,
self.net.test_tasks,
num_feats_expected=np.shape(
self.net.test_tasks[0]['X'])[1])
else:
test_preds_df = self.net.get_preds_for_df()
print "Got a test preds df! Saving it to:", self.results_path + "Preds-" + self.val_output_prefix + '.csv'
test_preds_df.to_csv(self.results_path + 'Preds-' +
self.val_output_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
print "Saving a copy of the final model!"
self.net.save_model(self.val_output_prefix, self.results_path)
def __init__(self,
dataset_name,
target_label=None,
trial_name=None,
multilabel=False,
multitask=False,
print_per_task=False,
test_steps=9001,
results_path=DEFAULT_RESULTS_PATH,
datasets_path=DEFAULT_DATASETS_PATH,
figures_path=DEFAULT_FIGURES_PATH,
val_output_file=None,
val_type=DEFAULT_VAL_TYPE,
cont=False,
architectures=None,
test_csv_filename=None):
assert not (multilabel and multitask)
self.multilabel = multilabel
self.multitask = multitask
self.results_path = results_path
self.figures_path = figures_path
self.datasets_path = datasets_path
self.dataset_name = dataset_name
self.test_steps = test_steps
self.val_type = val_type
self.cont = cont
self.print_per_task = print_per_task
if test_csv_filename is not None:
self.test_csv_filename = self.datasets_path + test_csv_filename
else:
self.test_csv_filename = None
if cont:
replace = True
else:
replace = False
if trial_name is None and target_label is not None:
trial_name = helper.getFriendlyLabelName(target_label)
self.trial_name = trial_name
self.val_output_prefix = self.getValOutputName(val_output_file,
dataset_name,
trial_name,
replace=replace)
#dataset stuff
if multitask:
train_tasks = pickle.load(
open(self.datasets_path + dataset_name + "Train.p", "rb"))
val_tasks = pickle.load(
open(self.datasets_path + dataset_name + "Val.p", "rb"))
test_tasks = pickle.load(
open(self.datasets_path + dataset_name + "Test.p", "rb"))
self.net = mtltf.TensorFlowNetworkMTL(
train_tasks,
val_tasks,
test_tasks,
verbose=False,
val_type=self.val_type,
print_per_task=print_per_task)
self.wanted_labels = self.net.optimize_labels
else:
self.data_df = pd.DataFrame.from_csv(self.datasets_path +
self.dataset_name)
self.wanted_feats = [
x for x in self.data_df.columns.values if x != 'user_id'
and x != 'timestamp' and x != 'dataset' and '_Label' not in x
]
if self.multilabel:
self.wanted_labels = [
x for x in self.data_df.columns.values
if '_Label' in x and 'tomorrow_' in x and 'Evening' in x
and 'Alertness' not in x and 'Energy' not in x
]
self.optimize_labels = [
x for x in self.wanted_labels
if 'tomorrow_' in x and 'Evening_' in x
]
else:
self.wanted_labels = [target_label]
#actual network
self.net = tfnet.TensorFlowNetwork(
self.data_df,
self.wanted_feats,
self.wanted_labels,
optimize_labels=self.wanted_labels,
multilabel=self.multilabel,
verbose=False,
val_type=self.val_type)
#parameters that can be tuned:
self.l2_regularizers = [1e-2, 1e-4]
self.dropout = [True, False]
self.decay = [True]
self.decay_steps = [1000]
self.decay_rates = [0.95]
self.optimizers = [
tf.train.AdamOptimizer
] #[tf.train.AdagradOptimizer, tf.train.GradientDescentOptimizer
self.train_steps = [5001]
if multitask:
self.batch_sizes = [20]
self.learning_rates = [.01, .001, .0001]
self.architectures = [[500, 50], [300, 20, 10]
] if architectures is None else architectures
else:
self.batch_sizes = [50, 75]
self.learning_rates = [.01, .001, .0001]
self.architectures = [[1024, 256], [500, 50], [1024]
] if architectures is None else architectures
#storing the results
self.time_sum = 0
if cont:
self.val_results_df = pd.DataFrame.from_csv(
self.results_path + self.val_output_prefix + '.csv')
print '\nPrevious validation results df loaded. It has', len(
self.val_results_df), "rows"
self.started_from = len(self.val_results_df)
else:
self.val_results_df = pd.DataFrame()
self.started_from = 0
def testOneSetting(self, hidden_layers, l2_beta, lrate, dropout, decay,
dsteps, drate, bsize, opt, tsteps, num_settings):
print "Testing setting with layers", hidden_layers, "beta", l2_beta, "lrate", lrate, "dropout", dropout, "decay", decay, "dsteps", dsteps, "drate", drate, "bsize", bsize, "opt", opt, "tsteps", tsteps
if self.cont:
if self.settingAlreadyDone(hidden_layers, l2_beta, lrate, dropout,
decay, dsteps, drate, bsize, opt,
tsteps):
return
t0 = time()
self.net.setParams(l2_beta=l2_beta,
initial_learning_rate=lrate,
decay=decay,
decay_steps=dsteps,
decay_rate=drate,
batch_size=bsize,
optimizer=opt,
n_steps=tsteps,
dropout=dropout)
self.constructNetwork(hidden_layers)
if self.val_type == 'cross':
acc, auc, f1, precision, recall = self.net.trainAndCrossValidate()
else:
acc, auc, f1, precision, recall = self.net.trainAndValidate()
results_dict = {
'hidden_layers': hidden_layers,
'l2_beta': l2_beta,
'learning_rate': lrate,
'dropout': dropout,
'decay': decay,
'decay_steps': dsteps,
'decay_rate': drate,
'batch_size': bsize,
'optimizer': opt,
'val_acc': acc,
'val_auc': auc,
'val_f1': f1,
'val_precision': precision,
'val_recall': recall
}
if self.multitask:
results_dict['train_nan_percent'] = self.net.train_nan_percent[-1]
results_dict['val_nan_percent'] = self.net.val_nan_percent[-1]
if self.multilabel or self.print_per_task:
for label in self.wanted_labels:
friendly_label = helper.getFriendlyLabelName(label)
results_dict[friendly_label +
'_acc'] = self.net.training_val_results_per_task[
'acc'][label][-1]
results_dict[friendly_label +
'_auc'] = self.net.training_val_results_per_task[
'auc'][label][-1]
results_dict[friendly_label +
'_f1'] = self.net.training_val_results_per_task[
'f1'][label][-1]
results_dict[
friendly_label +
'_precision'] = self.net.training_val_results_per_task[
'precision'][label][-1]
results_dict[
friendly_label +
'_recall'] = self.net.training_val_results_per_task[
'recall'][label][-1]
self.val_results_df = self.val_results_df.append(results_dict,
ignore_index=True)
print self.val_results_df.tail(n=1)
t1 = time()
this_time = t1 - t0
print "It took", this_time, "seconds to obtain this result"
self.time_sum = self.time_sum + this_time
self.printTimeEstimate(
len(self.val_results_df) - self.started_from, num_settings)
sys.stdout.flush()
#output the file every few iterations for safekeeping
if len(self.val_results_df) % OUTPUT_EVERY_NTH == 0:
self.val_results_df.to_csv(self.results_path +
self.val_output_prefix + '.csv')
def getUserTaskListFromDataset(datafile,
target_label,
suppress_output=False,
group_on='user_id',
subdivide_phys=False):
"""Partitions a .csv file into a task-dict-list pickle file by separating
different individuals (users) into the different tasks."""
df = pd.DataFrame.from_csv(datafile)
wanted_feats = [
x for x in df.columns.values if x != 'user_id' and x != 'timestamp'
and x != 'dataset' and x != 'classifier_friendly_ppt_id'
and 'Cluster' not in x and '_Label' not in x
]
df = helper.normalizeAndFillDataDf(df,
wanted_feats, [target_label],
suppress_output=True)
df = df.reindex(np.random.permutation(df.index))
dataset_name, datapath = getDatasetCoreNameAndPath(datafile)
label_name = helper.getFriendlyLabelName(target_label)
modality_dict = getModalityDict(wanted_feats,
subdivide_phys=subdivide_phys)
train_task_dict_list = []
val_task_dict_list = []
test_task_dict_list = []
for user in df[group_on].unique():
if not suppress_output:
print("Processing task", user)
mini_df = df[df[group_on] == user]
train_task_dict_list.append(
constructTaskDict(user, mini_df, wanted_feats, target_label,
modality_dict, 'Train'))
val_task_dict_list.append(
constructTaskDict(user, mini_df, wanted_feats, target_label,
modality_dict, 'Val'))
test_task_dict_list.append(
constructTaskDict(user, mini_df, wanted_feats, target_label,
modality_dict, 'Test'))
if group_on == 'user_id':
dataset_prefix = "datasetUserTaskList-"
elif group_on == 'Cluster':
dataset_prefix = 'datasetClusterTasks-'
else:
dataset_prefix = group_on
pickle.dump(
train_task_dict_list,
open(
datapath + dataset_prefix + dataset_name + "-" + label_name +
"_Train.p", "wb"))
pickle.dump(
val_task_dict_list,
open(
datapath + dataset_prefix + dataset_name + "-" + label_name +
"_Val.p", "wb"))
pickle.dump(
test_task_dict_list,
open(
datapath + dataset_prefix + dataset_name + "-" + label_name +
"_Test.p", "wb"))
return dataset_prefix + dataset_name + "-" + label_name
def getFinalResultsAndSave(self, setting_dict):
if self.val_type == 'cross':
print "\nPlotting cross-validation results for best settings..."
self.getCrossValidationResults(dict(),
setting_dict['tau10'],
setting_dict['tau20'],
setting_dict['sigma_multiplier'],
setting_dict['mu_multiplier'],
save_plots=True)
print "\nRetraining on training data with the best settings..."
self.initializeHBLRModel(self.train_tasks)
self.classifier.verbose = True
self.setClassifierToSetting(setting_dict['tau10'],
setting_dict['tau20'],
setting_dict['sigma_multiplier'],
setting_dict['mu_multiplier'])
self.classifier.trainUntilConverged()
print "\nPlotting and saving cool stuff about the final model..."
self.saveImagePlot(self.classifier.phi, 'Phi')
pd.DataFrame(self.classifier.phi).to_csv(self.results_path +
self.save_prefix + "-phi.csv")
self.saveConvergencePlots()
print "\nEvaluating results on held-out test set!! ..."
all_preds = []
all_true_y = []
all_X_data = []
per_task_accs = [np.nan] * self.n_tasks
per_task_aucs = [np.nan] * self.n_tasks
per_task_f1 = [np.nan] * self.n_tasks
per_task_precision = [np.nan] * self.n_tasks
per_task_recall = [np.nan] * self.n_tasks
for t in range(self.n_tasks):
preds = self.classifier.predictBinary(self.test_tasks[t]['X'], t)
true_y = list(self.test_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
all_X_data.extend(self.test_tasks[t]['X'])
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t] = t_acc
per_task_aucs[t] = t_auc
per_task_f1[t] = t_f1
per_task_precision[t] = t_precision
per_task_recall[t] = t_recall
print "\tHELD OUT TEST METRICS COMPUTED BY APPENDING ALL PREDS"
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
all_preds, all_true_y)
print '\t\tAcc:', acc, 'AUC:', auc, 'F1:', f1, 'Precision:', precision, 'Recall:', recall
print "\n\tHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print '\t\tAcc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
print "\n\tHELD OUT TEST METRICS COMPUTED FOR EACH TASK"
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = self.test_tasks[t]['Name']
if not self.users_as_tasks:
task_name = helper.getFriendlyLabelName(task_name)
print "\t\t", task_name, "- Acc:", per_task_accs[
t], "AUC:", per_task_aucs[t], 'F1:', per_task_f1[
t], 'Precision:', per_task_precision[
t], 'Recall:', per_task_recall[t]
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if self.users_as_tasks:
task_column = 'user_id'
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.classifier.predictBinary,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=False)
else:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.classifier.predictBinary,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
print "\t SAVING CLASSIFIER"
with open(
self.results_path + "PickledModel-" + self.save_prefix + '.p',
"w") as f:
pickle.dump(self.classifier, f)
def getCrossValidationResults(self,
results_dict,
tau10,
tau20,
sigma_mult,
mu_mult,
save_plots=False,
print_per_fold=False):
if save_plots:
same_task_matrix = np.zeros((self.n_tasks, self.n_tasks))
clusters = [0] * self.num_cross_folds
all_acc = []
all_auc = []
all_f1 = []
all_precision = []
all_recall = []
if not self.users_as_tasks:
per_task_accs = [[] for i in range(self.n_tasks)]
per_task_aucs = [[] for i in range(self.n_tasks)]
per_task_f1 = [[] for i in range(self.n_tasks)]
per_task_precision = [[] for i in range(self.n_tasks)]
per_task_recall = [[] for i in range(self.n_tasks)]
for f in range(self.num_cross_folds):
train_tasks, val_tasks = helper.loadCrossValData(
self.datasets_path, self.file_prefix, f, reshape=True)
self.initializeHBLRModel(train_tasks)
self.setClassifierToSetting(tau10, tau20, sigma_mult, mu_mult)
self.classifier.trainUntilConverged()
clusters[f] = self.classifier.K
if save_plots:
same_task_matrix = self.updateSameTaskMatrix(same_task_matrix)
# Get results!
fold_preds = []
fold_true_y = []
for t in range(self.n_tasks):
preds = self.classifier.predictBinary(val_tasks[t]['X'], t)
true_y = list(val_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
continue
if not self.users_as_tasks:
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t].append(t_acc)
per_task_aucs[t].append(t_auc)
per_task_f1[t].append(t_f1)
per_task_precision[t].append(t_precision)
per_task_recall[t].append(t_recall)
if print_per_fold:
print "Fold", f, "Task", val_tasks[t][
'Name'], "acc", t_acc, "auc", t_auc, "f1", t_f1, "precision", t_precision, "recall", t_recall
fold_preds.extend(preds)
fold_true_y.extend(true_y)
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
fold_preds, fold_true_y)
all_acc.append(acc)
all_auc.append(auc)
all_f1.append(f1)
all_precision.append(precision)
all_recall.append(recall)
if print_per_fold:
print "Fold", f, "acc", acc, "auc", auc, "f1", f1, "precision", precision, "recall", recall
print "accs for all folds", all_acc
print "aucs for all folds", all_auc
print "clusters for all folds", clusters
if save_plots:
self.plotAccuracyAucAndClusters(all_acc, all_auc, clusters)
self.saveHintonPlot(same_task_matrix, self.num_cross_folds)
pd.DataFrame(same_task_matrix).to_csv(self.results_path +
self.save_prefix +
"-same_task_matrix.csv")
# Add results to the dictionary
results_dict['val_acc'] = np.nanmean(all_acc)
results_dict['val_auc'] = np.nanmean(all_auc)
results_dict['val_f1'] = np.nanmean(all_f1)
results_dict['val_precision'] = np.nanmean(all_precision)
results_dict['val_recall'] = np.nanmean(all_recall)
results_dict['num_clusters'] = np.nanmean(clusters)
# Add per-task results to the dictionary
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = val_tasks[t]['Name']
results_dict[
'TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_accs[t])
results_dict[
'TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_aucs[t])
results_dict[
'TaskF1-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_f1[t])
results_dict[
'TaskPrecision-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_precision[t])
results_dict[
'TaskRecall-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_recall[t])
return results_dict
def get_final_results(self, optimize_for='val_acc'):
if self.users_as_tasks and not self.check_test:
print "check_test is set to false, Will not evaluate performance on held-out test set."
return
print "\nAbout to evaluate results on held-out test set!!"
print "Will use the settings that produced the best", optimize_for
all_preds = []
all_true_y = []
per_task_accs = []
per_task_aucs = []
per_task_f1 = []
per_task_precision = []
per_task_recall = []
for t in range(self.n_tasks):
task_settings = self.find_best_setting_for_task(
t, optimize_for=optimize_for)
assert (task_settings['task_num'] == t)
if not self.users_as_tasks:
print "\nBEST SETTING FOR TASK", t, "-", task_settings[
'task_name']
print "The highest", optimize_for, "of", task_settings[
optimize_for], "was found with the following settings:"
print task_settings
task_settings = self.convert_param_dict_for_use(task_settings)
preds, true_y = self.get_preds_true_for_task(
self.train_tasks, self.test_tasks, task_settings)
if preds is None or true_y is None:
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs.append(t_acc)
per_task_aucs.append(t_auc)
per_task_f1.append(t_f1)
per_task_precision.append(t_precision)
per_task_recall.append(t_recall)
if not self.users_as_tasks:
print "\nFINAL TEST RESULTS FOR", helper.getFriendlyLabelName(
self.train_tasks[t]['Name'])
print 'Acc:', t_acc, 'AUC:', t_auc, 'F1:', t_f1, 'Precision:', t_precision, 'Recall:', t_recall
print "\nHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print 'Acc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if self.users_as_tasks:
task_column = 'user_id'
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.predict_task,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=False)
else:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.predict_task,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
