def get_preds_for_df(self): X = self.data_df[self.wanted_feats].as_matrix() preds = self.predict(X) assert len(preds) == len(self.data_df) preds_df = copy.deepcopy(self.data_df) for i,wanted_label in enumerate(self.wanted_labels): label_name = helper.getFriendlyLabelName(wanted_label) preds_df['test_pred_'+label_name] = preds test_df = preds_df[preds_df['dataset']=='Test'] test_df = test_df.dropna(subset=[wanted_label], how='any') all_preds = test_df['test_pred_'+label_name].tolist() all_true = test_df[wanted_label].tolist() print "FINAL METRICS ON TEST SET for label", label_name, ":", helper.computeAllMetricsForPreds(all_preds, all_true) print "Predictions have been computed and are stored in dataframe." return preds_df
def get_cross_validation_results(self, param_dict, print_per_fold=False):
all_acc = []
all_auc = []
all_f1 = []
all_precision = []
all_recall = []
for f in range(self.num_cross_folds):
train_tasks, val_tasks = helper.loadCrossValData(
self.datasets_path, self.file_prefix, f, fix_y=True)
preds, true_y = self.get_preds_true_for_task(
train_tasks, val_tasks, param_dict)
if preds is None or true_y is None:
continue
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
preds, true_y)
all_acc.append(acc)
all_auc.append(auc)
all_f1.append(f1)
all_precision.append(precision)
all_recall.append(recall)
if print_per_fold:
print "Fold", f, "acc", acc, "auc", auc, "f1", f1, "precision", precision, "recall", recall
if print_per_fold:
print "accs for all folds", all_acc
print "aucs for all folds", all_auc
# Add results to the dictionary
param_dict['val_acc'] = np.nanmean(all_acc)
param_dict['val_auc'] = np.nanmean(all_auc)
param_dict['val_f1'] = np.nanmean(all_f1)
param_dict['val_precision'] = np.nanmean(all_precision)
param_dict['val_recall'] = np.nanmean(all_recall)
return param_dict
def getFinalResultsAndSave(self, results_dict):
print "\nRetraining on full training data with the best settings..."
self.drop20 = False
self.initializeAndTrainMTMKL(self.train_tasks,
results_dict['C'],
results_dict['beta'],
results_dict['kernel'],
results_dict['v'],
results_dict['regularizer'],
verbose=True)
print "\nEvaluating results on held-out test set!! ..."
all_preds = []
all_true_y = []
per_task_accs = [np.nan] * self.n_tasks
per_task_aucs = [np.nan] * self.n_tasks
per_task_f1 = [np.nan] * self.n_tasks
per_task_precision = [np.nan] * self.n_tasks
per_task_recall = [np.nan] * self.n_tasks
for t in range(self.n_tasks):
preds = self.classifier.predictOneTask(self.test_tasks, t)
true_y = list(self.test_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
print "no y for task", t, "... skipping"
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t] = t_acc
per_task_aucs[t] = t_auc
per_task_f1[t] = t_f1
per_task_precision[t] = t_precision
per_task_recall[t] = t_recall
print "\nPlotting cool stuff about the final model..."
self.saveImagePlot(self.classifier.eta, 'Etas')
pd.DataFrame(
self.classifier.eta).to_csv(self.etas_path + self.save_prefix +
"-etas.csv")
print "\tHELD OUT TEST METRICS COMPUTED BY APPENDING ALL PREDS"
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
all_preds, all_true_y)
print '\t\tAcc:', acc, 'AUC:', auc, 'F1:', f1, 'Precision:', precision, 'Recall:', recall
print "\n\tHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print '\t\tAcc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
print "\n\tHELD OUT TEST METRICS COMPUTED FOR EACH TASK"
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = self.test_tasks[t]['Name']
task_name = helper.getFriendlyLabelName(task_name)
print "\t\t", task_name, "- Acc:", per_task_accs[
t], "AUC:", per_task_aucs[t], 'F1:', per_task_f1[
t], 'Precision:', per_task_precision[
t], 'Recall:', per_task_recall[t]
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if 'Big5GenderKMeansCluster' in self.file_prefix:
task_column = 'Big5GenderKMeansCluster'
tasks_are_ints = True
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.classifier.predict_01,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=tasks_are_ints)
elif not self.users_as_tasks:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.classifier.predict_01,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
else:
print "Error! Cannot determine what type of model you are training and therefore cannot save predictions."
return
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
def getCrossValidationResults(self,
results_dict,
C,
beta,
kernel,
v,
regularizer,
save_plots=False,
print_per_fold=True):
all_acc = []
all_auc = []
all_f1 = []
all_precision = []
all_recall = []
if not self.users_as_tasks:
per_task_accs = [[] for i in range(self.n_tasks)]
per_task_aucs = [[] for i in range(self.n_tasks)]
per_task_f1 = [[] for i in range(self.n_tasks)]
per_task_precision = [[] for i in range(self.n_tasks)]
per_task_recall = [[] for i in range(self.n_tasks)]
for f in range(self.num_cross_folds):
train_tasks, val_tasks = helper.loadCrossValData(
self.datasets_path,
self.file_prefix,
f,
reshape=False,
fix_y=True)
converged = self.initializeAndTrainMTMKL(train_tasks, C, beta,
kernel, v, regularizer)
if not converged:
all_acc.append(np.nan)
all_auc.append(np.nan)
all_f1.append(np.nan)
all_precision.append(np.nan)
all_recall.append(np.nan)
continue
# Get results!
fold_preds = []
fold_true_y = []
for t in range(self.n_tasks):
preds = self.classifier.predictOneTask(val_tasks, t)
true_y = list(val_tasks[t]['Y'].flatten())
if not self.users_as_tasks:
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t].append(t_acc)
per_task_aucs[t].append(t_auc)
per_task_f1[t].append(t_f1)
per_task_precision[t].append(t_precision)
per_task_recall[t].append(t_recall)
if print_per_fold:
print "Fold", f, "Task", val_tasks[t][
'Name'], "acc", t_acc, "auc", t_auc, "f1", t_f1, "precision", t_precision, "recall", t_recall
fold_preds.extend(preds)
fold_true_y.extend(true_y)
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
fold_preds, fold_true_y)
all_acc.append(acc)
all_auc.append(auc)
all_f1.append(f1)
all_precision.append(precision)
all_recall.append(recall)
if print_per_fold:
print "Fold", f, "acc", acc, "auc", auc, "f1", f1, "precision", precision, "recall", recall
print "accs for all folds", all_acc
print "aucs for all folds", all_auc
# Add results to the dictionary
results_dict['val_acc'] = np.nanmean(all_acc)
results_dict['val_auc'] = np.nanmean(all_auc)
results_dict['val_f1'] = np.nanmean(all_f1)
results_dict['val_precision'] = np.nanmean(all_precision)
results_dict['val_recall'] = np.nanmean(all_recall)
# Add per-task results to the dictionary
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = val_tasks[t]['Name']
results_dict[
'TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_accs[t])
results_dict[
'TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_aucs[t])
results_dict[
'TaskF1-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_f1[t])
results_dict[
'TaskPrecision-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_precision[t])
results_dict[
'TaskRecall-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_recall[t])
return results_dict
def getAllMetricsForLabel(preds,labels,wanted_labels,target_label): binary_preds,true_y = getBinaryPredsTrueYForLabel(preds,labels,wanted_labels,target_label) return helper.computeAllMetricsForPreds(binary_preds, true_y)
def getFinalResultsAndSave(self, setting_dict):
if self.val_type == 'cross':
print "\nPlotting cross-validation results for best settings..."
self.getCrossValidationResults(dict(),
setting_dict['tau10'],
setting_dict['tau20'],
setting_dict['sigma_multiplier'],
setting_dict['mu_multiplier'],
save_plots=True)
print "\nRetraining on training data with the best settings..."
self.initializeHBLRModel(self.train_tasks)
self.classifier.verbose = True
self.setClassifierToSetting(setting_dict['tau10'],
setting_dict['tau20'],
setting_dict['sigma_multiplier'],
setting_dict['mu_multiplier'])
self.classifier.trainUntilConverged()
print "\nPlotting and saving cool stuff about the final model..."
self.saveImagePlot(self.classifier.phi, 'Phi')
pd.DataFrame(self.classifier.phi).to_csv(self.results_path +
self.save_prefix + "-phi.csv")
self.saveConvergencePlots()
print "\nEvaluating results on held-out test set!! ..."
all_preds = []
all_true_y = []
all_X_data = []
per_task_accs = [np.nan] * self.n_tasks
per_task_aucs = [np.nan] * self.n_tasks
per_task_f1 = [np.nan] * self.n_tasks
per_task_precision = [np.nan] * self.n_tasks
per_task_recall = [np.nan] * self.n_tasks
for t in range(self.n_tasks):
preds = self.classifier.predictBinary(self.test_tasks[t]['X'], t)
true_y = list(self.test_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
all_X_data.extend(self.test_tasks[t]['X'])
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t] = t_acc
per_task_aucs[t] = t_auc
per_task_f1[t] = t_f1
per_task_precision[t] = t_precision
per_task_recall[t] = t_recall
print "\tHELD OUT TEST METRICS COMPUTED BY APPENDING ALL PREDS"
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
all_preds, all_true_y)
print '\t\tAcc:', acc, 'AUC:', auc, 'F1:', f1, 'Precision:', precision, 'Recall:', recall
print "\n\tHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print '\t\tAcc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
print "\n\tHELD OUT TEST METRICS COMPUTED FOR EACH TASK"
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = self.test_tasks[t]['Name']
if not self.users_as_tasks:
task_name = helper.getFriendlyLabelName(task_name)
print "\t\t", task_name, "- Acc:", per_task_accs[
t], "AUC:", per_task_aucs[t], 'F1:', per_task_f1[
t], 'Precision:', per_task_precision[
t], 'Recall:', per_task_recall[t]
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if self.users_as_tasks:
task_column = 'user_id'
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.classifier.predictBinary,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=False)
else:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.classifier.predictBinary,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"
print "\t SAVING CLASSIFIER"
with open(
self.results_path + "PickledModel-" + self.save_prefix + '.p',
"w") as f:
pickle.dump(self.classifier, f)
def getCrossValidationResults(self,
results_dict,
tau10,
tau20,
sigma_mult,
mu_mult,
save_plots=False,
print_per_fold=False):
if save_plots:
same_task_matrix = np.zeros((self.n_tasks, self.n_tasks))
clusters = [0] * self.num_cross_folds
all_acc = []
all_auc = []
all_f1 = []
all_precision = []
all_recall = []
if not self.users_as_tasks:
per_task_accs = [[] for i in range(self.n_tasks)]
per_task_aucs = [[] for i in range(self.n_tasks)]
per_task_f1 = [[] for i in range(self.n_tasks)]
per_task_precision = [[] for i in range(self.n_tasks)]
per_task_recall = [[] for i in range(self.n_tasks)]
for f in range(self.num_cross_folds):
train_tasks, val_tasks = helper.loadCrossValData(
self.datasets_path, self.file_prefix, f, reshape=True)
self.initializeHBLRModel(train_tasks)
self.setClassifierToSetting(tau10, tau20, sigma_mult, mu_mult)
self.classifier.trainUntilConverged()
clusters[f] = self.classifier.K
if save_plots:
same_task_matrix = self.updateSameTaskMatrix(same_task_matrix)
# Get results!
fold_preds = []
fold_true_y = []
for t in range(self.n_tasks):
preds = self.classifier.predictBinary(val_tasks[t]['X'], t)
true_y = list(val_tasks[t]['Y'].flatten())
if len(preds) == 0 or len(true_y) == 0:
continue
if not self.users_as_tasks:
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs[t].append(t_acc)
per_task_aucs[t].append(t_auc)
per_task_f1[t].append(t_f1)
per_task_precision[t].append(t_precision)
per_task_recall[t].append(t_recall)
if print_per_fold:
print "Fold", f, "Task", val_tasks[t][
'Name'], "acc", t_acc, "auc", t_auc, "f1", t_f1, "precision", t_precision, "recall", t_recall
fold_preds.extend(preds)
fold_true_y.extend(true_y)
acc, auc, f1, precision, recall = helper.computeAllMetricsForPreds(
fold_preds, fold_true_y)
all_acc.append(acc)
all_auc.append(auc)
all_f1.append(f1)
all_precision.append(precision)
all_recall.append(recall)
if print_per_fold:
print "Fold", f, "acc", acc, "auc", auc, "f1", f1, "precision", precision, "recall", recall
print "accs for all folds", all_acc
print "aucs for all folds", all_auc
print "clusters for all folds", clusters
if save_plots:
self.plotAccuracyAucAndClusters(all_acc, all_auc, clusters)
self.saveHintonPlot(same_task_matrix, self.num_cross_folds)
pd.DataFrame(same_task_matrix).to_csv(self.results_path +
self.save_prefix +
"-same_task_matrix.csv")
# Add results to the dictionary
results_dict['val_acc'] = np.nanmean(all_acc)
results_dict['val_auc'] = np.nanmean(all_auc)
results_dict['val_f1'] = np.nanmean(all_f1)
results_dict['val_precision'] = np.nanmean(all_precision)
results_dict['val_recall'] = np.nanmean(all_recall)
results_dict['num_clusters'] = np.nanmean(clusters)
# Add per-task results to the dictionary
if not self.users_as_tasks:
for t in range(self.n_tasks):
task_name = val_tasks[t]['Name']
results_dict[
'TaskAcc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_accs[t])
results_dict[
'TaskAuc-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_aucs[t])
results_dict[
'TaskF1-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_f1[t])
results_dict[
'TaskPrecision-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_precision[t])
results_dict[
'TaskRecall-' +
helper.getFriendlyLabelName(task_name)] = np.nanmean(
per_task_recall[t])
return results_dict
def get_final_results(self, optimize_for='val_acc'):
if self.users_as_tasks and not self.check_test:
print "check_test is set to false, Will not evaluate performance on held-out test set."
return
print "\nAbout to evaluate results on held-out test set!!"
print "Will use the settings that produced the best", optimize_for
all_preds = []
all_true_y = []
per_task_accs = []
per_task_aucs = []
per_task_f1 = []
per_task_precision = []
per_task_recall = []
for t in range(self.n_tasks):
task_settings = self.find_best_setting_for_task(
t, optimize_for=optimize_for)
assert (task_settings['task_num'] == t)
if not self.users_as_tasks:
print "\nBEST SETTING FOR TASK", t, "-", task_settings[
'task_name']
print "The highest", optimize_for, "of", task_settings[
optimize_for], "was found with the following settings:"
print task_settings
task_settings = self.convert_param_dict_for_use(task_settings)
preds, true_y = self.get_preds_true_for_task(
self.train_tasks, self.test_tasks, task_settings)
if preds is None or true_y is None:
continue
all_preds.extend(preds)
all_true_y.extend(true_y)
# save the per-task results
t_acc, t_auc, t_f1, t_precision, t_recall = helper.computeAllMetricsForPreds(
preds, true_y)
per_task_accs.append(t_acc)
per_task_aucs.append(t_auc)
per_task_f1.append(t_f1)
per_task_precision.append(t_precision)
per_task_recall.append(t_recall)
if not self.users_as_tasks:
print "\nFINAL TEST RESULTS FOR", helper.getFriendlyLabelName(
self.train_tasks[t]['Name'])
print 'Acc:', t_acc, 'AUC:', t_auc, 'F1:', t_f1, 'Precision:', t_precision, 'Recall:', t_recall
print "\nHELD OUT TEST METRICS COMPUTED BY AVERAGING OVER TASKS"
avg_acc = np.nanmean(per_task_accs)
avg_auc = np.nanmean(per_task_aucs)
avg_f1 = np.nanmean(per_task_f1)
avg_precision = np.nanmean(per_task_precision)
avg_recall = np.nanmean(per_task_recall)
print 'Acc:', avg_acc, 'AUC:', avg_auc, 'F1:', avg_f1, 'Precision:', avg_precision, 'Recall:', avg_recall
if self.test_csv_filename is not None:
print "\tSAVING HELD OUT PREDICITONS"
if self.users_as_tasks:
task_column = 'user_id'
label_name = helper.getFriendlyLabelName(self.file_prefix)
wanted_label = helper.getOfficialLabelName(label_name)
predictions_df = helper.get_test_predictions_for_df_with_task_column(
self.predict_task,
self.test_csv_filename,
task_column,
self.test_tasks,
wanted_label=wanted_label,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1],
label_name=label_name,
tasks_are_ints=False)
else:
predictions_df = helper.get_test_predictions_for_df_with_no_task_column(
self.predict_task,
self.test_csv_filename,
self.test_tasks,
num_feats_expected=np.shape(self.test_tasks[0]['X'])[1])
predictions_df.to_csv(self.results_path + "Preds-" +
self.save_prefix + '.csv')
else:
print "Uh oh, the test csv filename was not set, can't save test preds"