def run(self): print "\nYou have chosen to test a total of", self.num_settings, "settings for each task" print "There are", self.n_tasks, "tasks, meaning you are training a total of..." print "\t", self.num_settings * self.n_tasks, "neural networks!!" sys.stdout.flush() if self.users_as_tasks: tasks = self.data_df['user_id'].unique() else: tasks = [helper.getFriendlyLabelName(x) for x in self.wanted_labels] i = 0 for t in range(len(tasks)): if self.users_as_tasks: self.runOneTask(tasks[i], self.target_label) else: self.runOneTask(tasks[i], self.wanted_labels[i]) if self.test_run and i > 2: break i += 1 self.val_results_df.to_csv(self.results_path + self.save_prefix + '.csv') if self.users_as_tasks: print "\n\nFINAL RESULTS - Averaging individual models:" print "\tValidation set: Accuracy =", np.nanmean(self.val_results_df['val_acc']), "AUC = ", np.nanmean(self.val_results_df['val_auc']) print "\tTest set: Accuracy =", np.nanmean(self.val_results_df['test_acc']), "AUC = ", np.nanmean(self.val_results_df['test_auc']) print "" print "FINAL RESULTS - Aggregating predictions of individual models" agg_auc = helper.computeAuc(self.cumulative_test_preds, self.cumulative_test_true) agg_acc = helper.getBinaryAccuracy(self.cumulative_test_preds, self.cumulative_test_true) print "\tTest set: Accuracy =", agg_acc, "AUC = ", agg_auc
def getAccuracyAucAllTasks(self, tasks):
all_task_Y = []
all_preds = []
for t in range(len(tasks)):
X_t, y_t = self.extractTaskData(tasks, t)
if len(X_t) == 0:
continue
preds = self.internal_predict(X_t, int(t))
all_task_Y.extend(y_t)
all_preds.extend(preds)
auc = roc_auc_score(all_task_Y, all_preds)
acc = helper.getBinaryAccuracy(all_preds, all_task_Y)
return acc, auc
def getAccuracyAucOnOneTask(self, task_list, task, debug=False):
X_t, y_t = self.extractTaskData(task_list, task)
if len(X_t) == 0:
return np.nan, np.nan
preds = self.internal_predict(X_t, int(task))
if debug:
print "y_t:", y_t
print "preds:", preds
acc = helper.getBinaryAccuracy(preds, y_t)
if len(y_t) > 1 and helper.containsEachSVMLabelType(
y_t) and helper.containsEachSVMLabelType(preds):
auc = roc_auc_score(y_t, preds)
else:
auc = np.nan
return acc, auc