def error_function(model, batch_loader, tasks):
"""
Calculates a metric to judge model. Must return a float.
Metric is experiment dependent could be AUROC, Accuracy, Error....
Metric must be "higher is better" (eg. accuracy)
Do not modify params. Abstract method for all experiments.
"""
# When training sequentially, look at previous tasks as well.
if len(tasks) == 1:
tasks = list(range(tasks[0] + 1))
confusion_matrix = build_confusion_matrix(model, batch_loader,
number_of_tasks, tasks, device)
confusion_matrix = confusion_matrix.to(torch.device("cpu"))
print(confusion_matrix.numpy().astype(int))
class_acc = sum(confusion_matrix.diag()) / sum(confusion_matrix.sum(1))
# score = 0
# for i in range(class_acc.shape[0]):
# score += class_acc[i]
# score /= class_acc.shape[0]
return class_acc
def error_function(model, batch_loader, tasks):
"""
Calculates a metric to judge model. Must return a float.
Metric is experiment dependent could be AUROC, Accuracy, Error....
Do not modify params. Abstract method for all experiments.
"""
confusion_matrix = build_confusion_matrix(model, batch_loader,
number_of_tasks, tasks, device)
class_acc = confusion_matrix.diag() / confusion_matrix.sum(1)
score = 0
for i in tasks:
score += class_acc[i]
score /= len(tasks)
return score
def error_function(model, batch_loader, tasks):
"""
Calculates a metric to judge model. Must return a float.
Metric is experiment dependent could be AUROC, Accuracy, Error....
Metric must be "higher is better" (eg. accuracy)
Do not modify params. Abstract method for all experiments.
"""
# When training sequentially, look at previous tasks as well.
if len(tasks) == 1:
tasks = list(range(tasks[0] + 1))
confusion_matrix = build_confusion_matrix(model, batch_loader, number_of_tasks, tasks, device)
confusion_matrix = confusion_matrix.to(torch.device("cpu"))
np.set_printoptions(suppress=True)
# print(np.round(confusion_matrix.numpy()))
num_samples = sum(confusion_matrix.sum(1))
correctly_classified = sum(confusion_matrix.diag())
return correctly_classified / num_samples