def q4confmat(full_dat, noisy_dat):
ref_dict = full_dat.getDictionary()
# ground truth labels
annotations = []
for attrib in noisy_dat.attrib:
attribString = ','.join(str(v) for v in attrib)
if not attribString in ref_dict:
print("ERROR: attribString not present!")
continue
annotations.append(ref_dict[attribString])
evaluator = Evaluator()
c_matrix = evaluator.confusion_matrix(noisy_dat.labels, annotations)
print(c_matrix)
target_names = ["A", "C", "E", "G", "O", "Q"]
plot_confusion_matrix(c_matrix, target_names, "Noisy vs Full")
precision, macro_p = evaluator.precision(c_matrix)
recall, macro_r = evaluator.recall(c_matrix)
f1, macro_f1 = evaluator.f1_score(c_matrix)
p = np.append(precision, macro_p)
r = np.append(recall, macro_r)
f1 = np.append(f1, macro_f1)
performance_matrix = np.vstack((p, np.vstack((r, f1))))
print(performance_matrix)
plot_other_stats(performance_matrix, "Train_noisy")
return
def test(model_file, test_file, device=-1):
context = utils.Saver.load_context(model_file)
if context.seed is not None:
utils.set_random_seed(context.seed, device)
test_dataset = context.loader.load(test_file, train=False, bucketing=True)
kwargs = dict(context)
if context.model_config is not None:
kwargs.update(context.model_config)
model = _build_parser(**dict(kwargs))
chainer.serializers.load_npz(model_file, model)
if device >= 0:
chainer.cuda.get_device_from_id(device).use()
model.to_gpu(device)
pbar = training.listeners.ProgressBar(lambda n: tqdm(total=n))
pbar.init(len(test_dataset))
evaluator = Evaluator(model, context.loader.rel_map, test_file,
logging.getLogger())
utils.chainer_train_off()
for batch in test_dataset.batch(context.batch_size,
colwise=True,
shuffle=False):
xs, ts = batch[:-1], batch[-1]
ys = model.forward(*xs)
evaluator.on_batch_end({'train': False, 'xs': xs, 'ys': ys, 'ts': ts})
pbar.update(len(ts))
evaluator.on_epoch_validate_end({})
def cross_validation(x, y, k):
xpart, ypart = data_split(x, y, k)
accuracy = np.zeros(k)
classifiers = np.empty(k, dtype=object)
for i in range(k):
# split data correctly
xval = xpart[i]
yval = ypart[i]
xtrain = np.delete(xpart, i, 0).reshape((k - 1) * xval.shape[0],
xval.shape[1])
ytrain = np.delete(ypart, i, 0).reshape((k - 1) * xval.shape[0], 1)
# train on training slice
classifiers[i] = DecisionTreeClassifier()
classifiers[i] = classifiers[i].train(xtrain, ytrain)
#predict for test class
predictions = classifiers[i].predict(xval)
# validate using statistics
eval = Evaluator()
confusion = eval.confusion_matrix(predictions, yval)
accuracy[i] = eval.accuracy(confusion)
return accuracy, classifiers
def main():
from utils.hyp import parse_args
args = parse_args()
trainer = Trainer(args)
evaluator = Evaluator(args)
for epoch in range(trainer.start_epoch, args.epochs):
trainer.training(epoch)
if not args.no_val and epoch % args.validate == (args.validate - 1):
map, mf1 = evaluator.validation(trainer.model, epoch)
val_svar_pf = '[mode: val ' +\
'mAP: %5.4g, ' % map +\
'mF1: %5.4g]' % mf1
trainer.saver.save_log(val_svar_pf)
if args.visdom:
update_vis_plot(trainer.vis, epoch, [map, mf1],
trainer.val_plot, 'append')
if evaluator.is_best:
trainer.best_pred = evaluator.new_pred
if args.is_save:
# save checkpoint every epoch
trainer.saver.save_checkpoint({
'epoch': epoch,
'state_dict': trainer.model.module.state_dict() \
if args.ng > 1 and args.use_multi_gpu else trainer.model.state_dict(),
'optimizer': trainer.optimizer.state_dict(),
'best_pred': evaluator.best_pred,
}, evaluator.is_best)
def getAccuracy(self):
evaluator = Evaluator()
predictions = self.decisionTreeClassifier.predict(
self.validationAttrib)
c_matrix = evaluator.confusion_matrix(predictions,
self.validationLabel)
return evaluator.accuracy(c_matrix)
def test(model_file, test_file, device=-1):
context = utils.Saver.load_context(model_file)
if context.seed is not None:
utils.set_random_seed(context.seed, device)
test_dataset = context.loader.load(test_file, train=False, bucketing=True)
model = _build_parser(**dict(context))
chainer.serializers.load_npz(model_file, model)
if device >= 0:
chainer.cuda.get_device_from_id(device).use()
model.to_gpu(device)
pbar = training.listeners.ProgressBar(lambda n: tqdm(total=n))
pbar.init(len(test_dataset))
evaluator = Evaluator(model, context.loader.rel_map, test_file,
Log.getLogger())
utils.chainer_train_off()
for batch in test_dataset.batch(context.batch_size,
colwise=True,
shuffle=False):
xs, ts = batch[:-1], batch[-1]
parsed = model.parse(*xs)
evaluator.append([tokens[1:] for tokens in xs[-1]], parsed)
pbar.update(len(ts))
evaluator.report(show_details=False)
def calculate_best_pruned_tree(self, original_tree, trees, x_val, y_val):
eval = Evaluator()
classifier = DecisionTreeClassifier()
classifier.is_trained = True
original_predictions = classifier.predict(x_val, original_tree)
original_error = self.get_apperent_error_rate(original_predictions,
y_val)
stored_j = 0
initial_alpha = 0
previous_diff = 0
previous_alpha = 0
#go through each tree and compute the ratio of caculated error (right/total)
for j in range(1, len(trees)):
predictions = classifier.predict(x_val, trees[j])
error = self.get_apperent_error_rate(predictions, y_val)
original_number_leaves = self.count_leaves(original_tree)
number_of_leaves = self.count_leaves(trees[j])
alpha = (original_error - error) / (original_number_leaves -
number_of_leaves)
diff = initial_alpha - alpha
if diff < previous_diff:
stored_j = j
previous_alpha = alpha
return trees[stored_j], previous_alpha
def get_evaluator(self):
evaldataset = CityscapesSelectDataset(im_size=self.args.img_size,
n_samples=self.args.eval_samples)
evaluator = Evaluator(evaldataset,
samples=25,
metrics=["miou"],
crf=False)
return evaluator
def prune(node, dataset):
"""
Recursively prunes a decision tree classifier
Parameters
----------
root : Node
the node being considered for pruning
dataset: Dataset
the dataset being used for pruning
"""
if not node.is_leaf():
two_leaves = node.right.is_leaf() and node.left.is_leaf()
if not two_leaves:
if (node.left.is_node()):
prune(node.left, dataset)
if (node.right.is_node()):
prune(node.right, dataset)
two_leaves = node.right.is_leaf() and node.left.is_leaf()
if two_leaves:
annotation = dataset.labels
attributes = dataset.attributes
# Try and prune current node
# Calculate accuracy before pruning
evaluator = Evaluator()
predictions_before = tree.predict(attributes)
confusion = evaluator.confusion_matrix(predictions_before, annotation)
accuracy_before = evaluator.accuracy(confusion)
# Store leaves and rule temporarily
temp_label_left = node.left.label
temp_label_right = node.right.label
temp_rule = node.rule
# Prune current node
node.label = node.majority_label
node.left.label = None
node.right.label = None
node.rule = None
# Calculate accuracy after pruning
predictions_after = tree.predict(attributes)
confusion = evaluator.confusion_matrix(predictions_after, annotation)
accuracy_after = evaluator.accuracy(confusion)
# Restore node if accuracy dropped
if (accuracy_after < accuracy_before):
node.left.label = temp_label_left
node.right.label = temp_label_right
node.label = None
node.rule = temp_rule
def old_test():
# data_read("data/toy.txt")
prediction = ["A", "B"]
annotation = ["A", "A"]
class_labels = ["B", "A"]
obj = Evaluator()
matrix = obj.confusion_matrix(prediction, annotation, class_labels)
print(str.format('{0:.15f}', obj.accuracy(matrix)))
print(obj.precision(matrix))
print(obj.recall(matrix))
print(obj.f1_score(matrix))
def prune(tree: BinTree):
vld_dataset = data_read("data/validation.txt")
x_val, y_val = vld_dataset.shim_to_arrays()
ev = Evaluator()
for i in range(10):
print(f"----prune attempt {i + 1}---")
tree.prune(node=tree.root_node,
og_vld_feats=x_val,
og_vld_lbls=y_val,
dataset=vld_dataset,
ev=ev,
is_aggressive=False)
def evaluate_model(model, num_classes, X_test, Y_test):
predicted = model.predict(X_test)
target_label = convert_prob_to_label(Y_test)
predicted_label = convert_prob_to_label(predicted)
target = np.ravel(target_label)
predicted = np.ravel(predicted_label)
evaluator = Evaluator(num_classes)
accuracy = evaluator.calculate_accuracy(target, predicted)
cm, precisions, recalls = evaluator.calculate_metrics(target, predicted)
precision = np.mean(precisions)
recall = np.mean(recalls)
f1 = 2 * precision * recall / (precision + recall)
return Result(cm, accuracy, precisions, precision, recalls, recall, f1)
def cross_validation(k, filename):
"""
Performs cross validation on a dataset
Parameters
----------
k : int
number of times dataset is split
filename : string
name of the file to load the dataset
Returns
-------
list of ints
containing the accuracies of each split
int
global error estimate
"""
file_path = "./data/" + filename
dataset = np.loadtxt(file_path, dtype=str, delimiter=',')
np.random.shuffle(dataset)
subsets = np.array_split(dataset, k)
accuracies = []
for i in range(k):
train = np.delete(subsets, i, axis=0)
train = np.concatenate(train)
train_att = train[:, :-1].astype(int)
train_labels = train[:, -1]
test = subsets[i]
test_att = test[:, :-1].astype(int)
test_labels = test[:, -1]
tree = DecisionTreeClassifier()
tree = tree.train(train_att, train_labels)
prediction = tree.predict(test_att)
evaluator = Evaluator()
confusion = evaluator.confusion_matrix(prediction, test_labels)
a = evaluator.accuracy(confusion)
accuracies.append(a)
global_error_estimate = np.mean(accuracies)
np.set_printoptions(formatter={'float': '{: 0.4f}'.format})
return accuracies, global_error_estimate
def main():
print("Loading the training dataset...")
x = np.array([[5, 7, 1], [4, 6, 2], [4, 6, 3], [1, 3, 1], [2, 1, 2],
[5, 2, 6]])
y = np.array(["A", "A", "A", "C", "C", "C"])
print("Training the decision tree...")
classifier = DecisionTreeClassifier()
classifier = classifier.train(x, y)
print("Loading the test set...")
x_test = np.array([[1, 6, 3], [0, 5, 5], [1, 5, 0], [2, 4, 2]])
y_test = np.array(["A", "A", "C", "C"])
predictions = classifier.predict(x_test)
print("Predictions: {}".format(predictions))
classes = ["A", "C"]
print("Evaluating test predictions...")
evaluator = Evaluator()
confusion = evaluator.confusion_matrix(predictions, y_test)
print("Confusion matrix:")
print(confusion)
accuracy = evaluator.accuracy(confusion)
print()
print("Accuracy: {}".format(accuracy))
(p, macro_p) = evaluator.precision(confusion)
(r, macro_r) = evaluator.recall(confusion)
(f, macro_f) = evaluator.f1_score(confusion)
print()
print("Class: Precision, Recall, F1")
for (i, (p1, r1, f1)) in enumerate(zip(p, r, f)):
print("{}: {:.2f}, {:.2f}, {:.2f}".format(classes[i], p1, r1, f1))
print()
print("Macro-averaged Precision: {:.2f}".format(macro_p))
print("Macro-averaged Recall: {:.2f}".format(macro_r))
print("Macro-averaged F1: {:.2f}".format(macro_f))
def print_stats(predictions, y_test):
eval = Evaluator()
confusion = eval.confusion_matrix(predictions, y_test)
accuracy = eval.accuracy(confusion)
precision = eval.precision(confusion)
recall = eval.recall(confusion)
f1 = eval.f1_score(confusion)
print("confusion", confusion)
print("accuracy", accuracy)
print("precision", precision)
print("recall", recall)
print("f1", f1)
return
def train(self):
# device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
valid_evaluator = Evaluator()
self.model = self.model.to(self.device)
for epochIndex in range(self.epochNum):
loss_train = self.one_pass_train(epochIndex)
loss_overall_train, loss_emo_train = LossLogger(
self.fo, 'train', loss_train, '--', epochIndex)
loss_valid = valid_evaluator.evaluation(self.X_valid, self.Y_valid,
self.model)
loss_overall_valid, loss_emo_valid = LossLogger(
self.fo, 'valid', loss_valid, '--', epochIndex)
self.trainValidWritter(loss_train, loss_valid, loss_overall_train,
loss_emo_train, loss_overall_valid,
loss_emo_valid, epochIndex)
if (self.betterSaver(loss_emo_valid) == True):
break
self.writter.close()
self.fo.close()
def _train_eval(diff_vec_class, desc_vec_class, alp):
metrics = "accuracy,recall,distance,nlg"
suf = "{}_{}_{}".format(_get_class_name(diff_vec_class),
_get_class_name(desc_vec_class), alp)
out_file = os.path.join(os.path.dirname(valid_file),
"nnupdater_result_{}.json".format(suf))
slice_test_main(train_file,
valid_file,
out_file,
alpha=alp,
slice_count=slice_count,
diff_vec_class=diff_vec_class,
desc_vec_class=desc_vec_class)
evaluator = Evaluator(args={
"--metrics": metrics,
"TEST_SET": valid_file,
"RESULT_FILE": out_file
})
r, lemma_r = evaluator.evaluate()
return suf, r, lemma_r
def prune_tree_reduced_error(self, tree, x_val, y_val):
"""
Function to accept prunes which increase the tree's accuracy, otherwise
ignore
Args:
tree (dict) - tree to be pruned
x_val (2D array) - 2D array of attributes of validation set where
each row is a differnt sample and each column is a differnt
attribute
y_val (1D array) - 1D array of correct labels for x_val validation
data
Output:
tree (dict or str) - tree pruned such that any additional pruning
would lower predictive accuracy on validation set.
"""
classifier = DecisionTreeClassifier()
classifier.is_trained = True
predictions = classifier.predict(x_val)
eval = Evaluator()
confusion = eval.confusion_matrix(predictions, y_val)
root_accuracy = eval.accuracy(confusion)
print("Results on Validation set")
print("Original Accuracy: ", root_accuracy)
is_pruned = True
while (is_pruned and isinstance(tree, dict)):
#make copy of tree then attempt to prune copy
tree_copy = copy.deepcopy(tree)
(is_pruned, tree_copy, tree) = self.prune(tree_copy, tree)
if is_pruned:
#compare accuracy of pruned tree to original
new_predictions = classifier.predict(x_val, tree_copy)
new_confusion = eval.confusion_matrix(new_predictions, y_val)
new_accuracy = eval.accuracy(new_confusion)
if new_accuracy >= root_accuracy:
#if greater or equal accuracy make tree = copy
root_accuracy = new_accuracy
tree = copy.deepcopy(tree_copy)
print("New Accuracy: ", root_accuracy)
return tree
def test_DecisionTreeClassifier(dataset_filename: str = "toy.txt",
should_load_file=False):
# train
extless_filename = dataset_filename[:-4]
start = time.time()
saved_tree_file = None
if should_load_file:
saved_tree_file = "tree_" + extless_filename + ".obj"
cl = DecisionTreeClassifier(saved_tree_file=saved_tree_file)
dataset = data_read("data/" + dataset_filename)
unique_lbls = np.unique([e.label for e in dataset.entries])
x, y = dataset.shim_to_arrays()
cl.train(x, y)
cl.tree.save_tree("tree_" + extless_filename + ".obj")
visualize_tree(cl.tree,
save_filename=f"visualize_tree_{extless_filename}.txt",
max_depth=8)
duration = time.time() - start
print("duration: ", duration)
# predict
test_dataset = data_read("data/test.txt")
x_test, y_test = test_dataset.shim_to_arrays()
preds = cl.predict(x_test)
# preds = [random.choice('ACEGOQ')
# for _ in range(len(y_test))] # testing random
# evaluate
ev = Evaluator()
matrix = ev.confusion_matrix(preds, y_test, unique_lbls)
print("real accuracy: ", accuracy_score(y_test, preds))
print("\nour calc accuracy: ", str.format('{0:.15f}', ev.accuracy(matrix)))
print("\n precision:", precision_score(y_test, preds, average="macro"))
print("\n our precision: ", ev.precision(matrix))
print("\nreal recall: ", recall_score(y_test, preds, average="macro"))
print("\n our recall: ", ev.recall(matrix))
print("\n f1_score", f1_score(y_test, preds, average="macro"))
print("\n f1_score: ", ev.f1_score(matrix))
print(matrix)
import sys
import torch
import torch.autograd as autograd
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.nn import init
torch.manual_seed(1)
SOS = 0
EOS = 1
e = Evaluator()
torch.cuda.device(0) # let's go
# precompute input tokens
tokens = [torch.LongTensor([[x]]).cuda() for x in range(22)]
token_lookup = {
'@': 1,
'A': 2,
'B': 3,
'C': 4,
'D': 5,
'E': 6,
'F': 7,
'G': 8,
pred = pd.DataFrame(te_files_wav, columns=["fname"])
pred['label'] = pred_labels
#
# # =================================================================================================== EVAL
# # =================================================================================================== EVAL
# # =================================================================================================== EVAL
print(
'\nEvaluate ACC and print score============================================================================'
)
# read ground truth
gt_test = pd.read_csv(params_files.get('gt_test'))
# init Evaluator object
evaluator = Evaluator(gt_test, pred, list_labels, params_ctrl, params_files)
print(
'\n=============================ACCURACY==============================================================='
)
print(
'=============================ACCURACY===============================================================\n'
)
evaluator.evaluate_acc()
evaluator.evaluate_acc_classwise()
evaluator.print_summary_eval()
end = time.time()
print(
'\n=============================Job finalized==========================================================\n'
)
def calc_stats(self, test_path, path_to_data, plt_title, prune,
pruneAggressively):
#load dataset, atttribs, labels
d_subset = ClassifierDataset()
d_subset.initFromFile(path_to_data)
attribs = d_subset.attrib
labels = d_subset.labels
ds_test = ClassifierDataset()
ds_test.initFromFile(test_path)
test_attribs = ds_test.attrib
test_labels = ds_test.labels
#train and predict
print("TRAINING")
tree = DecisionTreeClassifier()
tree.train(attribs, labels)
print("FINISHED TRAINING")
if prune == True:
print("PRUNING")
validationDataset = ClassifierDataset()
validationDataset.initFromFile(val_path)
Prune(tree, validationDataset.attrib, validationDataset.labels,
pruneAggressively)
print("FINISHED PRUNING")
predictions = tree.predict(test_attribs)
evaluator = Evaluator()
c_matrix = evaluator.confusion_matrix(predictions, test_labels)
print(c_matrix)
a = ["A", "C", "E", "G", "O", "Q"]
b = path_to_data[7:-4]
if prune:
if pruneAggressively:
b = b + "_aggressively_pruned"
else:
b += "_pruned"
else:
b += "_not_pruned"
plot_confusion_matrix(c_matrix, a, plt_title)
print(" ")
print("Accuracy: " + str(evaluator.accuracy(c_matrix)))
print(" ")
precision, macro_p = evaluator.precision(c_matrix)
recall, macro_r = evaluator.recall(c_matrix)
f1, macro_f1 = evaluator.f1_score(c_matrix)
p = np.append(precision, macro_p)
r = np.append(recall, macro_r)
f1 = np.append(f1, macro_f1)
performance_matrix = np.vstack((p, np.vstack((r, f1))))
print(performance_matrix)
plot_other_stats(performance_matrix, plt_title)
'''
print("Precision: " + str(precision))
print("Recall: " + str(recall))
print("F1 Score: " + str(f1))'''
print(" ")
print("Macro avg recall:" + str(macro_r))
print("Macro avg precision:" + str(macro_p))
print("Macro avg f1:" + str(macro_f1))
print(" ")
def __init__(self):
self.evaluator = Evaluator(fold=KFOLD_NUM)
return
# **************************************************************************** #
# #
# ::: :::::::: #
# test.py :+: :+: :+: #
# +:+ +:+ +:+ #
# By: tbrizon <*****@*****.**> +#+ +:+ +#+ #
# +#+#+#+#+#+ +#+ #
# Created: 2019/11/05 16:53:07 by tbrizon #+# #+# #
# Updated: 2019/11/05 17:52:45 by tbrizon ### ########.fr #
# #
# **************************************************************************** #
from eval import Evaluator
if __name__ == "__main__":
words = ["Le", "Lorem", "Ipsum", "est", "simple"]
coef = [1.0, 2.0, 1.0, 4.0, 0.5]
a = Evaluator(words, coef)
Evaluator.zip_evaluate(coef, words)
Evaluator.enumerate_evaluate(coef, words)
def test(args):
"""Run model testing."""
model_args = args.model_args
data_args = args.data_args
logger_args = args.logger_args
# import pdb; pdb.set_trace()
# Get logger.
logger = Logger(logger_args.log_path, logger_args.save_dir,
logger_args.results_dir)
# Get image paths corresponding to predictions for logging
paths = None
if model_args.config_path is not None:
# Instantiate the EnsemblePredictor class for obtaining
# model predictions.
predictor = EnsemblePredictor(config_path=model_args.config_path,
model_args=model_args,
data_args=data_args,
gpu_ids=args.gpu_ids,
device=args.device,
logger=logger)
# Obtain ensemble predictions.
# Caches both individual and ensemble predictions.
# We always turn off caching to ensure that we write the Path column.
predictions, groundtruth, paths = predictor.predict(cache=False,
return_paths=True,
all_gt_tasks=True)
else:
# Load the model at ckpt_path.
ckpt_path = model_args.ckpt_path
ckpt_save_dir = Path(ckpt_path).parent
model_uncertainty = model_args.model_uncertainty
# Get model args from checkpoint and add them to
# command-line specified model args.
model_args, transform_args\
= ModelSaver.get_args(cl_model_args=model_args,
dataset=data_args.dataset,
ckpt_save_dir=ckpt_save_dir,
model_uncertainty=model_uncertainty)
# TODO JBY: in test moco should never be true.
model_args.moco = args.model_args.moco
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=False)
# Instantiate the Predictor class for obtaining model predictions.
predictor = Predictor(model=model, device=args.device)
# Get phase loader object.
return_info_dict = True
loader = get_loader(phase=data_args.phase,
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=return_info_dict,
logger=logger)
# Obtain model predictions.
if return_info_dict:
predictions, groundtruth, paths = predictor.predict(loader)
else:
predictions, groundtruth = predictor.predict(loader)
# print(predictions[CHEXPERT_COMPETITION_TASKS])
if model_args.calibrate:
#open the json file which has the saved parameters
import json
with open(CALIBRATION_FILE) as f:
data = json.load(f)
i = 0
#print(predictions)
import math
def sigmoid(x):
return 1 / (1 + math.exp(-x))
for column in predictions:
predictions[column] = predictions[column].apply \
(lambda x: sigmoid(x * data[i][0][0][0] \
+ data[i][1][0]))
i += 1
# print(predictions[CHEXPERT_COMPETITION_TASKS])
#run forward on all the predictions in each row of predictions
# Log predictions and groundtruth to file in CSV format.
logger.log_predictions_groundtruth(predictions, groundtruth, paths)
if not args.inference_only:
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger, operating_points_path=CHEXPERT_RAD_PATH)
# Get model metrics and curves on the phase dataset.
metrics, curves = evaluator.evaluate_tasks(groundtruth, predictions)
# Log metrics to stdout and file.
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(metrics, save_csv=True)
# TODO: make this work with ensemble
# TODO: investigate if the eval_loader can just be the normal loader here
if logger_args.save_cams:
cams_dir = logger_args.save_dir / 'cams'
print(f'Save cams to {cams_dir}')
save_grad_cams(args,
loader,
model,
cams_dir,
only_competition=logger_args.only_competition_cams,
only_top_task=False)
logger.log("=== Testing Complete ===")
#QUESTION 2
print("Question 2")
print("Training the tree with two different methods")
print("Training the decision tree...")
classifier = classifier.train(x, y)
print("Loading the test set...")
filename = "data/test.txt"
x_test, y_test = classifier.load_data(filename)
print("\nPredicting on test.txt data with 4 different trees")
#Load the evaulator class
eval = Evaluator()
prune = Pruning()
print("\nTree 2 unpruned")
tree_3 = np.load('simple_tree.npy', allow_pickle=True).item()
predictions = classifier.predict(x_test)
confusion = eval.confusion_matrix(predictions, y_test)
accuracy_3 = eval.accuracy(confusion)
print("number of leaves:", prune.count_leaves(tree_3))
print("Tree 2 unpruned Accuracy: " + str(np.round(accuracy_3 * 100, 2)))
print("\nTree 2 pruned")
tree_4 = np.load('simple_tree_pruned.npy', allow_pickle=True).item()
predictions = classifier.predict(x_test, tree_4)
confusion = eval.confusion_matrix(predictions, y_test)
accuracy_4 = eval.accuracy(confusion)
if __name__ == "__main__":
print("Loading the datasets...")
trainingData = dataReader.parseFile("data/train_full.txt")
validationData = dataReader.parseFile("data/validation.txt")
testData = dataReader.parseFile("data/test.txt")
print("Training the decision tree...")
classifier = DecisionTreeClassifier()
classifier = classifier.train(trainingData[0], trainingData[1])
predictions = classifier.predict(testData[0])
print("Pre prunning predictions: {}".format(predictions))
print("Evaluating test predictions...")
evaluator = Evaluator()
confusion = evaluator.confusion_matrix(predictions, testData[1])
printMetric(confusion)
print("Pruning the decision tree...")
classifier.prune(validationData)
predictions = classifier.predict(testData[0])
print("Post prunning predictions: {}".format(predictions))
print("Evaluating test predictions...")
evaluator = Evaluator()
confusion = evaluator.confusion_matrix(predictions, testData[1])
printMetric(confusion)
classifier.plot_tree()
def train(args):
"""Run model training."""
print("Start Training ...")
# Get nested namespaces.
model_args = args.model_args
logger_args = args.logger_args
optim_args = args.optim_args
data_args = args.data_args
transform_args = args.transform_args
# Get logger.
print('Getting logger... log to path: {}'.format(logger_args.log_path))
logger = Logger(logger_args.log_path, logger_args.save_dir)
# For conaug, point to the MOCO pretrained weights.
if model_args.ckpt_path and model_args.ckpt_path != 'None':
print("pretrained checkpoint specified : {}".format(
model_args.ckpt_path))
# CL-specified args are used to load the model, rather than the
# ones saved to args.json.
model_args.pretrained = False
ckpt_path = model_args.ckpt_path
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=True)
if not model_args.moco:
optim_args.start_epoch = ckpt_info['epoch'] + 1
else:
optim_args.start_epoch = 1
else:
print(
'Starting without pretrained training checkpoint, random initialization.'
)
# If no ckpt_path is provided, instantiate a new randomly
# initialized model.
model_fn = models.__dict__[model_args.model]
if data_args.custom_tasks is not None:
tasks = NamedTasks[data_args.custom_tasks]
else:
tasks = model_args.__dict__[TASKS] # TASKS = "tasks"
print("Tasks: {}".format(tasks))
model = model_fn(tasks, model_args)
model = nn.DataParallel(model, args.gpu_ids)
# Put model on gpu or cpu and put into training mode.
model = model.to(args.device)
model.train()
print("========= MODEL ==========")
print(model)
# Get train and valid loader objects.
train_loader = get_loader(phase="train",
data_args=data_args,
transform_args=transform_args,
is_training=True,
return_info_dict=False,
logger=logger)
valid_loader = get_loader(phase="valid",
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=False,
logger=logger)
# Instantiate the predictor class for obtaining model predictions.
predictor = Predictor(model, args.device)
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger)
# Get the set of tasks which will be used for saving models
# and annealing learning rate.
eval_tasks = EVAL_METRIC2TASKS[optim_args.metric_name]
# Instantiate the saver class for saving model checkpoints.
saver = ModelSaver(save_dir=logger_args.save_dir,
iters_per_save=logger_args.iters_per_save,
max_ckpts=logger_args.max_ckpts,
metric_name=optim_args.metric_name,
maximize_metric=optim_args.maximize_metric,
keep_topk=logger_args.keep_topk)
# TODO: JBY: handle threshold for fine tuning
if model_args.fine_tuning == 'full': # Fine tune all layers.
pass
else:
# Freeze other layers.
models.PretrainedModel.set_require_grad_for_fine_tuning(
model, model_args.fine_tuning.split(','))
# Instantiate the optimizer class for guiding model training.
optimizer = Optimizer(parameters=model.parameters(),
optim_args=optim_args,
batch_size=data_args.batch_size,
iters_per_print=logger_args.iters_per_print,
iters_per_visual=logger_args.iters_per_visual,
iters_per_eval=logger_args.iters_per_eval,
dataset_len=len(train_loader.dataset),
logger=logger)
if model_args.ckpt_path and not model_args.moco:
# Load the same optimizer as used in the original training.
optimizer.load_optimizer(ckpt_path=model_args.ckpt_path,
gpu_ids=args.gpu_ids)
model_uncertainty = model_args.model_uncertainty
loss_fn = evaluator.get_loss_fn(
loss_fn_name=optim_args.loss_fn,
model_uncertainty=model_args.model_uncertainty,
mask_uncertain=True,
device=args.device)
# Run training
while not optimizer.is_finished_training():
optimizer.start_epoch()
# TODO: JBY, HACK WARNING # What is the hack?
metrics = None
for inputs, targets in train_loader:
optimizer.start_iter()
if optimizer.global_step and optimizer.global_step % optimizer.iters_per_eval == 0 or len(
train_loader.dataset
) - optimizer.iter < optimizer.batch_size:
# Only evaluate every iters_per_eval examples.
predictions, groundtruth = predictor.predict(valid_loader)
# print("predictions: {}".format(predictions))
metrics, curves = evaluator.evaluate_tasks(
groundtruth, predictions)
# Log metrics to stdout.
logger.log_metrics(metrics)
# Add logger for all the metrics for valid_loader
logger.log_scalars(metrics, optimizer.global_step)
# Get the metric used to save model checkpoints.
average_metric = evaluator.evaluate_average_metric(
metrics, eval_tasks, optim_args.metric_name)
if optimizer.global_step % logger_args.iters_per_save == 0:
# Only save every iters_per_save examples directly
# after evaluation.
print("Save global step: {}".format(optimizer.global_step))
saver.save(iteration=optimizer.global_step,
epoch=optimizer.epoch,
model=model,
optimizer=optimizer,
device=args.device,
metric_val=average_metric)
# Step learning rate scheduler.
optimizer.step_scheduler(average_metric)
with torch.set_grad_enabled(True):
logits, embedding = model(inputs.to(args.device))
loss = loss_fn(logits, targets.to(args.device))
optimizer.log_iter(inputs, logits, targets, loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
optimizer.end_iter()
optimizer.end_epoch(metrics)
logger.log('=== Training Complete ===')
def main():
parser = argparse.ArgumentParser(description='CNNF-TTT testing')
parser.add_argument('--dataset',
choices=['cifar10', 'fashion'],
default='cifar10',
help='the dataset for training the model')
parser.add_argument(
'--test',
choices=['average', 'last'],
default='average',
help='output averaged logits or logits from the last iteration')
parser.add_argument('--csv-dir',
default='results.csv',
help='Directory for Saving the Evaluation results')
parser.add_argument('--model-dir',
default='models',
help='Directory for Saved Models')
args = parser.parse_args()
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
clean_dir = 'data/'
# load in corrupted data
if args.dataset == 'cifar10':
dataloader = torch.utils.data.DataLoader(datasets.CIFAR10(
clean_dir,
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])),
batch_size=64,
shuffle=True,
num_workers=4,
pin_memory=True)
eps = 0.063
eps_iter = 0.02
nb_iter = 7
elif args.dataset == 'fashion':
dataloader = torch.utils.data.DataLoader(datasets.FashionMNIST(
clean_dir,
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, ), (0.5, )),
AddGaussianNoise(0., 0.5),
])),
batch_size=100,
shuffle=True)
eps = 0.025
eps_iter = 0.071
nb_iter = 7
log_acc_path = args.csv_dir
evalmethod = args.test
model_dir = args.model_dir
with open(log_acc_path, 'a') as f:
f.write(',clean,pgd_first,pgd_last,spsa_first,spsa_last,transfer,')
f.write('\n')
# Model to evaluate
if args.dataset == 'cifar10':
model_name = 'CNNF_2_cifar.pt'
model = WideResNet(40, 10, 2, 0.0, ind=5, cycles=2,
res_param=0.1).to(device)
elif args.dataset == 'fashion':
model_name = 'CNNF_1_fmnist.pt'
model = CNNF(10, ind=2, cycles=1, res_param=0.1).to(device)
model_path = os.path.join(model_dir, model_name)
model.load_state_dict(torch.load(model_path))
eval = Evaluator(device, model)
corrupted_acc = eval.corrupted_accuracy(dataloader)
optimizer = torch.optim.SGD(model.parameters(),
0.05,
momentum=0.9,
weight_decay=5e-4)
spsa_acc_ete = eval.ttt_accuracy(dataloader,
optimizer,
per_image=False,
batch_size=10)
with open(log_acc_path, 'a') as f:
f.write('%s,' % model_name)
#f.write('%0.2f,' % (100. * clean_acc))
#f.write('%0.2f,' % (100. * pgd_acc_first))
# f.write('%0.2f,' % (100. * pgd_acc_ete))
#f.write('%0.2f,' % (100. * spsa_acc_first))
# f.write('%0.2f,' % (100. * spsa_acc_ete))
# f.write('%0.2f,' % (100. * transfer_acc))
f.write('\n')
def main():
args = parse_args()
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.ERROR)
if args.predict_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if args.output_dir and os.path.exists(args.output_dir) and \
os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
if args.overwrite_output_dir and os.path.isdir(args.output_dir):
shutil.rmtree(args.output_dir)
os.mkdir(args.output_dir)
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
with open(os.path.join(args.output_dir, 'args.txt'), 'w') as f:
f.write(str(args))
for key, val in vars(args).items():
logger.info(f"{key} - {val}")
try:
write_meta_data(args.output_dir, args)
except git.exc.InvalidGitRepositoryError:
logger.info("didn't save metadata - No git repo!")
logger.info(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, amp training: %s",
args.local_rank, device, args.n_gpu, bool(args.local_rank != -1),
args.amp)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
# Barrier to make sure only the first process in distributed training download model & vocab
torch.distributed.barrier()
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
config = CONFIG_MAPPING[args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
#config.hidden_dropout_prob = args.encoder_dropout_prob
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
config_class = LongformerConfig
base_model_prefix = "longformer"
S2E.config_class = config_class
S2E.base_model_prefix = base_model_prefix
model = S2E.from_pretrained(args.model_name_or_path,
config=config,
cache_dir=args.cache_dir,
args=args)
model.to(args.device)
if args.local_rank == 0:
# End of barrier to make sure only the first process in distributed training download model & vocab
torch.distributed.barrier()
logger.info("Training/evaluation parameters %s", args)
evaluator = Evaluator(args, tokenizer)
# Training
if args.do_train:
train_dataset = get_dataset(args, tokenizer, evaluate=False)
global_step, tr_loss = train(args, train_dataset, model, tokenizer,
evaluator)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
# Saving best-practices: if you use save_pretrained for the model and tokenizer,
# you can reload them using from_pretrained()
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Create output directory if needed
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = model.module if hasattr(
model,
'module') else model # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
result = evaluator.evaluate(model,
prefix="final_evaluation",
official=True)
results.update(result)
return results
return results
