def nearsub(args, train_features, train_labels, test_features, test_labels):
"""Perform nearest subspace classification.
Options:
n_comp (int): number of components for PCA or SVD
"""
scores_pca = []
scores_svd = []
num_classes = train_labels.numpy().max() + 1 # should be correct most of the time
features_sort, _ = utils.sort_dataset(train_features.numpy(), train_labels.numpy(),
num_classes=num_classes, stack=False)
for j in range(num_classes):
pca = PCA(n_components=args.n_comp).fit(features_sort[j])
pca_subspace = pca.components_.T
mean = np.mean(features_sort[j], axis=0)
pca_j = (np.eye(params["fd"]) - pca_subspace @ pca_subspace.T) \
@ (test_features.numpy() - mean).T
score_pca_j = np.linalg.norm(pca_j, ord=2, axis=0)
svd = TruncatedSVD(n_components=args.n_comp).fit(features_sort[j])
svd_subspace = svd.components_.T
svd_j = (np.eye(params["fd"]) - svd_subspace @ svd_subspace.T) \
@ (test_features.numpy()).T
score_svd_j = np.linalg.norm(svd_j, ord=2, axis=0)
scores_pca.append(score_pca_j)
scores_svd.append(score_svd_j)
test_predict_pca = np.argmin(scores_pca, axis=0)
test_predict_svd = np.argmin(scores_svd, axis=0)
acc_pca = utils.compute_accuracy(test_predict_pca, test_labels.numpy())
acc_svd = utils.compute_accuracy(test_predict_svd, test_labels.numpy())
print('PCA: {}'.format(acc_pca))
print('SVD: {}'.format(acc_svd))
return acc_pca
def fit(self, X, y, X_valid, y_valid):
for epoch in range(self.num_of_epochs):
print("epoch number: " + str(epoch + 1))
permuted_indices = np.random.permutation(X.shape[0])
for i in range(0, X.shape[0], self.batch_size):
selected_data_points = np.take(permuted_indices,
range(i, i + self.batch_size),
mode='wrap')
delta_w = self._d_cost(X[selected_data_points],
y[selected_data_points], self.weights)
self.weights -= delta_w * self.learning_rate
training_accuracy = compute_accuracy(self.predict(X),
np.argmax(y, 1))
validation_accuracy = compute_accuracy(self.predict(X_valid),
np.argmax(y_valid, 1))
print("training accuracy: " + str(round(training_accuracy, 2)))
print("validation accuracy: " + str(round(validation_accuracy, 2)))
print("cost: " + str(self._cost(X, y, self.weights)))
if self.validation_accuracy < validation_accuracy:
self.validation_accuracy = validation_accuracy
self.old_weights = self.weights
else:
self.weights = self.old_weights
self.learning_rate = 0.5 * self.learning_rate
def fit(self, X, y, X_valid, y_valid):
for epoch in range(self.num_of_epochs):
print("epoch number: " + str(epoch + 1))
permuted_indices = np.random.permutation(X.shape[0])
for i in range(0, X.shape[0], self.batch_size):
selected_data_points = np.take(permuted_indices, range(i, i+self.batch_size), mode='wrap')
delta_w = self._d_cost(X[selected_data_points], y[selected_data_points], self.weights)
for w, d in zip(self.weights, delta_w):
w -= d*self.learning_rate
for i in range(len(self.weights)):
self.ema_weights[i] = self.ema_weights[i]*self.ema + self.weights[i]*(1-self.ema)
training_accuracy = compute_accuracy(self.predict(X, self.weights), np.argmax(y, 1))
validation_accuracy = compute_accuracy(self.predict(X_valid, self.weights), np.argmax(y_valid, 1))
print("training accuracy: " + str(round(training_accuracy, 2)))
print("validation accuracy: " + str(round(validation_accuracy, 2)))
print("cost: " + str(self._cost(X, y, self.weights)))
training_accuracy = compute_accuracy(self.predict(X, self.ema_weights), np.argmax(y, 1))
validation_accuracy = compute_accuracy(self.predict(X_valid, self.ema_weights), np.argmax(y_valid, 1))
print("training accuracy ema: " + str(round(training_accuracy, 2)))
print("validation accuracy ema: " + str(round(validation_accuracy, 2)))
self.save_average_and_std(X)
def train_model():
global best_acc, best_epoch
batch_idx = 0
model.train()
N = len(train_loader.dataset)
train_loss, all_preds, all_targets = 0., [], []
val_preds, val_targets = [], []
for batch in train_loader:
optimizer.zero_grad()
loss, output = model(batch)
#
if params.task == '1':
target = batch['labels'].numpy()
valid_mask = batch['valid_mask'].numpy()
test_mask = batch['test_mask'].numpy()
validation_flag = (1 - valid_mask) * test_mask
training_flag = test_mask * valid_mask
elif params.task == '2':
target = batch['ans'].numpy()
valid_mask = batch['valid_mask'].numpy()
test_mask = batch['test_mask'].numpy()
validation_flag = (1 - valid_mask) * test_mask
training_flag = test_mask * valid_mask
loss.backward()
optimizer.step()
all_preds.append(output[training_flag == 1])
all_targets.append(target[training_flag == 1])
val_preds.append(output[validation_flag == 1])
val_targets.append(target[validation_flag == 1])
train_loss += float(loss.detach().cpu().numpy())
batch_idx += 1
all_pred = np.concatenate(all_preds, axis=0)
all_target = np.concatenate(all_targets, axis=0)
val_pred = np.concatenate(val_preds, axis=0)
val_target = np.concatenate(val_targets, axis=0)
#model.eval()
if params.task == '1':
train_auc = compute_auc(all_target, all_pred)
val_auc = compute_auc(val_target, val_pred)
train_accuracy = compute_accuracy(all_target, all_pred)
val_accuracy = compute_accuracy(val_target, val_pred)
print(
'Train Epoch {} Loss: {} train auc: {} train acc: {} val auc: {} val accuracy: {} n_validation : {}'
.format(epoch, train_loss / batch_idx, train_auc, train_accuracy,
val_auc, val_accuracy, val_target.shape))
if params.task == '2':
train_accuracy = np.mean(all_target == all_pred)
val_accuracy = np.mean(val_target == val_pred)
print('Train Epoch {} Loss: {} train acc: {} val accuracy: {}'.format(
epoch, train_loss / batch_idx, train_accuracy, val_accuracy))
if best_acc is None or val_accuracy > best_acc:
best_acc = val_accuracy
best_epoch = epoch
print('Train Epoch {} best val accuracy: {} best epoch: {}'.format(
epoch, best_acc, best_epoch))
def test(test_loader, model, args):
print('Testing...')
losses = AverageMeter()
accuracy = AverageMeter()
# Switch to evaluate mode
model.eval()
with torch.no_grad():
for n_episode, batch in enumerate(test_loader, 1):
data, _ = [_.cuda(non_blocking=True) for _ in batch]
p = args.n_support * args.n_way
data_support, data_query = data[:p], data[p:]
# Compute class prototypes (n_way, output_dim)
class_prototypes = model(data_support).reshape(args.n_support, args.n_way, -1).mean(dim=0)
# Generate labels (n_way, n_query)
labels = torch.arange(args.n_way).repeat(args.n_query)
labels = labels.type(torch.cuda.LongTensor)
# Compute loss and metrics
logits = euclidean_dist(model(data_query), class_prototypes)
loss = F.cross_entropy(logits, labels)
acc = compute_accuracy(logits, labels)
# Record loss and accuracy
losses.update(loss.item(), data_query.size(0))
accuracy.update(acc, data_query.size(0))
print('Test Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Test Accuracy {accuracy.val:.3f} ({accuracy.avg:.3f})\t'.format(loss=losses, accuracy=accuracy))
return losses.avg, accuracy.avg
def test(DCN, gen):
import pdb
accuracies_test = [[] for ii in gen.scales['test']]
iterations_te = int(gen.num_examples_test / batch_size)
for it in range(iterations_te):
for i, scales in enumerate(gen.scales['test']):
# depth tells how many times the dynamic model will be unrolled
depth = 1
if args.dynamic:
depth = scales
_, length = gen.compute_length(scales, mode='test')
DCN.merge.n, DCN.split.n = [length] * 2
input, tar = gen.get_batch(batch=it, scales=scales, mode='test')
# forward DCN
out = DCN(input,
tar,
length,
depth,
it=it,
random_split=args.random_split,
mode='test',
dynamic=args.dynamic)
Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out
acc = utils.compute_accuracy(Perms[-1], target)
accuracies_test[i].append(acc)
print(sum(accuracies_test[i]) / float(it + 1))
accuracies_test = [sum(accs) / iterations_te for accs in accuracies_test]
print('acc test:', accuracies_test)
return accuracies_test
def test(DCN, gen):
accuracies_test = [[] for ii in gen.scales['test']]
iterations_te = int(gen.num_examples_test / batch_size)
for it in range(iterations_te):
for i, scales in enumerate(gen.scales['test']):
# depth tells how many times the dynamic model will be unrolled
depth = 1
if args.dynamic:
depth = scales
_, length = gen.compute_length(scales, mode='test')
DCN.merge.n, DCN.split.n = [length] * 2
input, tar = gen.get_batch(batch=it, scales=scales,
mode='test')
# forward DCN
out = DCN(input, tar, length, depth, it=it,
random_split=args.random_split,
mode='test', dynamic=args.dynamic)
Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out
acc = utils.compute_accuracy(Perms[-1], target)
accuracies_test[i].append(acc)
print(sum(accuracies_test[i]) / float(it + 1))
accuracies_test = [sum(accs) / iterations_te
for accs in accuracies_test]
print('acc test:', accuracies_test)
return accuracies_test
def get_object_confusion(class1, class2, similarity_model, config):
model_config = config['model']
benchmark_config = config['benchmark']
model_path = model_config['model_filename']
dataset_path = benchmark_config['dataset_path']
params = {
'dim': model_config['input_shape'],
'batch_size': benchmark_config['batch_size'],
'shuffle': False
}
test_dataset = ImageDataset(dataset_path, 'validation')
test_dataset.prepare_specific(benchmark_config['test_cases'] // 2, class1,
class2)
test_generator = DataGenerator(test_dataset, **params)
preds = np.array([])
gts = np.array([])
for i in tqdm(range(len(test_generator))):
batch = test_generator[i]
pred = similarity_model.predict_on_batch(batch[0])
preds = np.append(preds, pred.flatten())
gts = np.append(gts, batch[1])
if benchmark_config['vis_output'] and not i % benchmark_config[
'test_cases'] // (5 * benchmark_config['batch_size']):
show_output(batch[0][0], batch[0][1], pred, batch[1])
te_acc = compute_accuracy(preds, gts)
print("Class 1: " + class1 + ", Class2: " + class2 +
", Distinguishability Score: " + str(te_acc))
return te_acc
def test(args):
model = NTMOneShotLearningModel(args)
data_loader = OmniglotDataLoader(args)
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(args.save_dir + '/' + args.model +
'_' + args.label_type)
with tf.Session() as sess:
saver.restore(sess, ckpt.model_checkpoint_path)
print(
"Test Result\n1st\t2nd\t3rd\t4th\t5th\t6th\t7th\t8th\t9th\t10th\tloss"
)
y_list = []
output_list = []
loss_list = []
for episode in range(args.test_batch_num):
x_image, x_label, y = data_loader.fetch_batch(
args,
mode='test',
augment=args.augment,
sample_strategy=args.sample_strategy)
feed_dict = {
model.x_image: x_image,
model.x_label: x_label,
model.y: y
}
output, learning_loss = sess.run([model.o, model.loss],
feed_dict=feed_dict)
y_list.append(y)
output_list.append(output)
loss_list.append(learning_loss)
accuracy = compute_accuracy(args, np.concatenate(y_list, axis=0),
np.concatenate(output_list, axis=0))
for accu in accuracy:
print('%.4f' % accu)
print(np.mean(loss_list))
def main():
for file in [
'data/breast-cancer-assignment5.txt', 'data/german-assignment5.txt'
]:
data, labels, types = load_matrix_from_txt(file)
splices = k_fold_split(10, data, labels)
accuracies = []
for i in range(10):
train_indexes = splices[i][0]
test_indexes = splices[i][1]
train_data = np.copy(data[train_indexes])
train_label = np.copy(labels[train_indexes])
test_data = np.copy(data[test_indexes])
test_label = np.copy(labels[test_indexes])
boost = AdaBoost()
boost.train(train_data, train_label, types)
class_result = boost.test(test_data)
accuracy = compute_accuracy(class_result, test_label)
accuracies.append(accuracy)
print 'accuracy: %f' % accuracy
print('file: {}, mean: {}, std: {}'.format(file, np.mean(accuracies),
np.std(accuracies)))
def add_test_accuracy(self,
pred,
labels,
perms,
W,
cities,
oracle_costs,
last=False,
beam_size=2):
accuracy = utils.compute_accuracy(pred, labels)
costs, Paths = utils.beamsearch_hamcycle(pred.data,
W.data,
beam_size=beam_size)
self.accuracy_test_aux.append(accuracy)
self.cost_test_aux.append(np.array(costs.cpu().numpy()).mean())
self.cost_test_aux_oracle.append(np.array(oracle_costs).mean())
if last:
accuracy_test = np.array(self.accuracy_test_aux).mean()
self.accuracy_test.append(accuracy_test)
self.accuracy_test_aux = []
cost_test = np.array(self.cost_test_aux).mean()
self.cost_test.append(cost_test)
self.cost_test_aux = []
cost_test_oracle = np.array(self.cost_test_aux_oracle).mean()
self.cost_test_oracle.append(cost_test_oracle)
self.cost_test_aux_oracle = []
self.plot_example(Paths, costs, oracle_costs, perms, cities)
def test_step(model, test_loader, loss_func, device, epoch, results):
losses = AverageMeter("Loss", ':.4e')
top1 = AverageMeter("Acc@1", ':6.2f')
num_steps = len(test_loader)
model.eval()
with torch.no_grad():
for i, (data, labels) in enumerate(test_loader):
data, labels = data.to(device), labels.to(device)
predictions = model(data)
loss = loss_func(predictions, labels)
# Metrics
acc1 = compute_accuracy(predictions, labels, topk=(1, ))[0]
losses.update(loss.item(), data.size(0))
top1.update(acc1.item(), data.size(0))
print(
"[Epoch {}; Step {}/{}] Val Loss: {:.6f}; Val Accuracy: {:.6f}"\
.format(epoch+1, i+1, num_steps, loss, acc1)
)
sys.stdout.flush()
results["val"]["losses"].append(losses.avg)
results["val"]["top1_accs"].append(top1.avg)
# Save losses and accuracies every epoch so we can plot loss and accuracy
pickle.dump(results, open(results["fname"], "wb"))
return losses.avg, top1.avg
def test(DCN, gen):
accuracies_test = [[] for ii in gen.scales['test']]
iterations_te = int(gen.num_examples_test / batch_size)
for it in xrange(iterations_te):
for i, scales in enumerate(gen.scales['test']):
# depth tells how many times the dynamic model will be unrolled
depth = 1
if args.dynamic:
# at test time, split deeper
depth = scales + 2
_, length = gen.compute_length(scales)
DCN.merge.n, DCN.split.n = [length] * 2
input, tar = gen.get_batch(batch=it, scales=scales, mode='test')
# forward DCN
out = DCN(input,
tar,
length,
depth,
it=it,
mergesort_split=args.mergesort_split,
quicksort_merge=args.quicksort_merge,
mode='test',
dynamic=args.dynamic)
Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out
if not args.quicksort_merge:
acc = utils.compute_accuracy(Perms[-1], target)
else:
acc = 1 - loss.data.cpu().numpy()
accuracies_test[i].append(acc)
accuracies_test = [sum(accs) / iterations_te for accs in accuracies_test]
return accuracies_test
def build_model(self,):
utils.prepare_data(data_file=self.data_file)
self.lap_list, self.feature = utils.load_gcn_data(self.graph_file, self.num_support)
self.num_feature = self.feature.shape[1]
self.x = tf.placeholder(tf.float32, [None, self.d_input_step, self.d_input_size])
self.z = tf.placeholder(tf.float32, [None, self.g_input_step, self.g_input_size])
self.z_t = tf.placeholder(tf.float32, [None, self.g_input_step, self.g_input_size])
self.lap = tf.placeholder(tf.float32, [self.num_support, self.d_input_size, self.d_input_size])
self.fea = tf.placeholder(tf.float32, [self.d_input_size, self.num_feature])
self.x_ = self.generator(self.z, self.g_input_step, self.g_input_size, self.g_hidden_size, self.g_batch_size)
self.D = self.discriminator(self.x, self.d_input_step, self.d_input_size, self.d_hidden_size, 1, self.g_batch_size)
self.D_ = self.discriminator(self.x_, self.d_input_step, self.d_input_size, self.d_hidden_size, 1, self.g_batch_size, reuse=True)
if self.wgan == 1:
self.d_loss_real = tf.reduce_mean(self.D)
self.d_loss_fake = tf.reduce_mean(self.D_)
self.g_loss = self.d_loss_fake
self.d_loss = self.d_loss_real - self.d_loss_fake
else:
self.d_loss_real = utils.compute_loss(self.D, tf.ones_like(self.D))
self.d_loss_fake = utils.compute_loss(self.D_, tf.zeros_like(self.D_))
self.g_loss = utils.compute_loss(self.D_, tf.ones_like(self.D_))
self.d_loss = self.d_loss_real + self.d_loss_fake
self.accuracy = utils.compute_accuracy(self.z_t, self.z_)
def build(self):
self.add_placeholders()
self.keep_prob = tf.placeholder(tf.float32)
self.pred = self.add_prediction_op()
self.loss = self.add_loss_op(self.pred)
self.train_op = self.add_training_op(self.loss)
self.accuracy = utils.compute_accuracy(self.pred, self.y_)
def test_knn():
data, _ = utils.read_table("combined_data_normalized.csv", True)
class_index = 4
predictors = [2, 3, 5, 9]
results = knn.knn_classifier(data, class_index, predictors, 5, 5)
accuracy = utils.compute_accuracy(results)
print(accuracy)
utils.confusion_matrix(results, "Crime Rate?",
"KNN Classifier Prediction of Crime Rate")
def test_naive_bayes():
data, header = utils.read_table("combined_data_normalized.csv", True)
class_index = 4
predictors = [2, 3, 5, 7]
results = bayes.naive_bayes_classifier(data, header, 10, class_index,
predictors, [2, 3, 5, 9])
accuracy = utils.compute_accuracy(results)
print(accuracy)
utils.confusion_matrix(results, "Crime Rate?",
"Naive Bayes Classifier Prediction of Crime Rate")
def fit(self, X_train, y_train, X_valid, y_valid):
best_validation_accuracy = 0
for epoch in range(self.num_of_epochs):
print("epoch number: " + str(epoch + 1))
permuted_indices = np.random.permutation(X_train.shape[0])
for i in tqdm(range(0, X_train.shape[0], self.batch_size)):
selected_data_points = np.take(permuted_indices,
range(i, i + self.batch_size),
mode='wrap')
self.sess.run(
self.optimizer, {
self.X: X_train[selected_data_points],
self.y: y_train[selected_data_points],
self.prob: self.keep_prob,
self.is_training: True
})
training_predictions = self.get_predictions(X_train)
training_accuracy = compute_accuracy(training_predictions,
np.argmax(y_train, 1))
validation_preditions = self.get_predictions(X_valid)
validation_accuracy = compute_accuracy(validation_preditions,
np.argmax(y_valid, 1))
if validation_accuracy > best_validation_accuracy:
best_validation_accuracy = validation_accuracy
self.saver.save(sess=self.sess,
save_path=self.save_dir + "best_model")
print("training accuracy: " + str(round(training_accuracy, 2)))
print("validation accuracy: " + str(round(validation_accuracy, 2)))
summary = tf.Summary(value=[
tf.Summary.Value(tag="training accuracy",
simple_value=training_accuracy)
])
self.writer.add_summary(summary, epoch)
summary = tf.Summary(value=[
tf.Summary.Value(tag="validation accuracy",
simple_value=validation_accuracy)
])
self.writer.add_summary(summary, epoch)
def test_decision_tree():
data, header = utils.read_table("combined_data_discretized.csv", True)
class_index = 4
predictors = [2, 3, 5, 9]
results = dtree.decision_tree_classifier(data, header, class_index,
predictors, 30)
accuracy = utils.compute_accuracy(results)
print(accuracy)
utils.confusion_matrix(
results, "Crime Rate?",
"Decision Tree Classifier Prediction of Crime Rate")
def test_random_forest():
data, header = utils.read_table("combined_data_discretized.csv", True)
class_index = 4
predictors = [2, 3, 5, 9]
results = rforest.random_forest_classifier(data, header, class_index,
predictors, 100, 25, 3)
accuracy = utils.compute_accuracy(results)
print(accuracy)
utils.confusion_matrix(
results, "Crime Rate?",
"Random Forest Classifier Prediction of Crime Rate")
def src_supervised_step(self, src_end_points, src_labels):
# compute source classification loss
src_classification_loss = self.class_criterion(
src_end_points['logits'], src_labels)
self.losses_dict['src_classification_loss'] = src_classification_loss
# compute source train accuracy
src_train_accuracy = compute_accuracy(src_end_points['logits'],
src_labels,
acc_metric=self.acc_metric)
self.src_train_acc_queue.put(src_train_accuracy)
def eval_tgt(self, tgt_test_collection):
tgt_test_acc = compute_accuracy(tgt_test_collection['logits'],
tgt_test_collection['true_labels'],
acc_metric=self.acc_metric,
print_result=True)
tgt_test_acc = round(tgt_test_acc, 3)
self.acc_dict['tgt_test_acc'] = tgt_test_acc
# self.acc_dict['tgt_best_test_acc'] = max(self.acc_dict['tgt_best_test_acc'], tgt_test_acc)
if self.acc_dict['tgt_test_acc'] > self.acc_dict['tgt_best_test_acc']:
self.acc_dict['tgt_best_test_acc'] = self.acc_dict['tgt_test_acc']
self.save_checkpoint()
self.print_acc()
def validate(val_loader, model, att, args):
print('Validating...')
losses = AverageMeter()
accuracy = AverageMeter()
# Switch to evaluate mode
model.eval()
att.eval()
with torch.no_grad():
for n_episode, batch in enumerate(val_loader, 1):
data, _ = [_.cuda(non_blocking=True) for _ in batch]
p = args.n_support * args.n_way_val
data_support, data_query = data[:p], data[p:]
# Compute class prototypes (n_way, output_dim)
# Calculate weighted averages for class prototypes
# (n_support, n_way_val feature_dimension)
latent_vecs_val = model(data_support).reshape(
args.n_support, args.n_way_val, -1)
# (n_way_train, n_val, feature_dimension)
latent_vecs_val = latent_vecs_val.transpose(0, 1)
# _, scores_val = att(latent_vecs_val)
scores_val = F.softmax(att(latent_vecs_val), dim=1)
# scores_val = scores_val.unsqueeze(-1).expand_as(latent_vecs_val)
scores_val = scores_val.expand_as(latent_vecs_val)
# class_prototypes = torch.sum(
# torch.matmul(scores_val, latent_vecs_val), 1)
class_prototypes = torch.sum(
torch.mul(scores_val, latent_vecs_val), 1)
# class_prototypes = att(model(data_support)).reshape(
# args.n_support, args.n_way_val, -1).mean(dim=0)
# Generate labels (n_way, n_query)
labels = torch.arange(args.n_way_val).repeat(args.n_query_val)
labels = labels.type(torch.cuda.LongTensor)
# Compute loss and metrics
logits = euclidean_dist(model(data_query), class_prototypes)
loss = F.cross_entropy(logits, labels)
acc = compute_accuracy(logits, labels)
# Record loss and accuracy
losses.update(loss.item(), data_query.size(0))
accuracy.update(acc, data_query.size(0))
print('Validation Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Validation Accuracy {accuracy.val:.3f} ({accuracy.avg:.3f})\t'.
format(loss=losses, accuracy=accuracy))
return losses.avg, accuracy.avg
def test(self):
"""
test func.
runs the model on the test data.
:return: list of predictions and acc
"""
y_list = []
y_hat_list = []
for ex_dict in ut.TEST_LIST:
y_list.append(ex_dict[1])
y_hat_list.append(self.predict(ex_dict[0]))
acc = ut.compute_accuracy(y_hat_list, y_list)
return y_hat_list, acc
def knn(args, train_features, train_labels, test_features, test_labels):
"""Perform k-Nearest Neighbor classification using cosine similarity as metric.
Options:
k (int): top k features for kNN
"""
sim_mat = train_features @ test_features.T
topk = sim_mat.topk(k=args.k, dim=0)
topk_pred = train_labels[topk.indices]
test_pred = topk_pred.mode(0).values.detach()
acc = utils.compute_accuracy(test_pred.numpy(), test_labels.numpy())
print("kNN: {}".format(acc))
return acc
def test(self, flags, ite, log_prefix, log_dir='logs/', batImageGenTest=None):
# switch on the network test mode
self.network.eval()
images_test, labels_test = batImageGenTest
threshold = 50
if len(images_test) > threshold:
n_slices_test = len(images_test) / threshold
indices_test = []
for per_slice in range(int(n_slices_test - 1)):
indices_test.append(len(images_test) * (per_slice + 1) / n_slices_test)
test_image_splits = np.split(images_test, indices_or_sections=indices_test)
# Verify the splits are correct
test_image_splits_2_whole = np.concatenate(test_image_splits)
assert np.all(images_test == test_image_splits_2_whole)
# split the test data into splits and test them one by one
test_image_preds = []
for test_image_split in test_image_splits:
images_test = Variable(torch.from_numpy(np.array(test_image_split, dtype=np.float32)))
outputs, end_points = self.network(images_test)
predictions = end_points['Predictions']
predictions = predictions.data.numpy()
test_image_preds.append(predictions)
# concatenate the test predictions first
predictions = np.concatenate(test_image_preds)
else:
images_test = Variable(torch.from_numpy(np.array(images_test, dtype=np.float32)))
outputs, end_points = self.network(images_test)
predictions = end_points['Predictions']
predictions = predictions.data.numpy()
accuracy = compute_accuracy(predictions=predictions, labels=labels_test)
print('----------accuracy test----------:', accuracy)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, '{}.txt'.format(log_prefix))
write_log(str('ite:{}, accuracy:{}'.format(ite, accuracy)), log_path=log_path)
# switch on the network train mode after test
self.network.train()
return accuracy
def train(self):
"""
train function.
runs the model on the train data.
:return: acc
"""
y_list = []
y_hat_list = []
for ex_dict in ut.EXAMPLES_LIST:
y_list.append(ex_dict[1])
y_hat_list.append(self.predict(ex_dict[0]))
acc = ut.compute_accuracy(y_hat_list, y_list)
return acc
def train(self,
epochs: int,
val_frequency: int,
print_frequency: int = 20,
log_frequency: int = 5,
start_epoch: int = 0):
self.model.train()
for epoch in range(start_epoch, epochs):
self.model.train()
data_load_start_time = time.time()
for i, (inputs, target, filename) in enumerate(self.train_loader):
batch = inputs.to(self.device)
labels = target.to(self.device)
data_load_end_time = time.time()
logits = self.model.forward(batch)
loss = self.criterion(logits, labels)
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
with torch.no_grad():
preds = logits.argmax(-1)
accuracy = compute_accuracy(labels, preds)
data_load_time = data_load_end_time - data_load_start_time
step_time = time.time() - data_load_end_time
if ((self.step + 1) % log_frequency) == 0:
self.log_metrics(epoch, accuracy, loss, data_load_time,
step_time)
if ((self.step + 1) % print_frequency) == 0:
self.print_metrics(epoch, accuracy, loss, data_load_time,
step_time)
self.step += 1
data_load_start_time = time.time()
if (epoch + 1) % self.checkpoint_frequency == 0 or (epoch +
1) == epochs:
print(f"Saving model to {self.checkpoint_path}")
torch.save({
"epoch": epoch,
"model": self.model.state_dict()
}, self.checkpoint_path)
self.summary_writer.add_scalar("epoch", epoch, self.step)
if ((epoch + 1) % val_frequency) == 0:
self.validate()
self.model.train()
def fit(self, X_train, y_train, X_valid, y_valid):
for epoch in range(self.num_of_epochs):
print("epoch number: " + str(epoch + 1))
permuted_indices = np.random.permutation(X_train.shape[0])
for i in range(0, X_train.shape[0], self.batch_size):
selected_data_points = np.take(permuted_indices,
range(i, i + self.batch_size),
mode='wrap')
_, image = self.sess.run(
[self.optimizer, self.image], {
self.X: X_train[selected_data_points],
self.y: y_train[selected_data_points],
self.prob: self.keep_prob,
self.is_training: True
})
training_accuracy = compute_accuracy(self.predict(X_train),
np.argmax(y_train, 1))
validation_accuracy = compute_accuracy(self.predict(X_valid),
np.argmax(y_valid, 1))
print("training accuracy: " + str(round(training_accuracy, 2)))
print("validation accuracy: " + str(round(validation_accuracy, 2)))
summary = tf.Summary(value=[
tf.Summary.Value(tag="training accuracy",
simple_value=training_accuracy)
])
self.writer.add_summary(summary, epoch)
summary = tf.Summary(value=[
tf.Summary.Value(tag="validation accuracy",
simple_value=validation_accuracy)
])
self.writer.add_summary(summary, epoch)
self.writer.add_summary(image, epoch)
def prepare_tgt_conf_dataset(self):
src_test_collection = self.collect_samples('src_test')
tgt_test_collection = self.collect_samples('tgt_test')
tgt_pseudo_probabilities = self.tgt_pseudo_labeler.pseudo_label_tgt(
src_test_collection, tgt_test_collection)
tgt_pseudo_acc = compute_accuracy(tgt_pseudo_probabilities,
tgt_test_collection['true_labels'],
acc_metric=self.acc_metric,
print_result=False)
self.acc_dict['tgt_pseudo_acc'] = tgt_pseudo_acc
self.eval_tgt(tgt_test_collection)
tgt_pseudo_confidences, tgt_pseudo_labels = torch.max(
tgt_pseudo_probabilities, dim=1)
tgt_conf_mask = tgt_pseudo_confidences.ge(self.thresh)
tgt_conf_indices = torch.tensor(range(
self.tgt_size)).cuda()[tgt_conf_mask].tolist()
tgt_conf_predictions = tgt_pseudo_labels[tgt_conf_mask].tolist()
self.tgt_conf_pair = list(zip(tgt_conf_indices, tgt_conf_predictions))
self.data_loader['tgt_conf'] = ConfidentDataLoader(
self.tgt_file,
self.train_data_loader_kwargs,
self.tgt_conf_pair,
self.min_conf_samples,
training=True)
if self.data_loader['tgt_conf'].data_loader is None:
self.data_iterator['tgt_conf'] = None
else:
self.data_iterator['tgt_conf'] = iter(self.data_loader['tgt_conf'])
tgt_non_conf_mask = tgt_pseudo_confidences.lt(self.thresh)
self.tgt_non_conf_indices = torch.tensor(range(
self.tgt_size)).cuda()[tgt_non_conf_mask].tolist()
self.data_loader['tgt_non_conf'] = NonConfidentDataLoader(
self.tgt_file,
self.train_data_loader_kwargs,
self.tgt_non_conf_indices,
training=True)
if self.data_loader['tgt_non_conf'].data_loader is None:
self.data_iterator['tgt_non_conf'] = None
else:
self.data_iterator['tgt_non_conf'] = iter(
self.data_loader['tgt_non_conf'])
def validate(opt, model, data_transform):
total_TP = 0.0
total_FP = 0.0
total_FN = 0.0
# switch to evaluate mode
model.eval()
img_dir = '{:s}/images/val'.format(opt.train['data_dir'])
label_dir = opt.test['label_dir']
img_names = os.listdir(img_dir)
for img_name in img_names:
# load test image
img_path = '{:s}/{:s}'.format(img_dir, img_name)
img = Image.open(img_path)
name = os.path.splitext(img_name)[0]
label_path = '{:s}/{:s}_label_point.png'.format(label_dir, name)
gt = io.imread(label_path)
input, label = data_transform((img, Image.fromarray(gt)))
input = input.unsqueeze(0)
prob_map = get_probmaps(input, model, opt)
prob_map = prob_map.cpu().numpy()
pred = prob_map > opt.test['threshold'] # prediction
pred_labeled, N = measure.label(pred, return_num=True)
if N > 1:
bg_area = ski_morph.remove_small_objects(pred_labeled,
opt.post['max_area']) > 0
large_area = ski_morph.remove_small_objects(
pred_labeled, opt.post['min_area']) > 0
pred = pred * (bg_area == 0) * (large_area > 0)
TP, FP, FN = utils.compute_accuracy(pred, gt, radius=opt.r1)
total_TP += TP
total_FP += FP
total_FN += FN
recall = float(total_TP) / (total_TP + total_FN + 1e-8)
precision = float(total_TP) / (total_TP + total_FP + 1e-8)
F1 = 2 * precision * recall / (precision + recall + 1e-8)
logger.info('\t=> Val Avg:\tRecall {:.4f}\tPrec {:.4f}\tF1 {:.4f}'.format(
recall, precision, F1))
return recall, precision, F1
def train(DCN, logger, gen):
Loss, Loss_reg = [], []
Discard_rates = [[[] for jj in range(ii)] for ii in gen.scales['train']]
Accuracies_tr = [[] for ii in gen.scales['train']]
Accuracies_te = [[] for ii in gen.scales['test']]
iterations_tr = int(gen.num_examples_train / batch_size)
for epoch in range(num_epochs):
lr = DCN.upd_learning_rate(epoch)
for it in range(iterations_tr):
losses = 0.0
variances = 0.0
start = time.time()
for i, scales in enumerate(gen.scales['train']):
# depth tells how many times the dynamic model will be unrolled
depth = 1
if args.dynamic:
depth = scales
_, length = gen.compute_length(scales, mode='train')
DCN.merge.n, DCN.split.n = [length] * 2
input, tar = gen.get_batch(batch=it, scales=scales,
mode='train')
# forward DCN
out = DCN(input, tar, length, depth, it=it, epoch=epoch,
random_split=args.random_split,
mode='train', dynamic=args.dynamic)
Phis, Inputs_N, target, Perms, e, loss, pg_loss, var = out
# backward DCN
if args.dynamic:
DCN.step_split(pg_loss, var,
regularize=args.regularize_split)
DCN.step_merge(loss)
losses += loss
variances += var
# Save discard rates
discard_rates = utils.compute_dicard_rate(Perms)
for sc in range(len(Perms) - 1):
Discard_rates[i][sc].append(discard_rates[sc])
Accuracies_tr[i].append(utils.compute_accuracy(Perms[-1],
target))
losses /= len(gen.scales['train'])
variances /= len(gen.scales['train'])
# optimizer.step()
Loss.append(losses.data.cpu().numpy())
Loss_reg.append(variances.data.cpu().numpy())
elapsed = time.time() - start
if it % 64 == 0:
print('TRAINING --> | Epoch {} | Batch {} / {} | Loss {} |'
' Accuracy Train {} | Elapsed {} | dynamic {} |'
' Learning Rate {}'
.format(epoch, it, iterations_tr,
losses.data.cpu().numpy(), Accuracies_tr[-1][-1],
elapsed, args.dynamic, lr))
logger.plot_Phis_sparsity(Phis, fig=0)
logger.plot_norm_points(Inputs_N, e, Perms,
gen.scales['train'][-1], fig=1)
logger.plot_losses(Loss, Loss_reg, fig=2)
logger.plot_rates(Discard_rates, fig=3)
logger.plot_accuracies(Accuracies_tr,
scales=gen.scales['train'],
mode='train', fig=4)
# keep track of output examples
# for perm in Perms:
# print('permutation', perm.data[0, 1:].cpu().numpy())
# print('target', target[0].data.cpu().numpy())
if it % 1000 == 1000 - 1:
print('Saving model parameters')
DCN.save_split(args.path)
DCN.save_merge(args.path)
accuracies_test = test(DCN, gen)
for i, scales in enumerate(gen.scales['test']):
Accuracies_te[i].append(accuracies_test[i])
show_accs = ' | '.join(['Acc Len {} -> {}'
.format(scales, Accuracies_te[i][-1])
for i, scales
in enumerate(gen.scales['test'])])
print('TESTING --> epoch {} '.format(epoch) + show_accs)
logger.plot_accuracies(Accuracies_te,
scales=gen.scales['test'],
mode='test', fig=2)
logger.save_results(Loss, Accuracies_te, Discard_rates)