def test(net, recall_ids):
net.eval()
with torch.no_grad():
# obtain feature vectors for all data
for key in eval_dict.keys():
eval_dict[key]['features'] = []
for inputs, labels in tqdm(eval_dict[key]['data_loader'],
desc='processing {} data'.format(key)):
inputs, labels = inputs.cuda(), labels.cuda()
features, classes = net(inputs)
eval_dict[key]['features'].append(features)
eval_dict[key]['features'] = torch.cat(eval_dict[key]['features'],
dim=0)
# compute recall metric
if data_name == 'isc':
acc_list = recall(eval_dict['test']['features'],
test_data_set.labels, recall_ids,
eval_dict['gallery']['features'],
gallery_data_set.labels)
else:
acc_list = recall(eval_dict['test']['features'],
test_data_set.labels, recall_ids)
desc = 'Test Epoch {}/{} '.format(epoch, num_epochs)
for index, rank_id in enumerate(recall_ids):
desc += 'R@{}:{:.2f}% '.format(rank_id, acc_list[index] * 100)
results['test_recall@{}'.format(rank_id)].append(acc_list[index] * 100)
print(desc)
return acc_list[0]
def eval(net, data_dict, ensemble_num, recalls):
net.eval()
data_set = ImageReader(data_dict, get_transform(DATA_NAME, 'test'))
data_loader = DataLoader(data_set,
BATCH_SIZE,
shuffle=False,
num_workers=8)
features = []
with torch.no_grad():
for inputs, labels in data_loader:
out = net(inputs.to(DEVICE))
out = F.normalize(out)
features.append(out.cpu())
features = torch.cat(features, 0)
torch.save(
features,
'results/{}_test_features_{:03}.pth'.format(DATA_NAME, ensemble_num))
# load feature vectors
features = [
torch.load('results/{}_test_features_{:03}.pth'.format(DATA_NAME, d))
for d in range(1, ensemble_num + 1)
]
features = torch.cat(features, 1)
acc_list = recall(features, data_set.labels, rank=recalls)
desc = ''
for index, recall_id in enumerate(recalls):
desc += 'R@{}:{:.2f}% '.format(recall_id, acc_list[index] * 100)
print(desc)
def eval(net, loader, ep):
net.eval()
n_node = graph_net.n_node
graph_net.n_node = 1
net.return_base = False
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
test_iter.set_description("[Eval][Epoch %d]" % ep)
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
embedding = net(images)
if graph_net is not None:
embedding = graph_net(embedding)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all)
print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec))
graph_net.n_node = n_node
return rec
def vectorized_rf(x_train,
y_train,
x_test,
y_test,
checktrain=True,
ngram_range=(1, 1),
vector_type="count",
dataset="default"):
vectorizer = CountVectorizer(tokenizer=tokenize, ngram_range=ngram_range) if vector_type=="count" \
else TfidfVectorizer(tokenizer=tokenize, ngram_range=ngram_range)
vectorized_x_train = vectorizer.fit_transform(x_train)
vectorized_x_test = vectorizer.transform(x_test)
model = train(vectorized_x_train,
y_train,
checktrain,
ngram_range,
vector_type=vector_type,
dataset=dataset)
pred_x_train = predict(model, vectorized_x_train)
pred_x_test = predict(model, vectorized_x_test)
precision_test = precision(y_test, pred_x_test)
recall_test = recall(y_test, pred_x_test)
f1_test = f1(y_test, pred_x_test)
print("Accuracy training accuracy (" + dataset, vector_type,
" vectorized joint data) =", accuracy(y_train, pred_x_train))
print("Accuracy testing accuracy (" + dataset, vector_type,
"vectorized joint data) =", accuracy(y_test, pred_x_test), "\n")
print("Precision (" + dataset, vector_type + " vectorized test data) =",
precision_test)
print("Recall (" + dataset, vector_type + " test data) =", recall_test)
print("F1 (" + dataset, vector_type + " test data) =", f1_test, "\n")
def simple_lstm(x_train, y_train, x_test, y_test, dataset="default"):
model = train(x_train, y_train, dataset=dataset)
print(x_train.shape)
print(x_test.shape)
# print(model.summary())
pred_x_train = model.predict(x_train)
pred_x_test = model.predict(x_test)
precision_test = precision(np.argmax(y_test, axis=1),
np.argmax(pred_x_test, axis=1),
labels=(0, 1, 2))
recall_test = recall(np.argmax(y_test, axis=1),
np.argmax(pred_x_test, axis=1),
labels=(0, 1, 2))
f1_test = f1(np.argmax(y_test, axis=1),
np.argmax(pred_x_test, axis=1),
labels=(0, 1, 2))
#
print(
"Accuracy training accuracy = ",
accuracy(np.argmax(y_train, axis=1), np.argmax(pred_x_train, axis=1)))
print("Accuracy testing accuracy =",
accuracy(np.argmax(y_test, axis=1), np.argmax(pred_x_test, axis=1)),
"\n")
#
print("Precision (test data) =", precision_test)
print("Recall (test data) =", recall_test)
print("F1 (test data) =", f1_test, "\n")
def evaluate(self, Xt, Xc, y):
"""
Evaluates the model's accuracy, precision, recall, F1 score and loss value on a dataset.
:param Xt: Matrix containing title input.
:param Xc: Matrix containing content input.
:param y: Label vector.
:return: Accuracy, lists containing each topic's precision, recall, F1 score followed by the macro-average across topics, and the model's loss value.
"""
probs = self.model.predict([Xt, Xc], verbose=1, batch_size=100)
preds = np.argmax(probs, axis=1)
true = np.argmax(y, 1)
acc = np.sum(np.equal(preds, true)) / np.size(true, 0)
p, r, f1 = [], [], []
for i in range(len(self.classes)):
p.append(utils.precision(preds, true, i))
r.append(utils.recall(preds, true, i))
f1.append(utils.f1_score(p[i], r[i]))
p2 = [x for x in p if x is not None]
r2 = [x for x in r if x is not None]
f2 = [x for x in f1 if x is not None]
p.append(np.mean(p2))
r.append(np.mean(r2))
f1.append(np.mean(f2))
print('\nCalculating loss...')
self.compile()
loss = self.model.evaluate([Xt, Xc], y, batch_size=100)[0]
return acc, p, r, f1, loss
def eval_graph(net, loader, ep):
net.eval()
graph_net.eval()
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
test_iter.set_description("[Eval][Epoch %d]" % ep)
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
embedding = net(images)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
embeddings_all = torch.cat(embeddings_all)
labels_all = torch.cat(labels_all)
d = pdist(embeddings_all)
pos_idx = d.topk(11, dim=1, largest=False)[1][:, 1:]
neg_idx = torch.randint(0, len(d), (len(d), 1), device=d.device, dtype=torch.int64)
graph_embedding = []
for i, e in enumerate(embeddings_all):
pos_embedding = embeddings_all[pos_idx[i][1:]]
neg_embedding = embeddings_all[torch.cat([pos_idx[j] for j in range(i-3, i-1)])]
e = torch.cat((e.unsqueeze(0), pos_embedding, neg_embedding), dim=0)
e = graph_net(e)
graph_embedding.append(e[0])
graph_embedding = torch.stack(graph_embedding, dim=0)
rec = recall(graph_embedding, labels_all)
print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec))
return rec
def eval(net, loader, ep):
K = [1, 10, 100, 1000]
net.eval()
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
test_iter.set_description("[Eval][Epoch %d]" % ep)
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
embedding = net(images)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
print(cuda.memory_allocated(cuda.current_device()))
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all, K=K)
print("Embedding Size: %d" % len(embeddings_all))
print(labels_all.sum())
for k, r in zip(K, rec):
print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k+1, 100 * r))
return rec[0]
def test(net, recall_ids):
net.eval()
with torch.no_grad():
# obtain feature vectors for all data
for key in eval_dict.keys():
eval_dict[key]['features'] = []
for inputs, labels in tqdm(eval_dict[key]['data_loader'],
desc='processing {} data'.format(key),
dynamic_ncols=True):
features, classes = net(inputs.cuda())
eval_dict[key]['features'].append(features)
eval_dict[key]['features'] = torch.cat(eval_dict[key]['features'],
dim=0)
test_features = torch.sign(eval_dict['test']['features']).cpu()
# compute recall metric
if data_name == 'isc':
dense_acc_list = recall(eval_dict['test']['features'].cpu(),
test_data_set.labels, recall_ids,
eval_dict['gallery']['features'].cpu(),
gallery_data_set.labels)
gallery_features = torch.sign(
eval_dict['gallery']['features']).cpu()
binary_acc_list = recall(test_features,
test_data_set.labels,
recall_ids,
gallery_features,
gallery_data_set.labels,
binary=True)
else:
dense_acc_list = recall(eval_dict['test']['features'].cpu(),
test_data_set.labels, recall_ids)
binary_acc_list = recall(test_features,
test_data_set.labels,
recall_ids,
binary=True)
desc = 'Test Epoch {}/{} '.format(epoch, num_epochs + 1)
for index, rank_id in enumerate(recall_ids):
desc += 'R@{}:{:.2f}%[{:.2f}%] '.format(rank_id,
dense_acc_list[index] * 100,
binary_acc_list[index] * 100)
results['test_dense_recall@{}'.format(rank_id)].append(
dense_acc_list[index] * 100)
results['test_binary_recall@{}'.format(rank_id)].append(
binary_acc_list[index] * 100)
print(desc)
return dense_acc_list[0]
def test_F2Score():
print(true_positive(actual, estimated) == 2)
print(false_positive(actual, estimated) == 3)
print(false_negative(actual, estimated) == 1)
expected_prec = 2 / (2 + 3)
print(precision(actual, estimated) == expected_prec)
expected_recall = 2 / (2 + 1)
print(recall(actual, estimated) == expected_recall)
print(
F1score(actual, estimated) == 2 * expected_recall * expected_prec /
(expected_recall + expected_prec))
def eval(net, loader, ep):
net.eval()
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
output = net(images)
embeddings_all.append(output.data)
labels_all.append(labels.data)
test_iter.set_description("[Eval][Epoch %d]" % ep)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
print('[Epoch %d] Recall@1: [%.6f]\n' %
(ep, recall(embeddings_all, labels_all)))
def eval(ep, loader=loader_eval):
model.eval()
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
test_iter.set_description("[Eval][Epoch %d]" % ep)
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
embedding = model(images)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all)
print('[Epoch %d] Recall@1: [%.4f]\n' % (ep, 100 * rec))
return rec
def test(net, recall_ids):
net.eval()
# obtain feature vectors for all data
with torch.no_grad():
features = []
for inputs, labels in tqdm(test_data_loader, desc='processing test data', dynamic_ncols=True):
feature, _ = net(inputs.cuda())
features.append(feature)
features = torch.cat(features, dim=0)
# compute recall metric
acc_list = recall(features, test_data_set.labels, recall_ids)
desc = 'Test Epoch {}/{} '.format(epoch, num_epochs)
for index, rank_id in enumerate(recall_ids):
desc += 'R@{}:{:.2f}% '.format(rank_id, acc_list[index] * 100)
results['test_recall@{}'.format(rank_id)].append(acc_list[index] * 100)
print(desc)
data_base['test_features'] = features
return acc_list[0]
def eval(net, loader, ep):
net.eval()
embeddings_all, labels_all = [], []
with torch.no_grad():
for images, labels in loader:
images, labels = images.to(device), labels.to(device)
output = avgpool(net(images, True)[-1])
embeddings_all.append(output.data)
labels_all.append(labels.data)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all, K=[1])
for k, r in enumerate(rec):
print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r))
return rec[0]
def kfold(classification_algorithm, k):
res = {"accuracy": 0, "precision": 0, "recall": 0, "f1": 0}
for i in range(1, k + 1):
validation = utils.load_train(i)
validation = validation["plus"] + validation["minus"]
train = {"plus": [], "minus": []}
for j in range(1, k + 1):
if j != i:
extension = utils.load_train(j)
train["plus"].extend(extension["plus"])
train["minus"].extend(extension["minus"])
classification = classification_algorithm(train, validation)
res["accuracy"] += utils.accuracy(classification)
res["precision"] += utils.precision(classification)
res["recall"] += utils.recall(classification)
res["f1"] += utils.F1_score(classification)
for k in res:
res[k] /= k
print res
return res
def eval(net, net_normalize, loader, ep):
net.eval()
test_iter = tqdm(loader)
embeddings_all, labels_all = [], []
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
output = net(net_normalize(images))
embeddings_all.append(output.data)
labels_all.append(labels.data)
test_iter.set_description("[Eval][Epoch %d]" % ep)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all, K=[1])
for k, r in enumerate(rec):
print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r))
return rec[0]
def main(path_detection, path_truth, mode, iou):
print("Path to detections:", path_detection)
print("Path to ground truth:", path_truth)
print("Chosen mode:", mode)
print("IoU-threshold:", iou)
# acquire detections and false_negatives for each class
# detections: {class: [(1,0.95), (0,0.87), (0,0.6), (1,0.67), ...., (0,0.9), (1,0.7)]}
# false_negatives: int
detections, false_negatives = loop_detection_files(path_detection,
path_truth, iou, mode)
# store APs for calculating mAP
average_precisions = []
# analyse results of each class
for classification in detections.keys():
classification_detections = [
d[0] for d in sorted(detections.get(classification),
key=lambda tup: tup[1],
reverse=True)
]
# print precision, recall, AP and show precision-recall curve
print("-------------")
print("CLASS", classification)
print("Precision:", precision(classification_detections))
print(
"Recall:",
recall(classification_detections,
false_negatives.get(classification)))
average_precision, smoothed_precision, recall_samples = AP(
classification, classification_detections,
false_negatives.get(classification))
average_precisions.append(average_precision)
print("AP:", average_precision)
visualise_results(classification, classification_detections,
false_negatives.get(classification),
smoothed_precision, recall_samples)
print("-------------")
print("********")
print("mAP:", np.mean(average_precisions))
print("********")
def compute_test():
model.eval()
output = model(features, adjs)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
# mac = macro_f1(output[idx_test], labels[idx_test])
# mac_pre = macro_precision(output[idx_test], labels[idx_test])
# mac_rec = macro_recall(output[idx_test], labels[idx_test])
# print("Test set results:",
# "loss= {:.4f}".format(loss_test.item()),
# "accuracy= {:.4f}".format(acc_test.item()),
# "macro_f1= {:.4f}".format(mac),
# "macro_precision= {:.4f}".format(mac_pre),
# "macro_recall= {:.4f}".format(mac_rec))
mac = f1(output[idx_test], labels[idx_test])
mac_pre = precision(output[idx_test], labels[idx_test])
mac_rec = recall(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()),
"macro_f1= {:.4f}".format(mac),
"macro_precision= {:.4f}".format(mac_pre),
"macro_recall= {:.4f}".format(mac_rec))
def two_eval(net, loader1, loader2, ep):
net.eval()
test_iter = tqdm(loader1)
test_iter2 = tqdm(loader2)
embeddings_all, labels_all = [], []
test_iter.set_description("[Eval][Epoch %d]" % ep)
labels_in = []
with torch.no_grad():
for images, labels in test_iter:
images, labels = images.cuda(), labels.cuda()
embedding = net(images)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
#embeddings_all = torch.cat(embeddings_all).cpu()
#labels_all = torch.cat(labels_all).cpu()
for images, labels in test_iter2:
images, labels = images.cuda(), labels.cuda()
embedding = net(images)
embeddings_all.append(embedding.data)
labels_all.append(labels.data)
embeddings_all = torch.cat(embeddings_all).cpu()
labels_all = torch.cat(labels_all).cpu()
rec = recall(embeddings_all, labels_all, K=4)
print("Embedding Size: %d" % len(embeddings_all))
print(labels_all.sum())
for k, r in enumerate(rec):
print('[Epoch %d] Recall@%d: [%.4f]\n' % (ep, k + 1, 100 * r))
return rec[0]
-1)[:args.crop_height, :args.crop_width]
st = time.time()
output_image = sess.run(network, feed_dict={input: input_image})
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
out_eval_image = output_image[:, :, 0]
out_vis_image = helpers.colour_code_segmentation(output_image)
accuracy = utils.compute_avg_accuracy(out_eval_image, gt)
class_accuracies = utils.compute_class_accuracies(
out_eval_image, gt, num_classes)
prec = utils.precision(out_eval_image, gt)
rec = utils.recall(out_eval_image, gt)
f1 = utils.f1score(out_eval_image, gt)
iou = utils.compute_mean_iou(out_eval_image, gt)
file_name = utils.filepath_to_name(val_input_names[ind])
target.write("%s, %f, %f, %f, %f, %f" %
(file_name, accuracy, prec, rec, f1, iou))
for item in class_accuracies:
target.write(", %f" % (item))
target.write("\n")
scores_list.append(accuracy)
class_scores_list.append(class_accuracies)
precision_list.append(prec)
recall_list.append(rec)
f1_list.append(f1)
# Get mesurments for current frame
fscore, iou, map, bboxTP, bboxFN, bboxFP = bg.compute_metrics_general(
cbbox, bboxes_in_img, k=5, iou_thresh=0.5)
# collect of the measures
fscore_tot.append(fscore)
iou_tot.append(iou)
map_tot.append(map)
bboxTP_tot += bboxTP
bboxFN_tot += bboxFN
bboxFP_tot += bboxFP
# Precision -Recall of this Experiment
precision.append(ut.precision(bboxTP_tot, bboxFP_tot))
recall.append(ut.recall(bboxTP_tot, bboxFN_tot))
fsc.append(ut.fscore(bboxTP_tot, bboxFP_tot, bboxFN_tot))
# Compute the mAP over the precision and Recall
mAp = compute_mAP(precision, recall)
print('mAP : {}'.format(mAp))
# If there are bounding boxes in the ground truth
#if any(cbbox):
# check if it is a good example for ploting:
#if PLOT_FLAG:
# Plot different thresholds
#fig, axs = plt.subplots(2, 3, figsize=(15, 6), facecolor='w', edgecolor='g')
#fig.subplots_adjust(hspace=.5, wspace=.01)
axis=0) / 255.0
st = time.time()
output_image = sess.run(network, feed_dict={z: input_image})
gt_map = cv2.imread(test_output_names[ind],
-1)[:args.crop_height, :args.crop_width]
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
output_image = output_image[:, :, 0]
out_vis_image = helpers.colour_code_segmentation(output_image)
accuracy = utils.compute_avg_accuracy(output_image, gt_map)
class_accuracies = utils.compute_class_accuracies(output_image, gt_map)
prec = utils.precision(output_image, gt_map)
rec = utils.recall(output_image, gt_map)
f1 = utils.f1score(output_image, gt_map)
iou = utils.compute_mean_iou(output_image, gt_map)
file_name = utils.filepath_to_name(test_input_names[ind])
target.write("%s, %f, %f, %f, %f, %f" %
(file_name, accuracy, prec, rec, f1, iou))
for item in class_accuracies:
target.write(", %f" % (item))
target.write("\n")
scores_list.append(accuracy)
class_scores_list.append(class_accuracies)
precision_list.append(prec)
recall_list.append(rec)
f1_list.append(f1)
FN = 0
for index, name in enumerate(a_names):
prediction = cv2.imread(name)
ground_truth = gt[index]
pe = pixel_evaluation(ground_truth, prediction)
TP += pe[0]
TN += pe[1]
FP += pe[2]
FN += pe[3]
a_pe = np.array([TP, TN, FP, FN])
a_precision = precision(a_pe)
a_recall = recall(a_pe)
a_f1_score = f1_score(a_pe)
# test B results
regex = re.compile(".*(test_B).*")
b_names = [m.group(0) for l in glob.glob(TestDirectory + '*') for m in [regex.search(l)] if m]
b_names.sort()
TP = 0
TN = 0
FP = 0
FN = 0
for index, name in enumerate(b_names):
prediction = cv2.imread(name)
ground_truth = gt[index]
def test(self, data):
preds = []
actuals = []
source_data, source_loc_data, target_data, target_label = data
N = int(np.ceil(len(source_data) / self.batch_size))
cost = 0
x = np.ndarray([self.batch_size, 1], dtype=np.int32)
time = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32)
target = np.zeros([self.batch_size], dtype=np.int32)
context = np.ndarray([self.batch_size, self.mem_size], dtype=np.int32)
mask = np.ndarray([self.batch_size, self.mem_size])
context.fill(self.pad_idx)
m, acc = 0, 0
for i in range(N):
target.fill(0)
time.fill(self.mem_size)
context.fill(self.pad_idx)
mask.fill(-1.0 * np.inf)
raw_labels = []
for b in range(self.batch_size):
x[b][0] = target_data[m]
target[b] = target_label[m]
time[b, :len(source_loc_data[m])] = source_loc_data[m]
context[b, :len(source_data[m])] = source_data[m]
mask[b, :len(source_data[m])].fill(0)
raw_labels.append(target_label[m])
m += 1
loss = self.sess.run(
[self.loss],
feed_dict={
self.input: x,
self.time: time,
self.target: target,
self.context: context,
self.mask: mask
})
cost += np.sum(loss)
predictions = self.sess.run(self.correct_prediction,
feed_dict={
self.input: x,
self.time: time,
self.target: target,
self.context: context,
self.mask: mask
})
for b in range(self.batch_size):
preds.append(predictions[b])
actuals.append(raw_labels[b])
if raw_labels[b] == predictions[b]:
acc = acc + 1
prec = precision(actuals, preds, labels=[0, 1, 2])
rec = recall(actuals, preds, labels=[0, 1, 2])
f = f1(actuals, preds, labels=[0, 1, 2])
return cost, acc / float(len(source_data)), prec, rec, f
def main():
parser = get_parser("generation", MODEL_CLASSES, ALL_MODELS)
args = parser.parse_args()
if args.local_rank == -1:
args.device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
args.device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s",
args.local_rank, args.device, args.n_gpu, bool(args.local_rank != -1))
set_seed(args)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
args.model_type = args.model_type.lower()
model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
gpt2_model, tokenizer = init_gpt2_model(
checkpoint_dir=args.model_name_or_path,
args=args,
model_class=model_class,
tokenizer_class=tokenizer_class)
gpt2_model.eval()
if args.local_rank == 0:
torch.distributed.barrier()
config = gpt2_model.gpt2.config
if args.length < 0 and config.max_position_embeddings > 0:
args.length = config.max_position_embeddings
elif 0 < config.max_position_embeddings < args.length:
args.length = config.max_position_embeddings # No generation bigger than model size
elif args.length < 0:
args.length = MAX_LENGTH # avoid infinite loop
logger.info(args)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
eval_dataset = load_and_cache_examples(args, tokenizer)
if args.local_rank == 0:
torch.distributed.barrier()
eval_sampler = SequentialSampler(
eval_dataset) if args.local_rank == -1 else DistributedSampler(
eval_dataset, shuffle=False)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=args.per_gpu_eval_batch_size)
output_log = {
"true_text": [],
"generated_text": [],
"context": [],
"recall_score": [],
"context_suffix_styles": [],
"original_styles": [],
"metadata": []
}
for batch in tqdm(eval_dataloader,
desc="Evaluating",
disable=args.local_rank not in [-1, 0]):
sentences = batch["sentence"].to(args.device)
segments = batch["segment"].to(args.device)
global_dense_vectors = batch["global_dense_vectors"].to(args.device)
suffix_styles = batch["suffix_style"]
original_styles = batch["original_style"]
metadata = batch["metadata"]
# Assume init_context_size is same for all examples in minibatch
init_context_size = batch["init_context_size"][0].item()
out, dense_length, scores = gpt2_model.generate(
gpt2_sentences=sentences,
segments=segments,
eos_token_id=tokenizer.eos_token_id,
global_dense_vectors=global_dense_vectors,
init_context_size=init_context_size)
for sent_num in range(sentences.shape[0]):
output_sequence = out[sent_num][init_context_size:].tolist()
if tokenizer.eos_token_id in output_sequence:
output_sequence = output_sequence[:output_sequence.index(
tokenizer.eos_token_id)]
true_text = tokenizer.decode(
sentences[sent_num, init_context_size:].tolist(),
clean_up_tokenization_spaces=True,
skip_special_tokens=True)
generated_text = tokenizer.decode(
output_sequence,
clean_up_tokenization_spaces=True,
skip_special_tokens=True)
context = tokenizer.decode(
sentences[sent_num, :init_context_size].tolist(),
clean_up_tokenization_spaces=True,
skip_special_tokens=True)
recall_score = recall(true_text, context)
output_log["true_text"].append(true_text)
output_log["generated_text"].append(generated_text)
output_log["context"].append(context)
output_log["recall_score"].append("%.4f" % recall_score)
output_log["metadata"].append(metadata[sent_num])
if hasattr(eval_dataset, "reverse_label_dict"):
output_log["context_suffix_styles"].append(
class_number_to_str(eval_dataset, suffix_styles[sent_num]))
output_log["original_styles"].append(
class_number_to_str(eval_dataset,
original_styles[sent_num]))
else:
output_log["context_suffix_styles"].append("<none>")
output_log["original_styles"].append("<none>")
with open(
os.path.join(args.generation_output_dir,
"reference_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["true_text"]) + "\n")
with open(
os.path.join(args.generation_output_dir,
"generated_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["generated_text"]) + "\n")
with open(
os.path.join(args.generation_output_dir,
"context_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["context"]) + "\n")
with open(
os.path.join(args.generation_output_dir,
"recall_score_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["recall_score"]) + "\n")
with open(
os.path.join(
args.generation_output_dir,
"context_suffix_styles_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["context_suffix_styles"]) + "\n")
with open(
os.path.join(args.generation_output_dir,
"original_styles_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["original_styles"]) + "\n")
with open(
os.path.join(args.generation_output_dir,
"metadata_%d.txt" % max(args.local_rank, 0)),
"w") as f:
f.write("\n".join(output_log["metadata"]) + "\n")
x2 = D.get_data('Data1/Class2.txt')
x3 = D.get_data('Data1/Class3.txt')
if not os.path.isdir("plots_multi"):
os.mkdir("plots_multi")
w1 = [200.0, -200.0, 200.0]
w2 = [-200.0, 200.0, 200.0]
w3 = [200.0, 200.0, -200.0]
perceptron(w1, x1, x2, x3, 0.5, 100, 1)
perceptron(w2, x2, x3, x1, 0.5, 100, 2)
perceptron(w3, x3, x1, x2, 0.5, 100, 3)
plt.figure()
decision_boundary(plt, w1, w2, w3)
conf_mat = get_conf(x1, x2, x3, w1, w2, w3, plt)
plt.savefig('plots_multi/decision_boundary.png')
print(conf_mat)
print("Accuracy: ", U.accuracy(conf_mat))
print("Precision for class 1: ", U.precision(conf_mat, 0))
print("Precision for class 2: ", U.precision(conf_mat, 1))
print("Precision for class 3: ", U.precision(conf_mat, 2))
print("Recall for class 1: ", U.recall(conf_mat, 0))
print("Recall for class 2: ", U.recall(conf_mat, 1))
print("Recall for class 3: ", U.recall(conf_mat, 2))
print("F-Score for class 1: ", U.f_score(conf_mat, 0))
print("F-Score for class 2: ", U.f_score(conf_mat, 1))
print("F-Score for class 3: ", U.f_score(conf_mat, 2))
axis=0) / 255.0
gt = cv2.imread(val_output_names[ind],
-1)[:args.crop_height, :args.crop_width]
st = time.time()
output_image = sess.run(network, feed_dict={input: input_image})
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
out_vis_image = helpers.colour_code_segmentation(output_image)
accuracy = utils.compute_avg_accuracy(output_image, gt)
class_accuracies = utils.compute_class_accuracies(
output_image, gt, num_classes)
prec = utils.precision(output_image, gt)
rec = utils.recall(output_image, gt)
f1 = utils.f1score(output_image, gt)
iou = utils.compute_mean_iou(output_image, gt)
file_name = utils.filepath_to_name(val_input_names[ind])
target.write("%s, %f, %f, %f, %f, %f" %
(file_name, accuracy, prec, rec, f1, iou))
for item in class_accuracies:
target.write(", %f" % (item))
target.write("\n")
scores_list.append(accuracy)
class_scores_list.append(class_accuracies)
precision_list.append(prec)
recall_list.append(rec)
f1_list.append(f1)
theta = np.zeros((X.shape[1], classes))
l2_lambda = 0
# Calculate optimal theta using Newton Conjugate Gradient (closest alternative to fminunc)
X_train, X_test, y_train, y_test = split_data(X, y) # stratified split
theta = opt.minimize(fun=calculate_cost_and_gradient,
x0=theta,
method='TNC',
args=(X_train, y_train, classes, l2_lambda),
jac=True,
options={'maxiter': 50})
theta.x = theta.x.reshape(
X.shape[1], classes) # reshape theta into necessary dimensions
# Print out the cost and calculated theta
print('Testing cost: %f' % theta.fun)
print('----------------------------------------------------') # separator
# Calculate accuracy, precision and recall
print('Accuracy (percentage of guessed samples): %f' %
multiclass_accuracy(theta.x, X_test, y_test))
conf = confusion_matrix_c(y_test, softmax(X_test.dot(theta.x)), True)
plot_confusion_matrix(conf,
'Confusion matrix (actual vs predicted values)')
print('Precision (true positives / predicted positives): %f' %
precision(conf))
print('Recall (true positives / actual positives): %f' % recall(conf))
plt.show()
start = time.time()
# recommend_list = recommend_multi_thread(train_data, item_sims, nearest_k, top_n)
recommend_list = recommend_multi_process(train_data, item_sims, nearest_k,
top_n)
# recommend_list = recommend_single_thread(train_data, item_sims, nearest_k, top_n)
print("recommend done, cost " + str(time.time() - start) + " s")
return recommend_list
if __name__ == '__main__':
start_time = time.time()
train, test = utils.split_data(utils.load_data("./data/ratings.dat"), 8, 1)
W = item_similarity(train)
recommends = recommend(train, W, nearest_k=10, top_n=10)
p = utils.precision(train, test, recommends)
r = utils.recall(train, test, recommends)
c = utils.coverage(train, recommends)
po = utils.popularity(train, recommends)
cost_time = time.time() - start_time
print(p, r, c, po)
print("cost time " + str(cost_time) + " s")
save_model_path = os.path.join(
save_time_fold,
preprocess_path.split('/')[-1] + '_times_' + str(train_times) +
'_' + str(round(best_test, 4)) + '.pth.tar')
torch.save(model.state_dict(), save_model_path)
save_predict_target_path = os.path.join(
save_time_fold,
preprocess_path.split('/')[-1] + '_times_' + str(train_times) +
'_' + str(round(best_test, 4)) + '.txt')
predict_target = torch.cat((predicts, targets),
dim=0).detach().cpu().numpy()
np.savetxt(save_predict_target_path, predict_target)
precision_score[train_times] = round(precision(predicts, targets),
4)
recall_score[train_times] = round(recall(predicts, targets), 4)
specificity_score[train_times] = round(
specificity(predicts, targets), 4)
mcc_score[train_times] = round(mcc(predicts, targets), 4)
auc_score = round(auc(predicts, targets), 4)
aupr_score = round(aupr(predicts, targets), 4)
print('Epoch: {:04d}'.format(epoch + 1), 'Train_times:', train_times)
print(
"*****************test_score {:.4f} best_socre {:.4f}****************"
.format(test_score, best_test))
print("All Test Score:", acc_score)
print(args.dataset, " Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
print("acc Score:", acc_score)
print("precision Score:", precision_score)
print("recall score", recall_score)
for image_name in image_names:
recalls_image_names[image_name] = []
precisions_image_names[image_name] = []
if dataset == 'mirflickr':
image_names, _, image_matches = mirflickr_images()
absolute_matches = {}
for label, s in results.iteritems():
absolute_matches[label] = {}
for target_image, retrieved in s.iteritems():
if dataset == 'ukbench':
retrieved_filenames = retrieved[:3]
recall_value = recall(target_image,
retrieved_filenames,
retrieved,
method=result_test_ukbench
)
precision_value = precision(target_image,
retrieved_filenames,
retrieved,
method=result_test_ukbench
)
recalls[label][target_image] = recall_value
precisions[label][target_image] = precision_value
relevant, retrieved = result_test_ukbench(target_image, retrieved_filenames, retrieved)
elif dataset == 'mirflickr':
retrieved_filenames = retrieved[:image_matches[target_image]]
recall_value = recall(target_image,
retrieved_filenames,
retrieved,
y = df.iloc[:, 4].values # Aplicando a normalização/escalonamento das features X = feature_scaling(X) # Definindo os valores da matriz de confusão: tp, fp, fn, tn = [1, 1, 1, 2] # Calculando os valores da acurácia accuracy(tp, fp, fn, tn) # Calculando os valores da precisão precision(tp, fp) # Calculando os valores do recall recall(tp, fn) # Calculando os valores do informedness informedness(tp, fp, fn, tn) # Calculando os valores do markdness markdness(tp, fp, fn, tn) # Definindo os valores para calculo do ROC-AUC # Valores reais (ground truth) y_true = np.array([1, 1, 2, 2]) #Probabilidade de estimar as classes positivas: y_scores = np.array([0.1, 0.4, 0.35, 0.8]) # A probabilidade de predição de cada classe retornada por um classificador:
