def search(self, df, params):
skf = StratifiedKFold(n_splits=self.n_splits,
random_state=self.random_state)
params_combos = self._generate_params_combos(params)
self.top = []
for combo in params_combos:
print(combo)
scores = []
counter = 0
for train_index, test_index in skf.split(X=df, y=df.group):
print("{} split".format(counter))
counter += 1
df_train = df.iloc[train_index]
df_test = df.iloc[test_index]
df_neg = create_negative_examples(
df=df,
df_train=df_train,
num_neg_exs=10 * df_train.shape[0],
random_state=self.random_state)
svd = MySVD()
svd.set_params(**combo)
svd.fit(df_train, df_neg)
# transform df_test
svd_group_encoder_dict = dict(
zip(svd.group_encoder.classes_,
list(range(len(svd.group_encoder.classes_)))))
encoded_corpus_test = df_test.groupby("user").agg(
lambda x: [svd_group_encoder_dict[item]
for item in x]).reset_index()
encoded_corpus_test['recs'] = svd.predict(
encoded_corpus_test.user.values)
score = mean_average_precision(
encoded_corpus_test.group.values,
encoded_corpus_test.recs.values)
print('score: {}'.format(score))
scores.append(
mean_average_precision(encoded_corpus_test.group.values,
encoded_corpus_test.recs.values))
self.top.append(
TopItem(combo=combo, score=sum(scores) / len(scores)))
self.top = sorted(self.top, key=lambda item: item.score, reverse=True)
return self.top
def test_step():
results = defaultdict(list)
num_test = 0
num_correct = 0.0
test_batches = data_helpers.batch_iter(test_dataset,
FLAGS.batch_size,
1,
target_loss_weight,
FLAGS.max_utter_len,
FLAGS.max_utter_num,
FLAGS.max_response_len,
shuffle=False)
for test_batch in test_batches:
x_utterances, x_response, x_utterances_len, x_response_len, x_utters_num, x_target, x_target_weight, id_pairs = test_batch
feed_dict = {
imn.utterances: x_utterances,
imn.response: x_response,
imn.utterances_len: x_utterances_len,
imn.response_len: x_response_len,
imn.utters_num: x_utters_num,
imn.target: x_target,
imn.target_loss_weight: x_target_weight,
imn.dropout_keep_prob: 1.0
}
batch_accuracy, predicted_prob = sess.run(
[imn.accuracy, imn.probs], feed_dict)
num_test += len(predicted_prob)
if num_test % 1000 == 0:
print(num_test)
num_correct += len(predicted_prob) * batch_accuracy
for i, prob_score in enumerate(predicted_prob):
question_id, response_id, label = id_pairs[i]
results[question_id].append(
(response_id, label, prob_score))
#calculate top-1 precision
print('num_test_samples: {} test_accuracy: {}'.format(
num_test, num_correct / num_test))
accu, precision, recall, f1, loss = metrics.classification_metrics(
results)
print('Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'.
format(accu, precision, recall, f1, loss))
mvp = metrics.mean_average_precision(results)
mrr = metrics.mean_reciprocal_rank(results)
top_1_precision = metrics.top_1_precision(results)
total_valid_query = metrics.get_num_valid_query(results)
print(
'MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}'
.format(mvp, mrr, top_1_precision, total_valid_query))
return mrr
def compute_metrics():
with torch.no_grad():
man_pdists_np = dist_fn(X).sqrt().cpu().numpy()
ad = average_distortion(g_pdists_np, man_pdists_np)
if g is None:
return ad, None
# TODO(ccruceru): Make sure this is correct. Try to reproduce the
# result from the ref. paper on 10D Euclidean manifold.
man_pdists_sym = pdists_vec_to_sym(man_pdists_np, n)
mean_ap = mean_average_precision(g, man_pdists_sym)
return ad, mean_ap
def search(self, df, algotype, params):
assert algotype in ('lda', 'plsa')
skf = StratifiedKFold(n_splits=self.n_splits,
random_state=self.random_state)
params_combos = self._generate_params_combos(params)
self.top = []
transformer = Transformer()
transformer.fit(df)
for combo in params_combos:
print(combo)
scores = []
counter = 0
for train_index, test_index in skf.split(X=df, y=df.group):
print("{} split".format(counter))
counter += 1
df_train = df.iloc[train_index]
df_test = df.iloc[test_index]
corpus_train = transformer.transform(df_train)
corpus_test = transformer.transform(df_test)
if algotype == 'lda':
est = LDAAdapter()
else:
est = MyPLSA()
est.set_params(**combo)
if algotype == 'lda':
est.fit(corpus_train)
else:
num_groups = df.group.nunique()
est.fit(corpus_train, num_groups)
corpus_test['recs'] = est.predict(corpus_test.user.values)
corpus_test['group'] = corpus_test.group.apply(
lambda x: [item[0] for item in x])
score = mean_average_precision(corpus_test.group.values,
corpus_test.recs.values)
print('score: {}'.format(score))
scores.append(score)
self.top.append(
TopItem(combo=combo, score=sum(scores) / len(scores)))
self.top = sorted(self.top, key=lambda item: item.score, reverse=True)
return self.top
def compute_map_gt_det(det_file, gt_file):
det_list = read_detections(det_file)
gt_list = read_detections(gt_file)
frames = []
for i in range(0, len(det_list)):
frame = Frame(i)
frame.detections = det_list[i]
frame.ground_truth = gt_list[i]
frames.append(frame)
mAP = mean_average_precision(frames, ignore_classes=True)
return mAP
def run_test(dir_path, op_name, sess, training, accuracy, prob, pair_ids, output_layer):
results = defaultdict(list)
num_test = 0
num_correct = 0.0
n_updates = 0
mrr = 0
t0 = time()
try:
while True:
n_updates += 1
batch_accuracy, predicted_prob, pair_ = sess.run([accuracy, prob, pair_ids], feed_dict={training: False})
question_id, answer_id, label = pair_
num_test += len(predicted_prob)
num_correct += len(predicted_prob) * batch_accuracy
for i, prob_score in enumerate(predicted_prob):
# question_id, answer_id, label = pair_id[i]
results[question_id[i]].append((answer_id[i], label[i], prob_score[0]))
if n_updates%2000 == 0:
tf.logging.info("n_update %d , %s: Mins Used: %.2f" %
(n_updates, op_name, (time() - t0) / 60.0))
except tf.errors.OutOfRangeError:
# calculate top-1 precision
print('num_test_samples: {} test_accuracy: {}'.format(num_test, num_correct / num_test))
accu, precision, recall, f1, loss = metrics.classification_metrics(results)
print('Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'.format(accu, precision, recall, f1, loss))
mvp = metrics.mean_average_precision(results)
mrr = metrics.mean_reciprocal_rank(results)
top_1_precision = metrics.top_1_precision(results)
total_valid_query = metrics.get_num_valid_query(results)
print('MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}'.format(
mvp, mrr, top_1_precision, total_valid_query))
out_path = os.path.join(dir_path, "output_test.txt")
print("Saving evaluation to {}".format(out_path))
with open(out_path, 'w') as f:
f.write("query_id\tdocument_id\tscore\trank\trelevance\n")
for us_id, v in results.items():
v.sort(key=operator.itemgetter(2), reverse=True)
for i, rec in enumerate(v):
r_id, label, prob_score = rec
rank = i+1
f.write('{}\t{}\t{}\t{}\t{}\n'.format(us_id, r_id, prob_score, rank, label))
return mrr
def evaluate_model(model, n):
mean_map = 0.
mean_ndcg = 0.
for u in range(len(user_items)): # para cada user_id en el training
rec = [t[0] for t in model.recommend(u, user_item_matrix, n)]
user_item_test_list = [
items_ids[i] for i in user_items_test[user_ids_inv[u]]
] # lista con las peliculas que vio en el test
rel_vector = [
np.isin(user_item_test_list, rec, assume_unique=True).astype(int)
]
mean_map += metrics.mean_average_precision(rel_vector)
mean_ndcg += metrics.ndcg_at_k(rel_vector, n)
mean_map /= len(user_items_test)
mean_ndcg /= len(user_items_test)
return mean_map, mean_ndcg
def main():
r"""Main entry point in the graph embedding procedure."""
args = config_parser().parse_args()
g_pdists = load_pdists(args)
n = g_pdists.shape[0]
d = args.manifold_dim
# we are actually using only the upper diagonal part
g_pdists = g_pdists[np.triu_indices(n, 1)]
g_sq_pdists = g_pdists**2
# read the graph
# the distortion cost
def distortion_cost(X):
man_sq_pdists = manifold_pdists(X, squared=True)
return np.sum(np.abs(man_sq_pdists / g_sq_pdists - 1))
# the manifold, problem, and solver
manifold = PositiveDefinite(d, k=n)
problem = Problem(manifold=manifold, cost=distortion_cost, verbosity=2)
linesearch = ReduceLROnPlateau(start_lr=2e-2,
patience=10,
threshold=1e-4,
factor=0.1,
verbose=1)
solver = ConjugateGradient(linesearch=linesearch, maxiter=1000)
# solve it
with Timer('training') as t:
X_opt = solver.solve(problem, x=sample_init_points(n, d))
# the distortion achieved
man_pdists = manifold_pdists(X_opt)
print('Average distortion: ', average_distortion(g_pdists, man_pdists))
man_pdists_sym = pdists_vec_to_sym(man_pdists, n, 1e12)
print('MAP: ', mean_average_precision(g,
man_pdists_sym,
diag_adjusted=True))
def evalMetric(self, metric):
score, n = 0, 0
if self.verbose:
print('Running evaluation')
for w1 in self.eval:
if w1 not in self.ranks: # skip non-existing
continue
# TODO: Implement desired metrics here
if metric == 'precision_at_10':
k = 10
rs = [1 if w in [x[0] for x in self.eval[w1]] else 0 for (w, _) in self.ranks[w1][:k]]
# print w1
# print rs
# print self.ranks[w1][:k]
# print self.eval[w1]
score += metrics.precision_at_k(rs, k)
elif metric == 'map':
rs = [1 if w in [x[0] for x in self.eval[w1]] else 0 for (w, _) in self.ranks[w1]]
# print w1
# print rs
# print self.ranks[w1]
# print self.eval[w1]
score += metrics.mean_average_precision(rs)
elif metric == 'ndcg_at_100':
k = 100
d = dict(self.eval[w1])
rs = [d[w] if w in d else 0 for (w, s) in self.ranks[w1][:k]]
# print w1
# print rs
# print self.ranks[w1][:k]
# print self.eval[w1]
score += metrics.ndcg_at_k(rs, k)
n += 1
return (score / n, n)
def run_test(epoch_no, dir_path, op_name, sess, training, accuracy, prob,
pair_ids):
results = defaultdict(list)
num_test = 0
num_correct = 0.0
n_updates = 0
mrr = 0
t0 = time()
try:
while True:
n_updates += 1
batch_accuracy, predicted_prob, pair_ = sess.run(
[accuracy, prob, pair_ids], feed_dict={training: False})
question_id, answer_id, label = pair_
# question_id = question_id.eval()
# answer_id = answer_id.eval()
# label = label.eval()
num_test += len(predicted_prob)
# if num_test % 1000 == 0:
# print(num_test)
num_correct += len(predicted_prob) * batch_accuracy
for i, prob_score in enumerate(predicted_prob):
# question_id, answer_id, label = pair_id[i]
results[question_id[i]].append(
(answer_id[i], label[i], prob_score[0]))
if n_updates % 2000 == 0:
tf.logging.info(
"epoch: %i n_update %d , %s: Mins Used: %.2f" %
(epoch_no, n_updates, op_name, (time() - t0) / 60.0))
except tf.errors.OutOfRangeError:
threshold = 0.95
none_id = 10000000
print("threshold: {}".format(threshold))
for q_id, a_list in results.items():
correct_flag = 0
for (a_id, label, score) in a_list:
if int(label) == 1:
correct_flag = 1
if correct_flag == 0:
results[q_id].append((none_id, 1, threshold))
else:
results[q_id].append((none_id, 0, threshold))
# calculate top-1 precision
print('num_test_samples: {} test_accuracy: {}'.format(
num_test, num_correct / num_test))
accu, precision, recall, f1, loss = metrics.classification_metrics(
results)
print(
'Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'.format(
accu, precision, recall, f1, loss))
mvp = metrics.mean_average_precision(results)
mrr = metrics.mean_reciprocal_rank(results)
top_1_precision = metrics.top_1_precision(results)
total_valid_query = metrics.get_num_valid_query(results)
print(
'MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}'
.format(mvp, mrr, top_1_precision, total_valid_query))
out_path = os.path.join(dir_path,
"ubuntu_output_epoch_{}.txt".format(epoch_no))
print("Saving evaluation to {}".format(out_path))
with open(out_path, 'w') as f:
f.write("query_id\tdocument_id\tscore\trank\trelevance\n")
for us_id, v in results.items():
v.sort(key=operator.itemgetter(2), reverse=True)
for i, rec in enumerate(v):
r_id, label, prob_score = rec
rank = i + 1
f.write('{}\t{}\t{}\t{}\t{}\n'.format(
us_id, r_id, prob_score, rank, label))
global best_score
if op_name == 'valid' and mrr > best_score:
best_score = mrr
saver = tf.train.Saver()
dir_path = os.path.join(dir_path, "epoch {}".format(epoch_no))
if not os.path.exists(dir_path):
os.makedirs(dir_path)
saver.save(sess, dir_path)
tf.logging.info(">> save model!")
return mrr
labels = list(read_labels("data/" + labelfile))
y, X = zip(*match_labels_documents(documents, labels))
y, X = np.array(y), np.array(X)
kf = KFold(len(y), n_folds=10, shuffle=True, random_state=1)
rank_scores = np.zeros(10)
for i, (train, test) in enumerate(kf):
X_train, X_test, y_train, y_test = X[train], X[test], y[train], y[test]
labels = Counter(flatten(list(y_train)))
labels = [label for label, count in labels.items() if count >= 1]
model = IRSystem(k1=1.2, b=0.75, cutoff=0)
model.fit_raw(X_train, y_train, ngram_range=(1, 1), stop_words='english', min_df=2)
ranking = model.rank_labels(X_test, raw=True)
ranking = ranking.tolist()
ranking = map(lambda r: list(unique_everseen(r)), map(flatten, ranking))
ranking, y_test = zip(*[(r, y_) for r, y_ in zip(ranking, y_test) if any(l in labels for l in y_)])
rank_scores[i] = mean_average_precision(ranking, y_test)
print 'IR: (%s)' % (labelfile), rank_scores.mean(), rank_scores.std()
# Next, we'll do an IR experiment with Big Documents
documents = {doc_id: text for doc_id, text in read_dreams("data/dreambank.en.stanford.out")}
labels = list(read_labels("data/" + labelfile))
y, X = zip(*match_labels_documents(documents, labels))
y, X = np.array(y), np.array(X)
kf = KFold(len(y), n_folds=10, shuffle=True, random_state=1)
rank_scores = np.zeros(10)
for i, (train, test) in enumerate(kf):
X_train, X_test, y_train, y_test = X[train], X[test], y[train], y[test]
big_docs = defaultdict(str)
for (labels, doc) in zip(y_train, X_train):
for label in labels:
big_docs[label] += " " + doc
def score_single(sk_labels, im_labels, index):
res = np.equal(im_labels[index], sk_labels[:, None])
prec = np.mean([precision_at_k(r, 100) for r in res])
mAP = mean_average_precision(res)
return prec, mAP
def main():
video = Video("../datasets/AICity_data/train/S03/c010/vdo.avi")
gt = read_annotations('../annotations', start_frame, end_frame)
"""
DETECTIONS
"""
det_algs = ['yolo3', 'mask_rcnn', 'ssd512']
for alg in det_algs:
detections = read_detections(
'../datasets/AICity_data/train/S03/c010/det/det_{0}.txt'.format(
alg))
detections = detections[start_frame:end_frame + 1]
frames = []
# roi = cv2.imread('../datasets/AICity_data/train/S03/c010/roi.jpg')
for im, f in seq(video.get_frames(
start_frame_number=start_frame)).take(end_frame - start_frame +
1):
f.ground_truth = gt[f.id]
f.detections = detections[f.id]
frames.append(f)
if make_video:
make_video_frame(im, f, frames)
iou_over_time(frames)
mAP = mean_average_precision(frames)
print(alg, " mAP:", mAP)
"""
DETECTIONS FROM ALTERED GROUND TRUTH
"""
frames = []
for im, f in seq(video.get_frames()).take(end_frame - start_frame + 1):
f.ground_truth = gt[f.id]
f.detections = alter_detections(f.ground_truth)
frames.append(f)
if make_video:
make_video_frame(im, f, frames)
iou_over_time(frames)
mAP = mean_average_precision(frames)
print('Random alteration', " mAP:", mAP)
"""
OPTICAL FLOW
"""
of_det_1 = read_optical_flow(
'../datasets/optical_flow/detection/LKflow_000045_10.png')
of_det_2 = read_optical_flow(
'../datasets/optical_flow/detection/LKflow_000157_10.png')
of_gt_1 = read_optical_flow('../datasets/optical_flow/gt/000045_10.png')
of_gt_2 = read_optical_flow('../datasets/optical_flow/gt/000157_10.png')
img_1 = cv2.imread('../datasets/optical_flow/img/000045_10.png')
img_2 = cv2.imread('../datasets/optical_flow/img/000157_10.png')
msen_of = msen(of_det_2, of_gt_2)
pepn_of = pepn(of_det_2, of_gt_2)
print(msen_of, pepn_of)
show_optical_flow(of_gt_1)
show_optical_flow_arrows(img_1, of_gt_1)
msen_45 = msen(of_det_1, of_gt_1, plot=True)
pepn_45 = pepn(of_det_1, of_gt_1)
print("Sequence 045: MSEN", msen_45, "PEPN", pepn_45)
msen_157 = msen(of_det_2, of_gt_2, plot=True)
pepn_157 = pepn(of_det_2, of_gt_2)
print("Sequence 157: MSEN", msen_157, "PEPN", pepn_157)
show_optical_flow(of_gt_1)
def main():
parser = argparse.ArgumentParser(
description='Search the picture passed in a picture database.')
parser.add_argument('method',
help='Method to use',
choices=method_refs.keys())
parser.add_argument('--debug',
action='store_true',
help='Show debug plots')
args = parser.parse_args()
method = method_refs.get(args.method)
video = Video("../datasets/AICity_data/train/S03/c010/frames")
frames = []
for im, mask, frame in method(video, **{'debug': args.debug}):
frames.append(frame)
#iou = frame.get_detection_iou(ignore_classes=True)
#print(iou)
if not args.debug:
plt.figure(figsize=(12, 8))
plt.subplot2grid((2, 2), (0, 0))
im_frame_left = np.copy(im)
for d in frame.ground_truth:
cv2.rectangle(im_frame_left,
(int(d.top_left[1]), int(d.top_left[0])),
(int(d.get_bottom_right()[1]),
int(d.get_bottom_right()[0])), (255, 0, 0),
thickness=5)
for d in frame.detections:
cv2.rectangle(im_frame_left,
(int(d.top_left[1]), int(d.top_left[0])),
(int(d.get_bottom_right()[1]),
int(d.get_bottom_right()[0])), (0, 0, 255),
thickness=5)
plt.imshow(cv2.cvtColor(im_frame_left, cv2.COLOR_BGR2RGB))
plt.axis('off')
plt.legend([
Line2D([0], [0], color=(0, 0, 1)),
Line2D([0], [0], color=(1, 0, 0)),
], ['GT', 'Det'],
loc='lower right')
plt.subplot2grid((2, 2), (0, 1))
m = cv2.cvtColor(np.copy(mask), cv2.COLOR_GRAY2RGB)
for d in frame.ground_truth:
cv2.rectangle(m, (int(d.top_left[1]), int(d.top_left[0])),
(int(d.get_bottom_right()[1]),
int(d.get_bottom_right()[0])), (0, 0, 255),
thickness=5)
for d in frame.detections:
cv2.rectangle(m, (int(d.top_left[1]), int(d.top_left[0])),
(int(d.get_bottom_right()[1]),
int(d.get_bottom_right()[0])), (255, 0, 0),
thickness=5)
plt.imshow(m)
plt.axis('off')
plt.legend([
Line2D([0], [0], color=(0, 0, 1)),
Line2D([0], [0], color=(1, 0, 0)),
], ['GT', 'Det'],
loc='lower right')
plt.subplot2grid((2, 2), (1, 0), colspan=2)
plt.title('IoU over time' + str(frame.id))
iou_over_time(frames, ignore_classes=True, show=False)
axes = plt.gca()
axes.set_xlim((0, int(2041 * .75) + 1))
axes.set_ylim((0, 1.1))
plt.legend()
plt.savefig('../video/{:04d}.png'.format(frame.id))
#plt.show()
plt.close()
iou_over_time(frames, ignore_classes=True)
print('mAP:', mean_average_precision(frames, ignore_classes=True))
def off_the_shelf_yolo(tracking, debug=False, *args, **kwargs):
video = Video("../datasets/AICity_data/train/S03/c010/frames")
detection_transform = DetectionTransform()
classes = utils.load_classes('../config/coco.names')
gt = read_annotations(
'../datasets/AICity_data/train/S03/c010/m6-full_annotation.xml')
model = Darknet('../config/yolov3.cfg')
model.load_weights('../weights/fine_tuned_yolo_freeze.weights')
if torch.cuda.is_available():
model = model.cuda()
frames = []
last_im = None
model.eval()
with torch.no_grad():
for i, im in tqdm(enumerate(video.get_frames(start=len(video) // 4)),
total=len(video),
file=sys.stdout,
desc='Yolo'):
im_tensor = detection_transform(im)
im_tensor = im_tensor.view((-1, ) + im_tensor.size())
if torch.cuda.is_available():
im_tensor = im_tensor.cuda()
detections = model.forward(im_tensor)
detections = utils.non_max_suppression(detections,
80,
conf_thres=.6,
nms_thres=0.3)
frame = Frame(i + (len(video) // 4))
frame.ground_truth = gt[frame.id]
for d in detections[0]:
if int(d[6]) in VALID_LABELS:
bbox = d.cpu().numpy()
det = Detection(-1,
classes[int(d[6])], (bbox[0], bbox[1]),
width=bbox[2] - bbox[0],
height=bbox[3] - bbox[1],
confidence=d[5])
detection_transform.unshrink_detection(det)
frame.detections.append(det)
if tracking is not None:
last_frame = None if len(frames) == 0 else frames[-1]
tracking(frame=frame,
im=im,
last_frame=last_frame,
last_im=last_im,
frames=frames,
debug=False)
frames.append(frame)
last_im = im
if debug:
plt.figure()
for det in frame.detections:
rect = patches.Rectangle(det.top_left,
det.width,
det.height,
linewidth=2,
edgecolor='blue',
facecolor='none')
plt.gca().add_patch(rect)
if tracking is None:
text = '{}'.format(det.label)
else:
text = '{} ~ {}'.format(det.label, det.id)
plt.text(det.top_left[0],
det.top_left[1],
s=text,
color='white',
verticalalignment='top',
bbox={
'color': 'blue',
'pad': 0
})
plt.imshow(im)
plt.axis('off')
# plt.savefig('../video/video_yolo_fine_tune_good/frame_{:04d}'.format(i))
plt.show()
plt.close()
# iou_over_time(frames)
mAP = mean_average_precision(frames)
print("YOLO mAP:", mAP)
def off_the_shelf_ssd(tracking, debug=False, **kwargs):
if cuda.is_available():
torch.set_default_tensor_type('torch.cuda.FloatTensor')
gt = read_annotations(
'../datasets/AICity_data/train/S03/c010/m6-full_annotation.xml')
video = Video("../datasets/AICity_data/train/S03/c010/frames")
trans = transforms.Compose(
[transforms.Resize((300, 300)),
transforms.ToTensor()])
labels = ( # always index 0
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person',
'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor')
model = build_ssd('test', 300, 21) # initialize SSD
model.load_weights('../weights/ssd300_mAP_77.43_v2.pth')
if torch.cuda.is_available():
model = model.cuda()
frames = []
model.eval()
with torch.no_grad():
for i, im in enumerate(video.get_frames()):
im_tensor = trans(im)
im_tensor = im_tensor.view((-1, ) + im_tensor.size())
if torch.cuda.is_available():
im_tensor = im_tensor.cuda()
output = model.forward(im_tensor)
detections = output.data
w = im.width
h = im.height
frame = Frame(i)
frame.ground_truth = gt[frame.id]
# skip j = 0, because it's the background class
for j in (2, 6, 7, 14):
dets = detections[0, j, :]
mask = dets[:, 0].gt(0.).expand(5, dets.size(0)).t()
dets = torch.masked_select(dets, mask).view(-1, 5)
if dets.size(0) == 0:
continue
boxes = dets[:, 1:]
scores = dets[:, 0].cpu().numpy()
cls_dets = np.hstack((boxes.cpu().numpy(),
scores[:,
np.newaxis])).astype(np.float32,
copy=False)
for cls_det in cls_dets:
x1 = int(w * cls_det[0])
y1 = int(h * cls_det[1])
det = Detection(-1,
labels[j - 1], (x1, y1),
width=w * (cls_det[2] - cls_det[0]),
height=h * (cls_det[3] - cls_det[1]),
confidence=cls_det[4])
frame.detections.append(det)
# kalman(frame)
if tracking is not None:
tracking(frame, frames, debug=debug)
frames.append(frame)
if debug:
plt.figure()
for det in frame.detections:
rect = patches.Rectangle(det.top_left,
det.width,
det.height,
linewidth=2,
edgecolor='blue',
facecolor='none')
plt.gca().add_patch(rect)
plt.text(det.top_left[0],
det.top_left[1],
s='{} ~ {}'.format(det.label, det.id),
color='white',
verticalalignment='top',
bbox={
'color': 'blue',
'pad': 0
})
plt.imshow(im)
plt.axis('off')
# plt.savefig('../video/video_ssd_KalmanID/frame_{:04d}'.format(i))
plt.show()
plt.close()
#iou_over_time(frames)
mAP = mean_average_precision(frames)
print("SSD mAP:", mAP)
def dev_step():
results = defaultdict(list)
num_test = 0
num_correct = 0.0
valid_batches = data_helpers.batch_iter(valid_dataset,
FLAGS.batch_size,
1,
target_loss_weight,
FLAGS.max_utter_len,
FLAGS.max_utter_num,
FLAGS.max_response_len,
charVocab,
FLAGS.max_word_length,
shuffle=False)
for valid_batch in valid_batches:
x_utterances, x_response, x_utterances_len, x_response_len, x_utters_num, x_responses_num, x_dist, x_target, x_target_weight, id_pairs, x_u_char, x_u_char_len, x_r_char, x_r_char_len = valid_batch
feed_dict = {
u2u_imn.utterances: x_utterances,
u2u_imn.response: x_response,
u2u_imn.utterances_len: x_utterances_len,
u2u_imn.response_len: x_response_len,
u2u_imn.utters_num: x_utters_num,
u2u_imn.responses_num: x_responses_num,
u2u_imn.distance: x_dist,
u2u_imn.target: x_target,
u2u_imn.target_loss_weight: x_target_weight,
u2u_imn.dropout_keep_prob: 1.0,
u2u_imn.u_charVec: x_u_char,
u2u_imn.u_charLen: x_u_char_len,
u2u_imn.r_charVec: x_r_char,
u2u_imn.r_charLen: x_r_char_len
}
batch_accuracy, predicted_prob = sess.run(
[u2u_imn.accuracy, u2u_imn.probs], feed_dict)
num_test += len(predicted_prob)
if num_test % 1000 == 0:
print(num_test)
num_correct += len(predicted_prob) * batch_accuracy
for i, prob_score in enumerate(predicted_prob):
question_id, response_id, label = id_pairs[i]
results[question_id].append(
(response_id, label, prob_score))
#calculate top-1 precision
print('num_test_samples: {} test_accuracy: {}'.format(
num_test, num_correct / num_test))
accu, precision, recall, f1, loss = metrics.classification_metrics(
results)
print('Accuracy: {}, Precision: {} Recall: {} F1: {} Loss: {}'.
format(accu, precision, recall, f1, loss))
mvp = metrics.mean_average_precision(results)
mrr = metrics.mean_reciprocal_rank(results)
top_1_precision = metrics.top_1_precision(results)
total_valid_query = metrics.get_num_valid_query(results)
print(
'MAP (mean average precision: {}\tMRR (mean reciprocal rank): {}\tTop-1 precision: {}\tNum_query: {}'
.format(mvp, mrr, top_1_precision, total_valid_query))
all_preds = []
for i in range(len(results)):
all_preds.append([r[2] for r in results[str(i)]])
df = pd.DataFrame(all_preds,
columns=[
'prediction_' + str(i)
for i in range(len(all_preds[0]))
])
if not os.path.isdir(FLAGS.output_predictions_folder):
os.makedirs(FLAGS.output_predictions_folder)
with open(
os.path.join(FLAGS.output_predictions_folder,
'config.json'), 'w') as f:
conf = {}
for k, v in FLAGS.__dict__['__flags'].items():
conf[k] = v
conf['ranker'] = "U2U"
conf['seed'] = str(conf['random_seed'])
args_dict = {}
args_dict['args'] = conf
f.write(json.dumps(args_dict, indent=4, sort_keys=True))
df.to_csv(FLAGS.output_predictions_folder + "/predictions.csv",
index=False)
return mrr
