def analyze_outs(predict, labels, label_names, frame_thresh=0.7, dist_threshold=10):
# next apply non max suppression
# labels = labels[0]
num_labels = labels.shape[1]
all_matches = []
for i in range(num_labels):
ground_truth = labels[:, i]
gt_sup, gt_idx = post_processing.nonmax_suppress(
ground_truth, frame_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
predict[:, i], frame_thresh)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx, predict_idx, dist_threshold=dist_threshold
)
match_dict["tps"] = len(match_dict["tps"])
match_dict["fps"] = len(match_dict["fps"])
# # write processed file
# output_name = os.path.join(
# out_dir, exp_names[0], 'processed_%s.csv' % label_names[i]
# )
# create_proc_file(output_name, gt_sup, predict_sup, match_dict)
all_matches.append(match_dict)
return all_matches
def analyze_outs(predict, labels, label_names, frame_thresh=0.7):
# next apply non max suppression
# labels = labels[0]
num_labels = labels.shape[1]
all_dists = {
'lift': [],
'hand': [],
'grab': [],
'supinate': [],
'mouth': [],
'chew': []
}
dist_keys = label_names
all_matches = []
for i in range(num_labels):
ground_truth = labels[:, i]
gt_sup, gt_idx = post_processing.nonmax_suppress(
ground_truth, frame_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
predict[:, i], frame_thresh)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx, predict_idx
)
# predict_idx = match_dict["tps"]
predict_idx = match_dict["fps"]
# setup "greedy matching" to make a list of distances for each prediction.
dist_mat = numpy.zeros((len(gt_idx), len(predict_idx)))
abs_dist_mat = numpy.zeros((len(gt_idx), len(predict_idx)))
for j in range(len(gt_idx)):
for k in range(len(predict_idx)):
# dist_mat[j, k] = predict_idx[k][1] - gt_idx[j]
# abs_dist_mat[j, k] = abs(predict_idx[k][1] - gt_idx[j])
dist_mat[j, k] = predict_idx[k] - gt_idx[j]
abs_dist_mat[j, k] = abs(predict_idx[k] - gt_idx[j])
# get the min idx for each column (each prediction).
min_dists = []
if len(gt_idx) > 0 and len(predict_idx) > 0:
min_idx = numpy.argmin(abs_dist_mat, axis=0)
for j in range(len(predict_idx)):
min_dists.append(dist_mat[min_idx[j], j])
all_dists[dist_keys[i]] += min_dists
# if len(gt_idx) > 1 and len(predict_idx) > 1:
# import pdb; pdb.set_trace()
# match_dict, dist_mat = hungarian_matching.apply_hungarian(
# gt_idx, predict_idx
# )
# if len(match_dict["tps"]) > 0:
# import pdb; pdb.set_trace()
# match_dict["tps"] = len(match_dict["tps"])
# match_dict["fps"] = len(match_dict["fps"])
# # write processed file
# output_name = os.path.join(
# out_dir, exp_names[0], 'processed_%s.csv' % label_names[i]
# )
# create_proc_file(output_name, gt_sup, predict_sup, match_dict)
# all_matches.append(dist_mat)
return all_dists
def eval_scores(scores, score_thresh=0.7, dist_thresh=30.0):
gt_sup, gt_idx = post_processing.nonmax_suppress(scores[:, 1],
score_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
scores[:, 0], score_thresh)
# go through each prediction and see if it is a false positive or true
# positive.
predicts = predict_sup[predict_idx]
is_tp = numpy.zeros(predicts.shape)
is_gt = numpy.zeros(predicts.shape)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx,
predict_idx,
val_threshold=score_thresh,
dist_threshold=dist_thresh)
for j in range(len(predict_idx)):
for i in range(len(gt_idx)):
# check if in the right range?
# dist mat is buffered by dummy nodes (of the number of gt's)
if numpy.argmin(dist_mat[:, j + len(gt_idx)]) < dist_thresh:
# is this prediction the lowest score for any gt?
dists = dist_mat[i, :]
if numpy.argmin(dists) == j + len(gt_idx):
is_tp[j] = dists[j + len(gt_idx)]
is_gt[j] = True
# no reason to check for other matches
break
return predicts, is_tp, is_gt, len(gt_idx)
def analyze_outs(csv_dir, predict, labels, label_names, frame_thresh=0.1):
# next apply non max suppression
# labels = labels[0]
num_labels = labels.shape[1]
# create a mask for the network predictions.
# apply an argmax to figure out which locations the network was most sure
# that a behavior has occured
predict_mask = numpy.zeros(predict.shape)
predict_max_idx = numpy.argmax(predict, axis=1)
for i in range(len(predict_max_idx)):
# "i" should be the row.
j = predict_max_idx[i]
predict_mask[i, j] = 1
# next convert the mask into predictions using the rules described by
# startnet/odas paper.
# c_t = argmax is an action.
# c_t != c_{t-1}
# as_t^{c_t} exceeds a threshold. This isn't available for odas.
# set the previous behavior to background
c_tminus1 = 6
predict_starts = numpy.zeros(predict.shape)
for i in range(predict_mask.shape[0]):
c_t = predict_max_idx[i]
if c_t != 6 and c_t != c_tminus1:
predict_starts[i, c_t] = 1
c_tminus1 = c_t
# write predictions back to disk
# copy the templates
for label_name in label_names:
csv_name = "odas_" + label_name + ".csv"
base_out = csv_dir
sequences_helper.create_html_file(base_out, csv_name, "movie_comb.avi",
30)
# write the predictions.
exp_name = os.path.basename(base_out)
write_csvs(base_out, exp_name, label_names, labels, predict_starts)
all_matches = []
for i in range(num_labels):
ground_truth = labels[:, i]
gt_sup, gt_idx = post_processing.nonmax_suppress(
ground_truth, frame_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
predict[:, i], frame_thresh)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx, predict_idx)
match_dict["tps"] = len(match_dict["tps"])
match_dict["fps"] = len(match_dict["fps"])
# # write processed file
# output_name = os.path.join(
# out_dir, exp_names[0], 'processed_%s.csv' % label_names[i]
# )
# create_proc_file(output_name, gt_sup, predict_sup, match_dict)
all_matches.append(match_dict)
return all_matches
def eval_scores(scores, score_thresh=0.7, dist_thresh=30.0):
gt_sup, gt_idx = post_processing.nonmax_suppress(scores[:, 1],
score_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
scores[:, 0], score_thresh)
# go through each prediction and see if it is a false positive or true
# positive.
predicts = predict_sup[predict_idx]
is_tp = numpy.zeros(predicts.shape)
is_gt = numpy.zeros(predicts.shape)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx,
predict_idx,
val_threshold=score_thresh,
dist_threshold=dist_thresh)
# # loop over the predictions, and create a is_tp_fp indicator
# for i in range(len(match_dict["tps"])):
# # second element is the prediction index
# is_tp[match_dict["tps"][i][1]] = 1
# is_tp = is_tp[predict_idx]
# for i in range(len(match_dict["fps"])):
# # second element is the prediction index
# is_tp[match_dict["tps"][i][1]] = 1
# # try:
# # is_tp[match_dict["tps"][i][1] == numpy.asarray(predict_idx)] = 1
# # except:
# # import pdb; pdb.set_trace()
for j in range(len(predict_idx)):
for i in range(len(gt_idx)):
# check if in the right range?
# dist mat is buffered by dummy nodes (of the number of gt's)
if numpy.argmin(dist_mat[:, j + len(gt_idx)]) < dist_thresh:
# is this prediction the lowest score for any gt?
dists = dist_mat[i, :]
if numpy.argmin(dists) == j + len(gt_idx):
is_tp[j] = dists[j + len(gt_idx)]
is_gt[j] = True
# no reason to check for other matches
break
# else:
# is_tp[j] = -1
# import pdb; pdb.set_trace()
# if len(gt_idx) > 2:
# import pdb; pdb.set_trace()
return predicts, is_tp, is_gt, len(gt_idx)
def analyze_outs(out_dir, exp_names, predict, labels, frame_thresh=0.7):
# next apply non max suppression
labels = labels[0]
num_labels = labels.shape[2]
all_matches = []
for i in range(num_labels):
ground_truth = labels[:, 0, i]
gt_sup, gt_idx = post_processing.nonmax_suppress(
ground_truth, frame_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
predict[:, 0, i], frame_thresh)
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx, predict_idx)
# write processed file
output_name = os.path.join(out_dir, exp_names[0].decode("utf-8"),
'processed_%s.csv' % g_label_names[i])
create_proc_file(output_name, gt_sup, predict_sup, match_dict)
all_matches.append(match_dict)
# import pdb; pdb.set_trace()
return all_matches
def proc_prediction_file(predict_file):
"""proc_prediction
Process a single prediction file. Given a csv output of the form
frame num, prediction, ground truth.
"""
# Post processing flow:
# nonmax suppress
# Apply hungarian matching
# Compute stats.
csv_data = post_processing.load_predict_csv(predict_file)
# always assume there is a ground truth key name, and skip anything called
# frames
key_names = csv_data.keys()
# get the values out of the dictionary
ground_truth = []
predict = []
for key in key_names:
if "ground truth" in key:
ground_truth = csv_data[key]
elif "frame" not in key:
predict = csv_data[key]
# next apply non max suppression
gt_sup, gt_idx = post_processing.nonmax_suppress(ground_truth, 0.7)
predict_sup, predict_idx = post_processing.nonmax_suppress(predict, 0.7)
# hungarian matching
match_dict, dist_mat = hungarian_matching.apply_hungarian(
gt_idx, predict_idx
)
# create the post processed file
output_name = predict_file.replace("predict_", "processed_")
create_proc_file(output_name, gt_sup, predict_sup, match_dict)
return match_dict
def process_csv(filename):
values = []
with open(filename, "r") as fid:
# first line is header
line = fid.readline().strip()
line = fid.readline()
while line:
# csv, want columns 2 and 3.
scores = line.split(',')[:3]
scores = [float(scores[0]), float(scores[1]), float(scores[2])]
values.append(scores)
line = fid.readline()
values = numpy.asarray(values)
frame_thresh = 0.7
gt_sup, gt_idx = post_processing.nonmax_suppress(values[:, 2],
frame_thresh)
predict_sup, predict_idx = post_processing.nonmax_suppress(
values[:, 1], frame_thresh)
values = numpy.stack([values[:, 0], predict_sup, gt_sup]).T
return values
def create_match_array(opts, net_out, org_labels, label_weight):
"""Create the match array."""
val_threshold = 0.7
# frame_threshold = [5, 15, 15, 20, 30, 30]
frame_threshold = [10, 10, 10, 10, 10, 10]
# frame_threshold = 10
y_org = org_labels
COST_FP = 20
# COST_FN = 20
net_out = net_out.data.cpu().numpy()
num_frames, num_vids, num_classes = net_out.shape
TP_weight = np.zeros((num_frames, num_vids, num_classes), dtype="float32")
FP_weight = np.zeros((num_frames, num_vids, num_classes), dtype="float32")
num_false_neg = []
num_false_pos = []
for i in range(num_vids):
temp_false_neg = 0
temp_false_pos = 0
for j in range(num_classes):
processed, max_vals = post_processing.nonmax_suppress(
net_out[:, i, j], val_threshold)
processed = processed.reshape((processed.shape[0], 1))
data = np.zeros((len(processed), 3), dtype="float32")
data[:, 0] = list(range(len(processed)))
data[:, 1] = processed[:, 0]
data[:, 2] = y_org[:, i, j]
# if opts["flags"].debug is True:
# import pdb; pdb.set_trace()
# after suppression, apply hungarian.
labelled = np.argwhere(y_org[:, i, j] == 1)
labelled = labelled.flatten().tolist()
num_labelled = len(labelled)
dist_mat = post_processing.create_frame_dists(
data, max_vals, labelled)
rows, cols, dist_mat = post_processing.apply_hungarian(
dist_mat, frame_threshold[j])
# missed classifications
# false_neg = len(labelled) - len(
# [k for k in range(len(max_vals)) if cols[k] < len(labelled)])
# num_false_neg += false_neg
# temp_false_neg += false_neg
num_matched = 0
for pos in range(len(max_vals)):
ref_idx = max_vals[pos]
# if cols[pos] < len(labelled):
row_idx = rows[pos]
col_idx = cols[pos]
if col_idx < len(labelled) and\
dist_mat[row_idx, col_idx] < frame_threshold[j]:
# True positive
label_idx = labelled[cols[pos]]
TP_weight[ref_idx, i, j] = np.abs(ref_idx - label_idx)
# import pdb; pdb.set_trace()
# if we are reweighting based off label rariety
if opts["flags"].reweight is True:
TP_weight[ref_idx, i, j] =\
TP_weight[ref_idx, i, j] * label_weight[j]
num_matched += 1
else:
# False positive
FP_weight[ref_idx, i, j] = COST_FP
if opts["flags"].reweight is True:
FP_weight[ref_idx, i, j] =\
FP_weight[ref_idx, i, j] * label_weight[j]
temp_false_pos += 1
temp_false_neg += num_labelled - num_matched
num_false_neg.append(temp_false_neg)
num_false_pos.append(temp_false_pos)
num_false_neg = np.asarray(num_false_neg).astype("float32")
return TP_weight, FP_weight, num_false_neg, num_false_pos