def analyze_outs(predict, labels, label_names, frame_thresh=0.7, dist_threshold=10): # next apply non max suppression # labels = labels[0] num_labels = labels.shape[1] all_matches = [] for i in range(num_labels): ground_truth = labels[:, i] gt_sup, gt_idx = post_processing.nonmax_suppress( ground_truth, frame_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( predict[:, i], frame_thresh) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx, dist_threshold=dist_threshold ) match_dict["tps"] = len(match_dict["tps"]) match_dict["fps"] = len(match_dict["fps"]) # # write processed file # output_name = os.path.join( # out_dir, exp_names[0], 'processed_%s.csv' % label_names[i] # ) # create_proc_file(output_name, gt_sup, predict_sup, match_dict) all_matches.append(match_dict) return all_matches
def analyze_outs(predict, labels, label_names, frame_thresh=0.7): # next apply non max suppression # labels = labels[0] num_labels = labels.shape[1] all_dists = { 'lift': [], 'hand': [], 'grab': [], 'supinate': [], 'mouth': [], 'chew': [] } dist_keys = label_names all_matches = [] for i in range(num_labels): ground_truth = labels[:, i] gt_sup, gt_idx = post_processing.nonmax_suppress( ground_truth, frame_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( predict[:, i], frame_thresh) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx ) # predict_idx = match_dict["tps"] predict_idx = match_dict["fps"] # setup "greedy matching" to make a list of distances for each prediction. dist_mat = numpy.zeros((len(gt_idx), len(predict_idx))) abs_dist_mat = numpy.zeros((len(gt_idx), len(predict_idx))) for j in range(len(gt_idx)): for k in range(len(predict_idx)): # dist_mat[j, k] = predict_idx[k][1] - gt_idx[j] # abs_dist_mat[j, k] = abs(predict_idx[k][1] - gt_idx[j]) dist_mat[j, k] = predict_idx[k] - gt_idx[j] abs_dist_mat[j, k] = abs(predict_idx[k] - gt_idx[j]) # get the min idx for each column (each prediction). min_dists = [] if len(gt_idx) > 0 and len(predict_idx) > 0: min_idx = numpy.argmin(abs_dist_mat, axis=0) for j in range(len(predict_idx)): min_dists.append(dist_mat[min_idx[j], j]) all_dists[dist_keys[i]] += min_dists # if len(gt_idx) > 1 and len(predict_idx) > 1: # import pdb; pdb.set_trace() # match_dict, dist_mat = hungarian_matching.apply_hungarian( # gt_idx, predict_idx # ) # if len(match_dict["tps"]) > 0: # import pdb; pdb.set_trace() # match_dict["tps"] = len(match_dict["tps"]) # match_dict["fps"] = len(match_dict["fps"]) # # write processed file # output_name = os.path.join( # out_dir, exp_names[0], 'processed_%s.csv' % label_names[i] # ) # create_proc_file(output_name, gt_sup, predict_sup, match_dict) # all_matches.append(dist_mat) return all_dists
def eval_scores(scores, score_thresh=0.7, dist_thresh=30.0): gt_sup, gt_idx = post_processing.nonmax_suppress(scores[:, 1], score_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( scores[:, 0], score_thresh) # go through each prediction and see if it is a false positive or true # positive. predicts = predict_sup[predict_idx] is_tp = numpy.zeros(predicts.shape) is_gt = numpy.zeros(predicts.shape) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx, val_threshold=score_thresh, dist_threshold=dist_thresh) for j in range(len(predict_idx)): for i in range(len(gt_idx)): # check if in the right range? # dist mat is buffered by dummy nodes (of the number of gt's) if numpy.argmin(dist_mat[:, j + len(gt_idx)]) < dist_thresh: # is this prediction the lowest score for any gt? dists = dist_mat[i, :] if numpy.argmin(dists) == j + len(gt_idx): is_tp[j] = dists[j + len(gt_idx)] is_gt[j] = True # no reason to check for other matches break return predicts, is_tp, is_gt, len(gt_idx)
def analyze_outs(csv_dir, predict, labels, label_names, frame_thresh=0.1): # next apply non max suppression # labels = labels[0] num_labels = labels.shape[1] # create a mask for the network predictions. # apply an argmax to figure out which locations the network was most sure # that a behavior has occured predict_mask = numpy.zeros(predict.shape) predict_max_idx = numpy.argmax(predict, axis=1) for i in range(len(predict_max_idx)): # "i" should be the row. j = predict_max_idx[i] predict_mask[i, j] = 1 # next convert the mask into predictions using the rules described by # startnet/odas paper. # c_t = argmax is an action. # c_t != c_{t-1} # as_t^{c_t} exceeds a threshold. This isn't available for odas. # set the previous behavior to background c_tminus1 = 6 predict_starts = numpy.zeros(predict.shape) for i in range(predict_mask.shape[0]): c_t = predict_max_idx[i] if c_t != 6 and c_t != c_tminus1: predict_starts[i, c_t] = 1 c_tminus1 = c_t # write predictions back to disk # copy the templates for label_name in label_names: csv_name = "odas_" + label_name + ".csv" base_out = csv_dir sequences_helper.create_html_file(base_out, csv_name, "movie_comb.avi", 30) # write the predictions. exp_name = os.path.basename(base_out) write_csvs(base_out, exp_name, label_names, labels, predict_starts) all_matches = [] for i in range(num_labels): ground_truth = labels[:, i] gt_sup, gt_idx = post_processing.nonmax_suppress( ground_truth, frame_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( predict[:, i], frame_thresh) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx) match_dict["tps"] = len(match_dict["tps"]) match_dict["fps"] = len(match_dict["fps"]) # # write processed file # output_name = os.path.join( # out_dir, exp_names[0], 'processed_%s.csv' % label_names[i] # ) # create_proc_file(output_name, gt_sup, predict_sup, match_dict) all_matches.append(match_dict) return all_matches
def eval_scores(scores, score_thresh=0.7, dist_thresh=30.0): gt_sup, gt_idx = post_processing.nonmax_suppress(scores[:, 1], score_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( scores[:, 0], score_thresh) # go through each prediction and see if it is a false positive or true # positive. predicts = predict_sup[predict_idx] is_tp = numpy.zeros(predicts.shape) is_gt = numpy.zeros(predicts.shape) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx, val_threshold=score_thresh, dist_threshold=dist_thresh) # # loop over the predictions, and create a is_tp_fp indicator # for i in range(len(match_dict["tps"])): # # second element is the prediction index # is_tp[match_dict["tps"][i][1]] = 1 # is_tp = is_tp[predict_idx] # for i in range(len(match_dict["fps"])): # # second element is the prediction index # is_tp[match_dict["tps"][i][1]] = 1 # # try: # # is_tp[match_dict["tps"][i][1] == numpy.asarray(predict_idx)] = 1 # # except: # # import pdb; pdb.set_trace() for j in range(len(predict_idx)): for i in range(len(gt_idx)): # check if in the right range? # dist mat is buffered by dummy nodes (of the number of gt's) if numpy.argmin(dist_mat[:, j + len(gt_idx)]) < dist_thresh: # is this prediction the lowest score for any gt? dists = dist_mat[i, :] if numpy.argmin(dists) == j + len(gt_idx): is_tp[j] = dists[j + len(gt_idx)] is_gt[j] = True # no reason to check for other matches break # else: # is_tp[j] = -1 # import pdb; pdb.set_trace() # if len(gt_idx) > 2: # import pdb; pdb.set_trace() return predicts, is_tp, is_gt, len(gt_idx)
def analyze_outs(out_dir, exp_names, predict, labels, frame_thresh=0.7): # next apply non max suppression labels = labels[0] num_labels = labels.shape[2] all_matches = [] for i in range(num_labels): ground_truth = labels[:, 0, i] gt_sup, gt_idx = post_processing.nonmax_suppress( ground_truth, frame_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( predict[:, 0, i], frame_thresh) match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx) # write processed file output_name = os.path.join(out_dir, exp_names[0].decode("utf-8"), 'processed_%s.csv' % g_label_names[i]) create_proc_file(output_name, gt_sup, predict_sup, match_dict) all_matches.append(match_dict) # import pdb; pdb.set_trace() return all_matches
def proc_prediction_file(predict_file): """proc_prediction Process a single prediction file. Given a csv output of the form frame num, prediction, ground truth. """ # Post processing flow: # nonmax suppress # Apply hungarian matching # Compute stats. csv_data = post_processing.load_predict_csv(predict_file) # always assume there is a ground truth key name, and skip anything called # frames key_names = csv_data.keys() # get the values out of the dictionary ground_truth = [] predict = [] for key in key_names: if "ground truth" in key: ground_truth = csv_data[key] elif "frame" not in key: predict = csv_data[key] # next apply non max suppression gt_sup, gt_idx = post_processing.nonmax_suppress(ground_truth, 0.7) predict_sup, predict_idx = post_processing.nonmax_suppress(predict, 0.7) # hungarian matching match_dict, dist_mat = hungarian_matching.apply_hungarian( gt_idx, predict_idx ) # create the post processed file output_name = predict_file.replace("predict_", "processed_") create_proc_file(output_name, gt_sup, predict_sup, match_dict) return match_dict
def process_csv(filename): values = [] with open(filename, "r") as fid: # first line is header line = fid.readline().strip() line = fid.readline() while line: # csv, want columns 2 and 3. scores = line.split(',')[:3] scores = [float(scores[0]), float(scores[1]), float(scores[2])] values.append(scores) line = fid.readline() values = numpy.asarray(values) frame_thresh = 0.7 gt_sup, gt_idx = post_processing.nonmax_suppress(values[:, 2], frame_thresh) predict_sup, predict_idx = post_processing.nonmax_suppress( values[:, 1], frame_thresh) values = numpy.stack([values[:, 0], predict_sup, gt_sup]).T return values
def create_match_array(opts, net_out, org_labels, label_weight): """Create the match array.""" val_threshold = 0.7 # frame_threshold = [5, 15, 15, 20, 30, 30] frame_threshold = [10, 10, 10, 10, 10, 10] # frame_threshold = 10 y_org = org_labels COST_FP = 20 # COST_FN = 20 net_out = net_out.data.cpu().numpy() num_frames, num_vids, num_classes = net_out.shape TP_weight = np.zeros((num_frames, num_vids, num_classes), dtype="float32") FP_weight = np.zeros((num_frames, num_vids, num_classes), dtype="float32") num_false_neg = [] num_false_pos = [] for i in range(num_vids): temp_false_neg = 0 temp_false_pos = 0 for j in range(num_classes): processed, max_vals = post_processing.nonmax_suppress( net_out[:, i, j], val_threshold) processed = processed.reshape((processed.shape[0], 1)) data = np.zeros((len(processed), 3), dtype="float32") data[:, 0] = list(range(len(processed))) data[:, 1] = processed[:, 0] data[:, 2] = y_org[:, i, j] # if opts["flags"].debug is True: # import pdb; pdb.set_trace() # after suppression, apply hungarian. labelled = np.argwhere(y_org[:, i, j] == 1) labelled = labelled.flatten().tolist() num_labelled = len(labelled) dist_mat = post_processing.create_frame_dists( data, max_vals, labelled) rows, cols, dist_mat = post_processing.apply_hungarian( dist_mat, frame_threshold[j]) # missed classifications # false_neg = len(labelled) - len( # [k for k in range(len(max_vals)) if cols[k] < len(labelled)]) # num_false_neg += false_neg # temp_false_neg += false_neg num_matched = 0 for pos in range(len(max_vals)): ref_idx = max_vals[pos] # if cols[pos] < len(labelled): row_idx = rows[pos] col_idx = cols[pos] if col_idx < len(labelled) and\ dist_mat[row_idx, col_idx] < frame_threshold[j]: # True positive label_idx = labelled[cols[pos]] TP_weight[ref_idx, i, j] = np.abs(ref_idx - label_idx) # import pdb; pdb.set_trace() # if we are reweighting based off label rariety if opts["flags"].reweight is True: TP_weight[ref_idx, i, j] =\ TP_weight[ref_idx, i, j] * label_weight[j] num_matched += 1 else: # False positive FP_weight[ref_idx, i, j] = COST_FP if opts["flags"].reweight is True: FP_weight[ref_idx, i, j] =\ FP_weight[ref_idx, i, j] * label_weight[j] temp_false_pos += 1 temp_false_neg += num_labelled - num_matched num_false_neg.append(temp_false_neg) num_false_pos.append(temp_false_pos) num_false_neg = np.asarray(num_false_neg).astype("float32") return TP_weight, FP_weight, num_false_neg, num_false_pos