def get_cs_pointing_data(self, frame):
frame_ycrcb = cv.cvtColor(frame, cv.COLOR_BGR2YCrCb)
skin_binary = gesture.apply_skin_hist2d(frame_ycrcb,
self.skin_prob_binary_crcb)
blur = cv.GaussianBlur(skin_binary, (11, 11), 0)
ret, thresh = cv.threshold(blur, 50, 255, cv.THRESH_BINARY)
# Detect one or two hands
# One
cnts = gesture.get_biggest_contours(thresh, 40)
# Two
# cnts = gesture.get_biggest_two_contours(thresh, 40)
if cnts:
hand = cnts[0]
icx, icy = gesture.get_contour_centroid(hand)
fingertips, p1_p2_points, defects, hull = gesture.get_fingertips(
hand, icy)
if fingertips.shape[0] == 1:
fingertip = fingertips[0]
p1_p2_points = p1_p2_points[0]
p3 = p1_p2_points[2]
return fingertip, p3
else:
return None, None
return None, None
def process_frame(self, frame):
# cropping and resizing current frame
frame_cropped = crop_and_resize(frame, self._crops)
frame_ycrcb = cv.cvtColor(frame_cropped, cv.COLOR_BGR2YCrCb)
# Get initial skin binary
skin_binary = gesture.apply_skin_hist2d(frame_ycrcb, self.skin_model)
return skin_binary, frame_cropped
def get_gwr_data(self, frame, last_five_angles):
frame_ycrcb = cv.cvtColor(frame, cv.COLOR_BGR2YCrCb)
skin_binary = gesture.apply_skin_hist2d(frame_ycrcb,
self.skin_prob_binary_crcb)
blur = cv.GaussianBlur(skin_binary, (11, 11), 0)
ret, thresh = cv.threshold(blur, 50, 255, cv.THRESH_BINARY)
# Detect one or two hands
# One
cnts = gesture.get_biggest_contours(thresh, 40)
# Two
# cnts = gesture.get_biggest_two_contours(thresh, 40)
if cnts:
hand = cnts[0]
icx, icy = gesture.get_contour_centroid(hand)
fingertips, p1_p2_points, defects, hull = gesture.get_fingertips(
hand, icy)
if fingertips.shape[0] == 1:
fingertip = fingertips[0]
fitted_line = gesture.calculate_contour_line(frame, hand)
angle = gesture.get_hand_features_for_GWR(
frame, fingertip, fitted_line)
mean_angle, last_five_angles = self.handle_angle_list(
angle, last_five_angles)
return mean_angle, icx, icy, f, last_five_angles
return None, None, None, None, last_five_angles
return None, None, None, None, last_five_angles
def evaluate_gwr_with_amb_detection(self, testing_data):
test_data, test_labels, side_data = testing_data
samples = test_data.shape[0]
number_nodes = self.weights.shape[0]
distances = np.zeros(number_nodes)
activations = np.zeros(samples)
intersection_over_union = []
# List of lists: 0: overall, 1-4 respective ambiguity class
# In list 0. TP, 1. FP, 2. FN, 3. misses, 4. correct_amb 5. false noise 6. total count
confusion_values = [[0, 0, 0, 0, 0, 0, samples],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0]]
for iteration in range(0, samples):
observation = test_data[iteration]
c_side_data = side_data[iteration]
filename = c_side_data[0]
ambiguity_class = int(filename.split("_")[0][-1])
# label in this case => 0: red, 1: yellow, 2: green
color = int(c_side_data[1])
confusion_values[ambiguity_class][6] += 1
corres_image = cv.imread("resources/current_training/bb_test/all_obj_permutations/" + filename + ".jpg")
frame_ycrcb = cv.cvtColor(corres_image, cv.COLOR_BGR2YCrCb)
skin_binary = gesture.apply_skin_hist2d(frame_ycrcb, self.skin_model)
frame, hand_positions_t0, tracking_state = state.detection_step(corres_image, skin_binary, [None, None],
"None")
# saves outcome for visualizations
outcome = ""
for i in range(0, number_nodes):
distances[i] = self.metric_distance(self.weights[i], observation)
# test_bbs: 1: prediction, target bb, if available other false positive bbs
first_index = distances.argmin()
first_distance = distances.min()
activations[iteration] = math.exp(-first_distance)
if activations[iteration] > self.aT:
p_label = self.labels[first_index]
# here activate neighborhood approach or take 4 best nodes
# neighborhood_labels = self.check_neighborhood(first_index)
x_best_labels = self.get_x_best_labels(distances, self.x_b_l_value)
union_of_best = self.union_bounding_boxes(x_best_labels)
bmu = self.convert_bb_to_pxv(p_label)
ambiguity, bounding_boxes, b_frame = obj_d.check_for_ambiguity(corres_image, union_of_best,
hand_positions_t0)
# Here infer which of the detected object is the target and get target and false bbs
# compare gt and detected positions
iou_target = []
if ambiguity:
# object_bb: 0: "object_color", 1: bb_wh, 2: bb])
detected = False
for d_object in bounding_boxes:
if d_object[2] == color:
iou_target.append(self.calc_iou(d_object[0], bmu))
iou = self.calc_iou(d_object[0], bmu)
if iou > .5:
detected = True
if d_object[2] == color:
# add TP
confusion_values[0][0] += 1
confusion_values[ambiguity_class][0] += 1
intersection_over_union.append(iou)
outcome = "TP_"
else:
# add FP
confusion_values[0][1] += 1
confusion_values[ambiguity_class][1] += 1
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
outcome = "FP_"
outcome += "FN_"
if len(iou_target) > 0:
intersection_over_union.append(iou_target[0])
# self.visualize_pointings(d_object[0], bmu, iou, outcome, filename,
# color, d_object[2], iteration)
if not detected:
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
# add Miss
confusion_values[0][3] += 1
confusion_values[ambiguity_class][3] += 1
outcome = "FN_Miss"
# self.visualize_pointings(bounding_boxes, bmu, 0.0, outcome, filename,
# color, None, iteration)
# add correct amb
if ambiguity_class > 2:
confusion_values[0][4] += 1
confusion_values[ambiguity_class][4] += 1
else:
if len(bounding_boxes) > 0:
d_object = bounding_boxes[0]
if d_object[2] == color:
iou_target.append(self.calc_iou(d_object[0], bmu))
confusion_values[0][0] += 1
confusion_values[ambiguity_class][0] += 1
intersection_over_union.append(self.calc_iou(d_object[0], bmu))
outcome = "TP_"
# self.visualize_pointings(d_object[0], union_of_best, 0.0, outcome, filename,
# color, d_object[2], iteration)
else:
# add FP
confusion_values[0][1] += 1
confusion_values[ambiguity_class][1] += 1
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
outcome = "FP_FN_"
# self.visualize_pointings(d_object[0], union_of_best, 0.0, outcome, filename,
# color, d_object[2], iteration)
else:
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
# add Miss
confusion_values[0][3] += 1
confusion_values[ambiguity_class][3] += 1
outcome = "FN_Miss"
# self.visualize_pointings(bounding_boxes, union_of_best, 0.0, outcome, filename,
# color, None, iteration)
# add correct amb
if ambiguity_class < 3:
confusion_values[0][4] += 1
confusion_values[ambiguity_class][4] += 1
else:
# add false amb detec
confusion_values[0][5] += 1
confusion_values[ambiguity_class][5] += 1
return np.asarray(intersection_over_union), np.asarray(confusion_values)
def calc_raw_confusion_data(data, side_data, dimensions):
# Bayesian Hist
skin_prob_crcb_link = "resources/skin_color_data/histograms/skin_probabilities_crcb.npy"
thresh_crcb = 10
skin_prob_binary_crcb = gesture.get_skin_histogram(thresh_crcb,
skin_prob_crcb_link)
sample_count = data.shape[0]
# List of lists: 0: overall, 1-4 respective ambiguity class
# In list 0. TP, 1. FP, 2. FN, 3. misses, 4. no intersections 5. total count
confusion_values = [[0, 0, 0, 0, 0, sample_count], [0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0]]
pointing_qualities_tp = []
pointing_qualities_fp = []
for iteration in range(0, sample_count):
observation = data[iteration]
c_side_data = side_data[iteration]
filename = c_side_data[0]
ambiguity_class = int(filename.split("_")[0][-1])
# label in this case => 0: red, 1: yellow, 2: green
target_color = int(c_side_data[1])
confusion_values[ambiguity_class][5] += 1
corres_image = cv.imread(
"resources/current_training/bb_test/all_obj_permutations/" +
filename + ".jpg")
frame_ycrcb = cv.cvtColor(corres_image, cv.COLOR_BGR2YCrCb)
skin_binary = gesture.apply_skin_hist2d(frame_ycrcb,
skin_prob_binary_crcb)
frame, hand_positions_t0, tracking_state = state.detection_step(
corres_image, skin_binary, [None, None], "None")
object_bb = []
detected_objects = obj_d.detect_objects(corres_image,
hand_positions_t0)
for i, d_object in enumerate(detected_objects):
if d_object is not None:
bb, color_str, color_int = d_object
object_bb.append([
color_int, (bb[0], bb[1], bb[2] - bb[0], bb[3] - bb[1]), bb
])
fingertip = observation[0], observation[1]
p3 = observation[2], observation[3]
hit_target, (pointing_quality,
p_color) = calculate_pointing(fingertip, p3, dimensions,
object_bb)
if hit_target is not None:
if hit_target:
if pointing_quality > 0.2:
if target_color == p_color:
# TP
confusion_values[0][0] += 1
confusion_values[ambiguity_class][0] += 1
pointing_qualities_tp.append(pointing_quality)
else:
# add FP
confusion_values[0][1] += 1
confusion_values[ambiguity_class][1] += 1
pointing_qualities_fp.append(pointing_quality)
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
else:
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
# add Miss
confusion_values[0][3] += 1
confusion_values[ambiguity_class][3] += 1
else:
# add FN
confusion_values[0][2] += 1
confusion_values[ambiguity_class][2] += 1
# add Miss
confusion_values[0][3] += 1
confusion_values[ambiguity_class][3] += 1
else:
# add FN
confusion_values[0][4] += 1
confusion_values[ambiguity_class][4] += 1
return confusion_values, [pointing_qualities_tp, pointing_qualities_fp]