def non_maximum_suppression(predicted_tensors, dbox_params, threshold, threshold2, overlap, img_size=768): dict_conf = get_confidences(predicted_tensors, dbox_params, img_size) dict_conf = sorted(dict_conf.items(), key=lambda x: x[1], reverse=True) list_tuple_candidate = deque([(key, item) for key, item in dict_conf if item > threshold]) list_tuple = list() list_predicted_img = list() max_conf = 0 while True: if len(list_tuple_candidate) == 0 or len(list_tuple) > 20: if max_conf > threshold2: return list_tuple, list_predicted_img else: return list(), list() max_deg = 0 pred, conf = list_tuple_candidate.popleft() max_conf = max(max_conf, conf) predicted_tensor = predicted_tensors[pred[0]] step = img_size / predicted_tensor.shape[2] x_points = np.arange(step / 2 - 0.5, img_size, step) y_points = np.arange(step / 2 - 0.5, img_size, step) dbox_param = dbox_params.iloc[pred[3]] _, predict_x, predict_y, predict_height, predict_width, predict_rotate = predict_boxes_numpy( predicted_tensor, pred[3], pred[1], pred[2], x_points[pred[1]], y_points[pred[2]], dbox_param, step) img_box = coord2_img(predict_x, predict_y, predict_height, predict_width, predict_rotate, img_size) for ref_box in list_predicted_img: matching_degree = calc_matching_degree(img_box / 255.0, ref_box / 255.0) max_deg = max(max_deg, matching_degree) if max_deg < overlap: # when overlap region is small enough list_tuple.append(pred) list_predicted_img.append(img_box)
def suppression(predicted_tensors, dbox_params, threshold, threshold2, overlap, img_size=2048): dict_conf = get_confidences(predicted_tensors, dbox_params) dict_conf = sorted(dict_conf.items(), key=lambda x: x[1][-1]) list_tuple_candidate = deque([ (key, item) for key, item in dict_conf if item[-1] < threshold # assignment for null class is less than threshold ]) list_tuple = list() list_predicted_img = list() list_pred_coords = list() min_conf = 1.0 base_img = np.zeros((img_size, img_size), dtype=np.uint8) while True: if len(list_tuple_candidate) == 0 or len(list_tuple) > 100: if min_conf < threshold2: return list_predicted_img, list_pred_coords else: return list(), list() pred, conf = list_tuple_candidate.popleft() min_conf = min(min_conf, conf[-1]) predicted_tensor = predicted_tensors[pred[0]] assignment, pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height = predict_boxes_numpy_3d( predicted_tensor, pred[3], pred[1], pred[2]) if pred_x < 0 or pred_x >= img_size or pred_y < 0 or pred_y >= img_size: continue xy = base_img[int(pred_x), int(pred_y)] if xy > 0: continue img_box = coord2_img(pred_x, pred_y, pred_length, pred_width, pred_rotate, img_size) degree = calc_overlap(img_box / 255.0, base_img / 255.0) if degree < overlap: # when overlap region is small enough list_tuple.append(pred) list_predicted_img.append(img_box) list_pred_coords.append( (pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height, np.argmax(assignment[:-1]), np.max(assignment[:-1]))) base_img = coord2_img(pred_x, pred_y, pred_length, pred_width, pred_rotate, img_size, img_base=base_img)
def suppression_single(predicted_tensors, dbox_params, threshold, overlap, target_class, img_size): dict_conf = get_confidences(predicted_tensors, dbox_params) dict_conf = sorted(dict_conf.items(), key=lambda x: x[1][target_class], reverse=True) list_tuple_candidate = deque([ (key, item) for key, item in dict_conf if item[target_class] > threshold # assignment for null class is less than threshold ]) if len(list_tuple_candidate) == 0: for key, item in dict_conf: list_tuple_candidate = deque([(key, item)]) break # take only first element list_tuple = list() list_predicted_img = list() list_pred_coords = list() base_img = np.zeros((img_size, img_size), dtype=np.uint8) while True: if len(list_tuple_candidate) == 0 or len(list_tuple) > 100: return list_predicted_img, list_pred_coords pred, conf = list_tuple_candidate.popleft() predicted_tensor = predicted_tensors[pred[0]] assignment, pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height = predict_boxes_numpy_3d( predicted_tensor, pred[3], pred[1], pred[2]) if pred_x < 0 or pred_x >= img_size or pred_y < 0 or pred_y >= img_size: continue xy = base_img[int(pred_x), int(pred_y)] if xy > 0: continue img_box = coord2_img(pred_x, pred_y, pred_length, pred_width, pred_rotate, img_size) degree = calc_overlap(img_box / 255.0, base_img / 255.0) if degree < overlap: # when overlap region is small enough list_tuple.append(pred) list_predicted_img.append(img_box) list_pred_coords.append( (pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height, target_class)) base_img = coord2_img(pred_x, pred_y, pred_length, pred_width, pred_rotate, img_size, img_base=base_img)
def explore(inputs): b, targets, net_out_batch, tree, positive_iou, negative_iou, distance_upper_bound = inputs poss_all = dict() negs_all = list() ins_ind_all = list() target_imgs_cache = dict() target_rotates = np.repeat(targets[['rotate']].values, len(dbox_params), 1) dbox_rotates = np.repeat(np.expand_dims(dbox_params['rotate_vars'], 0), len(targets), 0) angle1 = (target_rotates - dbox_rotates) % math.pi angle2 = (dbox_rotates - target_rotates) % math.pi ideal_is = np.apply_along_axis(arg_min2, 1, np.minimum(angle1, angle2)) for l, l_tensor in enumerate(net_out_batch): size_x = l_tensor.shape[-2] size_y = l_tensor.shape[-1] ll = l_tensor.reshape(len(dbox_params), -1, size_x, size_y) predict_xy = np.stack([ ll[:, 10, :, :].flatten(), # x ll[:, 11, :, :].flatten() # y ]).T dists, target_indeces = tree.query( predict_xy, distance_upper_bound=distance_upper_bound) hit_indeces = target_indeces[dists != np.inf] # ins_ind_all.extend(hit_indeces) hit_ixy = np.arange(len(dbox_params) * size_x * size_y)[dists != np.inf] hit_i = hit_ixy // (size_x * size_y) hit_x = hit_ixy % (size_x * size_y) // size_y hit_y = hit_ixy % (size_x * size_y) % size_y for i, x, y, index in zip(hit_i, hit_x, hit_y, hit_indeces): if i not in ideal_is[index]: negs_all.append((l, x, y, i, b)) continue if index in target_imgs_cache: target_img = target_imgs_cache[index] else: target_coord = targets.iloc[index] target_img = coord2_img(target_coord['x'], target_coord['y'], target_coord['length'], target_coord['width'], target_coord['rotate']) / 255.0 target_imgs_cache[index] = target_img pred_img = coord2_img( ll[i, 10, x, y], # x ll[i, 11, x, y], # y ll[i, 12, x, y], # length ll[i, 13, x, y], # width ll[i, 16, x, y] # rotate ) / 255.0 deg = calc_matching_degree(target_img, pred_img) # logger.debug('deg: {:.3f}'.format(deg)) if deg < negative_iou: negs_all.append((l, x, y, i, b)) elif deg > positive_iou: poss_all[(l, x, y, i, b)] = name_to_class(targets.iloc[index]['name']) ins_ind_all.append(index) non_hit_ixy = np.arange(len(dbox_params) * size_x * size_y)[dists == np.inf] non_hit_i = non_hit_ixy // (size_x * size_y) non_hit_x = non_hit_ixy % (size_x * size_y) // size_y non_hit_y = non_hit_ixy % (size_x * size_y) % size_y random_far_negatives = list() for i, x, y in zip(non_hit_i, non_hit_x, non_hit_y): random_far_negatives.append(((l, x, y, i, b), ll[i, 9, x, y])) hard_far_negatives = [ p[0] for p in sorted(random_far_negatives, key=lambda x: x[1]) ][:500] negs_all.extend(random.sample(hard_far_negatives, 100)) negs_all = list(set(negs_all) - set(poss_all.keys())) assert len(poss_all) == len(ins_ind_all) return poss_all, negs_all, ins_ind_all
def search_boxes(predicted_tensors, target, index, dbox_params, img_size=768): if np.isnan(target.x) and np.isnan(target.y): return return_only_negative(predicted_tensors, index, dbox_params, img_size=768) target_img = coord2_img(target.x, target.y, target.height, target.width, target.rotate) / 255.0 positive_boxes = [] fp_boxes = dict() best_degree = 0 best_box = None for l, input_tensor in enumerate(predicted_tensors): step = img_size / input_tensor.shape[2] x_points = np.arange(step / 2 - 0.5, img_size, step) y_points = np.arange(step / 2 - 0.5, img_size, step) for x, x_point in enumerate(x_points): for y, y_point in enumerate(y_points): # for i in range(len(dbox_params)): i = (dbox_params.rotate_vars - target.rotate).abs().idxmin() # print('i: {}'.format(i)) dbox_param = dbox_params.iloc[i] confidence, predict_x, predict_y, predict_height, predict_width, predict_rotate = predict_boxes_numpy( input_tensor, i, x, y, x_point, y_point, dbox_param, step) for j in range(len(dbox_params)): if j != i: fp_boxes[(l, x, y, j)] = numpy_sigmoid(input_tensor[6 * j, x, y]) if is_far(predict_x, predict_y, target, step): fp_boxes[(l, x, y, i)] = numpy_sigmoid(input_tensor[6 * i, x, y]) continue img_dbox = coord2_img(predict_x, predict_y, predict_height, predict_width, predict_rotate, img_size) img_dbox_norm = img_dbox / 255 matching_degree = calc_matching_degree(target_img, img_dbox_norm) if matching_degree > best_degree: best_degree = matching_degree best_box = (l, x, y, i) if matching_degree > 0.7: positive_boxes.append((l, x, y, i)) if matching_degree < 0.4: fp_boxes[(l, x, y, i)] = numpy_sigmoid(input_tensor[6 * i, x, y]) if len(positive_boxes) == 0 and best_box is not None: positive_boxes.append(best_box) num_train = 0 for positive_box in positive_boxes: if len(positive_box) == 0: continue if positive_box in fp_boxes: fp_boxes.pop(positive_box) num_train += 1 # negative mining # fp_boxes = [key for key, item in fp_boxes.items()] negative_boxes = { key: item for key, item in sorted( fp_boxes.items(), key=lambda x: x[1], reverse=True)[:512] } return positive_boxes, negative_boxes, index