def non_maximum_suppression(predicted_tensors,
dbox_params,
threshold,
threshold2,
overlap,
img_size=768):
dict_conf = get_confidences(predicted_tensors, dbox_params, img_size)
dict_conf = sorted(dict_conf.items(), key=lambda x: x[1], reverse=True)
list_tuple_candidate = deque([(key, item) for key, item in dict_conf
if item > threshold])
list_tuple = list()
list_predicted_img = list()
max_conf = 0
while True:
if len(list_tuple_candidate) == 0 or len(list_tuple) > 20:
if max_conf > threshold2:
return list_tuple, list_predicted_img
else:
return list(), list()
max_deg = 0
pred, conf = list_tuple_candidate.popleft()
max_conf = max(max_conf, conf)
predicted_tensor = predicted_tensors[pred[0]]
step = img_size / predicted_tensor.shape[2]
x_points = np.arange(step / 2 - 0.5, img_size, step)
y_points = np.arange(step / 2 - 0.5, img_size, step)
dbox_param = dbox_params.iloc[pred[3]]
_, predict_x, predict_y, predict_height, predict_width, predict_rotate = predict_boxes_numpy(
predicted_tensor, pred[3], pred[1], pred[2], x_points[pred[1]],
y_points[pred[2]], dbox_param, step)
img_box = coord2_img(predict_x, predict_y, predict_height,
predict_width, predict_rotate, img_size)
for ref_box in list_predicted_img:
matching_degree = calc_matching_degree(img_box / 255.0,
ref_box / 255.0)
max_deg = max(max_deg, matching_degree)
if max_deg < overlap:
# when overlap region is small enough
list_tuple.append(pred)
list_predicted_img.append(img_box)
def suppression(predicted_tensors,
dbox_params,
threshold,
threshold2,
overlap,
img_size=2048):
dict_conf = get_confidences(predicted_tensors, dbox_params)
dict_conf = sorted(dict_conf.items(), key=lambda x: x[1][-1])
list_tuple_candidate = deque([
(key, item) for key, item in dict_conf if item[-1] <
threshold # assignment for null class is less than threshold
])
list_tuple = list()
list_predicted_img = list()
list_pred_coords = list()
min_conf = 1.0
base_img = np.zeros((img_size, img_size), dtype=np.uint8)
while True:
if len(list_tuple_candidate) == 0 or len(list_tuple) > 100:
if min_conf < threshold2:
return list_predicted_img, list_pred_coords
else:
return list(), list()
pred, conf = list_tuple_candidate.popleft()
min_conf = min(min_conf, conf[-1])
predicted_tensor = predicted_tensors[pred[0]]
assignment, pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height = predict_boxes_numpy_3d(
predicted_tensor, pred[3], pred[1], pred[2])
if pred_x < 0 or pred_x >= img_size or pred_y < 0 or pred_y >= img_size:
continue
xy = base_img[int(pred_x), int(pred_y)]
if xy > 0:
continue
img_box = coord2_img(pred_x, pred_y, pred_length, pred_width,
pred_rotate, img_size)
degree = calc_overlap(img_box / 255.0, base_img / 255.0)
if degree < overlap:
# when overlap region is small enough
list_tuple.append(pred)
list_predicted_img.append(img_box)
list_pred_coords.append(
(pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z,
pred_height, np.argmax(assignment[:-1]),
np.max(assignment[:-1])))
base_img = coord2_img(pred_x,
pred_y,
pred_length,
pred_width,
pred_rotate,
img_size,
img_base=base_img)
def suppression_single(predicted_tensors, dbox_params, threshold, overlap,
target_class, img_size):
dict_conf = get_confidences(predicted_tensors, dbox_params)
dict_conf = sorted(dict_conf.items(),
key=lambda x: x[1][target_class],
reverse=True)
list_tuple_candidate = deque([
(key, item) for key, item in dict_conf if item[target_class] >
threshold # assignment for null class is less than threshold
])
if len(list_tuple_candidate) == 0:
for key, item in dict_conf:
list_tuple_candidate = deque([(key, item)])
break # take only first element
list_tuple = list()
list_predicted_img = list()
list_pred_coords = list()
base_img = np.zeros((img_size, img_size), dtype=np.uint8)
while True:
if len(list_tuple_candidate) == 0 or len(list_tuple) > 100:
return list_predicted_img, list_pred_coords
pred, conf = list_tuple_candidate.popleft()
predicted_tensor = predicted_tensors[pred[0]]
assignment, pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z, pred_height = predict_boxes_numpy_3d(
predicted_tensor, pred[3], pred[1], pred[2])
if pred_x < 0 or pred_x >= img_size or pred_y < 0 or pred_y >= img_size:
continue
xy = base_img[int(pred_x), int(pred_y)]
if xy > 0:
continue
img_box = coord2_img(pred_x, pred_y, pred_length, pred_width,
pred_rotate, img_size)
degree = calc_overlap(img_box / 255.0, base_img / 255.0)
if degree < overlap:
# when overlap region is small enough
list_tuple.append(pred)
list_predicted_img.append(img_box)
list_pred_coords.append(
(pred_x, pred_y, pred_length, pred_width, pred_rotate, pred_z,
pred_height, target_class))
base_img = coord2_img(pred_x,
pred_y,
pred_length,
pred_width,
pred_rotate,
img_size,
img_base=base_img)
def explore(inputs):
b, targets, net_out_batch, tree, positive_iou, negative_iou, distance_upper_bound = inputs
poss_all = dict()
negs_all = list()
ins_ind_all = list()
target_imgs_cache = dict()
target_rotates = np.repeat(targets[['rotate']].values, len(dbox_params), 1)
dbox_rotates = np.repeat(np.expand_dims(dbox_params['rotate_vars'], 0),
len(targets), 0)
angle1 = (target_rotates - dbox_rotates) % math.pi
angle2 = (dbox_rotates - target_rotates) % math.pi
ideal_is = np.apply_along_axis(arg_min2, 1, np.minimum(angle1, angle2))
for l, l_tensor in enumerate(net_out_batch):
size_x = l_tensor.shape[-2]
size_y = l_tensor.shape[-1]
ll = l_tensor.reshape(len(dbox_params), -1, size_x, size_y)
predict_xy = np.stack([
ll[:, 10, :, :].flatten(), # x
ll[:, 11, :, :].flatten() # y
]).T
dists, target_indeces = tree.query(
predict_xy, distance_upper_bound=distance_upper_bound)
hit_indeces = target_indeces[dists != np.inf]
# ins_ind_all.extend(hit_indeces)
hit_ixy = np.arange(len(dbox_params) * size_x *
size_y)[dists != np.inf]
hit_i = hit_ixy // (size_x * size_y)
hit_x = hit_ixy % (size_x * size_y) // size_y
hit_y = hit_ixy % (size_x * size_y) % size_y
for i, x, y, index in zip(hit_i, hit_x, hit_y, hit_indeces):
if i not in ideal_is[index]:
negs_all.append((l, x, y, i, b))
continue
if index in target_imgs_cache:
target_img = target_imgs_cache[index]
else:
target_coord = targets.iloc[index]
target_img = coord2_img(target_coord['x'], target_coord['y'],
target_coord['length'],
target_coord['width'],
target_coord['rotate']) / 255.0
target_imgs_cache[index] = target_img
pred_img = coord2_img(
ll[i, 10, x, y], # x
ll[i, 11, x, y], # y
ll[i, 12, x, y], # length
ll[i, 13, x, y], # width
ll[i, 16, x, y] # rotate
) / 255.0
deg = calc_matching_degree(target_img, pred_img)
# logger.debug('deg: {:.3f}'.format(deg))
if deg < negative_iou:
negs_all.append((l, x, y, i, b))
elif deg > positive_iou:
poss_all[(l, x, y, i,
b)] = name_to_class(targets.iloc[index]['name'])
ins_ind_all.append(index)
non_hit_ixy = np.arange(len(dbox_params) * size_x *
size_y)[dists == np.inf]
non_hit_i = non_hit_ixy // (size_x * size_y)
non_hit_x = non_hit_ixy % (size_x * size_y) // size_y
non_hit_y = non_hit_ixy % (size_x * size_y) % size_y
random_far_negatives = list()
for i, x, y in zip(non_hit_i, non_hit_x, non_hit_y):
random_far_negatives.append(((l, x, y, i, b), ll[i, 9, x, y]))
hard_far_negatives = [
p[0] for p in sorted(random_far_negatives, key=lambda x: x[1])
][:500]
negs_all.extend(random.sample(hard_far_negatives, 100))
negs_all = list(set(negs_all) - set(poss_all.keys()))
assert len(poss_all) == len(ins_ind_all)
return poss_all, negs_all, ins_ind_all
def search_boxes(predicted_tensors, target, index, dbox_params, img_size=768):
if np.isnan(target.x) and np.isnan(target.y):
return return_only_negative(predicted_tensors,
index,
dbox_params,
img_size=768)
target_img = coord2_img(target.x, target.y, target.height, target.width,
target.rotate) / 255.0
positive_boxes = []
fp_boxes = dict()
best_degree = 0
best_box = None
for l, input_tensor in enumerate(predicted_tensors):
step = img_size / input_tensor.shape[2]
x_points = np.arange(step / 2 - 0.5, img_size, step)
y_points = np.arange(step / 2 - 0.5, img_size, step)
for x, x_point in enumerate(x_points):
for y, y_point in enumerate(y_points):
# for i in range(len(dbox_params)):
i = (dbox_params.rotate_vars - target.rotate).abs().idxmin()
# print('i: {}'.format(i))
dbox_param = dbox_params.iloc[i]
confidence, predict_x, predict_y, predict_height, predict_width, predict_rotate = predict_boxes_numpy(
input_tensor, i, x, y, x_point, y_point, dbox_param, step)
for j in range(len(dbox_params)):
if j != i:
fp_boxes[(l, x, y,
j)] = numpy_sigmoid(input_tensor[6 * j, x,
y])
if is_far(predict_x, predict_y, target, step):
fp_boxes[(l, x, y, i)] = numpy_sigmoid(input_tensor[6 * i,
x, y])
continue
img_dbox = coord2_img(predict_x, predict_y, predict_height,
predict_width, predict_rotate, img_size)
img_dbox_norm = img_dbox / 255
matching_degree = calc_matching_degree(target_img,
img_dbox_norm)
if matching_degree > best_degree:
best_degree = matching_degree
best_box = (l, x, y, i)
if matching_degree > 0.7:
positive_boxes.append((l, x, y, i))
if matching_degree < 0.4:
fp_boxes[(l, x, y, i)] = numpy_sigmoid(input_tensor[6 * i,
x, y])
if len(positive_boxes) == 0 and best_box is not None:
positive_boxes.append(best_box)
num_train = 0
for positive_box in positive_boxes:
if len(positive_box) == 0:
continue
if positive_box in fp_boxes:
fp_boxes.pop(positive_box)
num_train += 1
# negative mining
# fp_boxes = [key for key, item in fp_boxes.items()]
negative_boxes = {
key: item
for key, item in sorted(
fp_boxes.items(), key=lambda x: x[1], reverse=True)[:512]
}
return positive_boxes, negative_boxes, index