def calculate_intersection(tetragon1, tetragon2):
"""Calculate the intersection area of two tetragons
Arguments:
tetragon1 -- Array of shape (4,2) defining the first polygon
tetragon2 -- Array of shape (4,2) defining the second polygon
Returns:
float -- The intersection area in pixels
"""
if not no_intersection(tetragon1, tetragon2):
pts3 = get_intersection_pts(tetragon1, tetragon2)
pts3 = perspective.order_points(pts3)
intersection = calculate_area(pts3)
return intersection
return 0
def infer_frame(frame, extraction, descriptors, histograms, hparams):
""" Predict the room given 1 frame of the video
Arguments:
frame -- The BGR video frame
extraction {FeatureExtraction} -- an instance of the FeatureExtraction class
descriptors {dict} -- The collection of descriptors, indexed by image path
histograms {dict} -- The collection of histograms, indexed by image path
hparams {dict} -- The full hyper parameters dictionary object
Returns:
(prediction, confidence, frame_annotated)
"""
points, frame = feature_detection.detect_perspective(
frame, hparams['video'])
best = None
best_score = 0
if len(points) == 4:
points = perspective.order_points(points)
painting = perspective.perspective_transform(frame, points)
descriptor = extraction.extract_keypoints(painting, hparams)
histogram = extraction.extract_hist(painting)
logits_descriptor = []
logits_histogram = []
labels = []
for path in descriptors:
if descriptors[path] is not None and histograms[path] is not None:
score_key = extraction.match_keypoints(descriptor,
descriptors[path],
hparams)
score_hist = extraction.compare_hist(histogram,
histograms[path])
logits_descriptor.append(score_key)
logits_histogram.append(score_hist)
labels.append(path)
scores_descriptor = math.softmax(logits_descriptor)
scores_histogram = math.softmax(logits_histogram)
scores = hparams['keypoints_weight'] * scores_descriptor + \
hparams['histogram_weight'] * scores_histogram
best_idx = np.argmax(scores)
best_score = scores[best_idx]
best = labels[best_idx]
return best, best_score, frame
def calculate_area(pts):
"""Calculate the inside area of a tetragon
Arguments:
pts -- (4,2) array containing 4 corners in orhogonal coordinates
Returns:
float -- The area in pixels
"""
pts = perspective.order_points(pts)
a = euclid_dist(pts[2], pts[3])
b = euclid_dist(pts[3], pts[0])
c = euclid_dist(pts[0], pts[1])
d = euclid_dist(pts[1], pts[2])
t = 0.5 * (a + b + c + d)
angle1 = angle_betw_lines(
np.array([pts[2], pts[3]]), np.array([pts[3], pts[0]]))
angle2 = angle_betw_lines(
np.array([pts[0], pts[1]]), np.array([pts[1], pts[2]]))
area = math.sqrt(((t - a) * (t - b) * (t - c) * (t - d))
- (a * b * c * d * ((math.cos((angle1 + angle2)/2)) ** 2)))
return area
def get_intersection_pts(pts1, pts2):
pts1 = perspective.order_points(pts1)
pts2 = perspective.order_points(pts2)
pts3 = pts1
# first check if either corner lies completely inside of the other (that's the two first ifs
# if not, check which lies most inside to get the inside intersection
if not no_intersection(pts1, pts2):
if point_inside_quad(pts1[0], pts2):
pts3[0] = pts1[0]
elif point_inside_quad(pts2[0], pts1):
pts3[0] = pts2[0]
else:
pta = intersections(np.array([pts1[0][0], pts1[0][1], pts1[3][0], pts1[3][1]]),
np.array([pts2[0][0], pts2[0][1], pts2[1][0], pts2[1][1]]))
ptb = intersections(np.array([pts1[0][0], pts1[0][1], pts1[1][0], pts1[1][1]]),
np.array([pts2[0][0], pts2[0][1], pts2[3][0], pts2[3][1]]))
if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000:
pts3[0] = ptb
elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000:
pts3[0] = pta
elif ptb[0] > pta[0]:
pts3[0] = ptb
else:
pts3[0] = pta
if point_inside_quad(pts1[1], pts2):
pts3[1] = pts1[1]
elif point_inside_quad(pts2[1], pts1):
pts3[1] = pts2[1]
else:
pta = intersections(np.array([pts1[1][0], pts1[1][1], pts1[2][0], pts1[2][1]]),
np.array([pts2[0][0], pts2[0][1], pts2[1][0], pts2[1][1]]))
ptb = intersections(np.array([pts1[0][0], pts1[0][1], pts1[1][0], pts1[1][1]]),
np.array([pts2[1][0], pts2[1][1], pts2[2][0], pts2[2][1]]))
if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000:
pts3[1] = ptb
elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000:
pts3[1] = pta
elif ptb[0] < pta[0]:
pts3[1] = ptb
else:
pts3[1] = pta
if point_inside_quad(pts1[2], pts2):
pts3[2] = pts1[2]
elif point_inside_quad(pts2[2], pts1):
pts3[2] = pts2[2]
else:
pta = intersections(np.array([pts1[1][0], pts1[1][1], pts1[2][0], pts1[2][1]]),
np.array([pts2[2][0], pts2[2][1], pts2[3][0], pts2[3][1]]))
ptb = intersections(np.array([pts1[2][0], pts1[2][1], pts1[3][0], pts1[3][1]]),
np.array([pts2[1][0], pts2[1][1], pts2[2][0], pts2[2][1]]))
if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000:
pts3[2] = ptb
elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000:
pts3[2] = pta
elif ptb[0] < pta[0]:
pts3[2] = ptb
else:
pts3[2] = pta
if point_inside_quad(pts1[3], pts2):
pts3[3] = pts1[3]
elif point_inside_quad(pts2[3], pts1):
pts3[3] = pts2[3]
else:
pta = intersections(np.array([pts1[3][0], pts1[3][1], pts1[0][0], pts1[0][1]]),
np.array([pts2[2][0], pts2[2][1], pts2[3][0], pts2[3][1]]))
ptb = intersections(np.array([pts1[2][0], pts1[2][1], pts1[3][0], pts1[3][1]]),
np.array([pts2[0][0], pts2[0][1], pts2[3][0], pts2[3][1]]))
if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000:
pts3[3] = ptb
elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000:
pts3[3] = pta
elif ptb[0] > pta[0]:
pts3[3] = ptb
else:
pts3[3] = pta
return pts3
return np.array([(-1, -1), (-1, -1), (-1, -1), (-1, -1)])
def bounding_rect_2(lines, hparams, shape, theta_threshold=.1):
""" Algorithm 2
Pick 4 lines which are most likely to be the edges of the painting,
used for perspective correction.
Returns:
Numpy array of shape (4,2) containing 4 corners, ordered clockwise from
the top left.
"""
lines_polar = [to_polar(l) for l in lines]
buckets = [[] for _ in range(18)]
for idx, (rho, theta) in enumerate(lines_polar):
buckets[int(theta * 17.5 / math.pi)].append(idx)
top = np.array([len(b) for b in buckets]).argsort()[-2:]
first = list(sorted(buckets[top[0]], key=lambda idx: lines_polar[idx][0]))
second = list(sorted(buckets[top[1]], key=lambda idx: lines_polar[idx][0]))
indices = [0, 0, len(first)-1, len(second)-1]
bad_ratio = True
while(bad_ratio and indices[0] < indices[2] and indices[1] < indices[3]):
line1 = first[indices[0]]
line2 = second[indices[1]]
line3 = first[indices[2]]
line4 = second[indices[3]]
width = abs(lines_polar[line1][0] - lines_polar[line3][0])
height = abs(lines_polar[line2][0] - lines_polar[line4][0])
if out_of_ratio(width, height, hparams['ratio']):
if width > height:
indices[0] += 1
else:
indices[1] += 1
else:
corners = np.int32([
intersections(lines[line1], lines[line2]),
intersections(lines[line2], lines[line3]),
intersections(lines[line3], lines[line4]),
intersections(lines[line4], lines[line1]),
])
out_of_frame = False
for idx, (x, y) in enumerate(corners):
if not (0 <= x < shape[1] and 0 <= y < shape[0]):
out_of_frame = True
if idx <= 1:
indices[idx] += 1
else:
indices[idx] -= 1
break
bad_ratio = out_of_frame
line1 = first[indices[0]]
line2 = second[indices[1]]
line3 = first[indices[2]]
line4 = second[indices[3]]
width = abs(lines_polar[line1][0] - lines_polar[line3][0])
height = abs(lines_polar[line2][0] - lines_polar[line4][0])
if out_of_ratio(width, height, hparams['ratio']):
# Ratio is still bad, so there is no good bounding rectangle
logging.warning('Perspective transform: Bad aspect ratio (%f) ',
min(width / height, height/width))
return []
elif width < hparams['min_rect_size'] or height < hparams['min_rect_size']:
# Rectangle is too small
logging.warning('Perspective transform: Detected rectangle too small (%f) ',
min(width, height))
return []
corners = np.int32([
intersections(lines[line1], lines[line2]),
intersections(lines[line2], lines[line3]),
intersections(lines[line3], lines[line4]),
intersections(lines[line4], lines[line1]),
])
corners = perspective.order_points(corners)
return corners
def bounding_rect(lines, hparams, theta_threshold=.1):
""" Algorithm 1
Pick 4 lines which are most likely to be the edges of the painting,
used for perspective correction
Returns:
Numpy array of shape (4,2) containing 4 corners, ordered clockwise from
the top left.
"""
ratio = hparams['ratio']
if len(lines) < 4:
logging.warning(
'Perspective transform: Not enough lines found (%d).', len(lines))
return []
straight = np.pi / 2
parallel = []
perpendicular = []
best = 0
# Foreach line, find +/- parallel and +/- perpendicular lines
for i in range(len(lines)):
parallel.append([])
perpendicular.append([])
r1, theta1 = to_polar(lines[i])
for j in range(len(lines)):
r2, theta2 = to_polar(lines[j])
if i != j:
angle_diff = min(abs(theta1 - theta2), abs(theta2 - theta1))
if angle_diff < theta_threshold:
parallel[i].append((j, abs(r1-r2)))
elif straight - theta_threshold < angle_diff < straight + theta_threshold:
perpendicular[i].append((j, r2))
if len(perpendicular[i]) + len(parallel[i]) > len(perpendicular[best]) + len(parallel[best]) \
and len(perpendicular[i]) >= 2 \
and len(parallel[i]) >= 1:
best = i
# Sort the lines by rho-difference
parallel[best].sort(key=lambda x: x[1], reverse=True)
perpendicular[best].sort(key=lambda x: x[1], reverse=True)
if len(parallel[best]) < 1 or len(perpendicular[best]) < 2:
logging.warning(
'Perspective transform: Not enough parallel/perpendicular lines found.')
return []
# Initialize indices of the bounding rectangle
par = 0 # highest Δrho
perp1 = 0 # highest Δrho
perp2 = -1 # lowest Δrho
# While the ratio of the bounding rectangle is not realistic for a
# painting, try to decrease te size and get a better ratio
good_ratio = False
while not good_ratio:
# Get 3 corners of the rectangle
p1 = intersections(
lines[best], lines[perpendicular[best][perp1][0]])
p2 = intersections(
lines[best], lines[perpendicular[best][perp2][0]])
p3 = intersections(
lines[parallel[best][par][0]], lines[perpendicular[best][perp1][0]])
# Not really width and height, it can be the other way around as well
width = euclid_dist(p1, p2)
height = euclid_dist(p1, p3)
if out_of_ratio(width, height, ratio):
# Bad ratio
if width > height:
# Look for maximum distance to remove (perp1 or perp2)
# Calculate new intersections of best with perp1 + 1 and with perp2 -1 and the corresponding distances
p = intersections(
lines[best], lines[perpendicular[best][perp1+1][0]])
dist_perp1 = euclid_dist(p2, p)
p = intersections(
lines[best], lines[perpendicular[best][perp2-1][0]])
dist_perp2 = euclid_dist(p1, p)
# change index with biggest distance from previous line
if dist_perp1 > dist_perp2 and perp1 + 1 < len(perpendicular[best]) + perp2:
perp1 += 1
elif len(perpendicular[best]) - 1 + perp2 > perp1 + 1:
perp2 -= 1
else:
# It is not possible to make the bounding rectangle smaller
good_ratio = True
elif par + 1 < len(parallel[best]) - 1:
par += 1
else:
# It is impossible to make the bounding rectangle smaller
good_ratio = True
else:
good_ratio = True
# Calculate final intersections
p1 = intersections(
lines[best], lines[perpendicular[best][perp1][0]])
p2 = intersections(
lines[best], lines[perpendicular[best][perp2][0]])
p3 = intersections(
lines[parallel[best][par][0]], lines[perpendicular[best][perp1][0]])
width = euclid_dist(p1, p2)
height = euclid_dist(p1, p3)
# Check if the ratio is now good
if out_of_ratio(width, height, ratio):
# Ratio is still bad, so there is no good bounding rectangle
logging.warning('Perspective transform: Bad aspect ratio (%f) ',
min(width / height, height/width))
return []
elif width < hparams['min_rect_size'] or height < hparams['min_rect_size']:
# Rectangle is too small
logging.warning('Perspective transform: Detected rectangle too small (%f) ',
min(width, height))
return []
else:
# The ratio is good, return the bounding rectangle
l1 = lines[best]
l2 = lines[perpendicular[best][perp1][0]]
l3 = lines[parallel[best][par][0]]
l4 = lines[perpendicular[best][perp2][0]]
corners = np.int32([
intersections(l1, l2),
intersections(l2, l3),
intersections(l3, l4),
intersections(l4, l1),
])
corners = perspective.order_points(corners)
return corners