def calculate_intersection(tetragon1, tetragon2): """Calculate the intersection area of two tetragons Arguments: tetragon1 -- Array of shape (4,2) defining the first polygon tetragon2 -- Array of shape (4,2) defining the second polygon Returns: float -- The intersection area in pixels """ if not no_intersection(tetragon1, tetragon2): pts3 = get_intersection_pts(tetragon1, tetragon2) pts3 = perspective.order_points(pts3) intersection = calculate_area(pts3) return intersection return 0
def infer_frame(frame, extraction, descriptors, histograms, hparams): """ Predict the room given 1 frame of the video Arguments: frame -- The BGR video frame extraction {FeatureExtraction} -- an instance of the FeatureExtraction class descriptors {dict} -- The collection of descriptors, indexed by image path histograms {dict} -- The collection of histograms, indexed by image path hparams {dict} -- The full hyper parameters dictionary object Returns: (prediction, confidence, frame_annotated) """ points, frame = feature_detection.detect_perspective( frame, hparams['video']) best = None best_score = 0 if len(points) == 4: points = perspective.order_points(points) painting = perspective.perspective_transform(frame, points) descriptor = extraction.extract_keypoints(painting, hparams) histogram = extraction.extract_hist(painting) logits_descriptor = [] logits_histogram = [] labels = [] for path in descriptors: if descriptors[path] is not None and histograms[path] is not None: score_key = extraction.match_keypoints(descriptor, descriptors[path], hparams) score_hist = extraction.compare_hist(histogram, histograms[path]) logits_descriptor.append(score_key) logits_histogram.append(score_hist) labels.append(path) scores_descriptor = math.softmax(logits_descriptor) scores_histogram = math.softmax(logits_histogram) scores = hparams['keypoints_weight'] * scores_descriptor + \ hparams['histogram_weight'] * scores_histogram best_idx = np.argmax(scores) best_score = scores[best_idx] best = labels[best_idx] return best, best_score, frame
def calculate_area(pts): """Calculate the inside area of a tetragon Arguments: pts -- (4,2) array containing 4 corners in orhogonal coordinates Returns: float -- The area in pixels """ pts = perspective.order_points(pts) a = euclid_dist(pts[2], pts[3]) b = euclid_dist(pts[3], pts[0]) c = euclid_dist(pts[0], pts[1]) d = euclid_dist(pts[1], pts[2]) t = 0.5 * (a + b + c + d) angle1 = angle_betw_lines( np.array([pts[2], pts[3]]), np.array([pts[3], pts[0]])) angle2 = angle_betw_lines( np.array([pts[0], pts[1]]), np.array([pts[1], pts[2]])) area = math.sqrt(((t - a) * (t - b) * (t - c) * (t - d)) - (a * b * c * d * ((math.cos((angle1 + angle2)/2)) ** 2))) return area
def get_intersection_pts(pts1, pts2): pts1 = perspective.order_points(pts1) pts2 = perspective.order_points(pts2) pts3 = pts1 # first check if either corner lies completely inside of the other (that's the two first ifs # if not, check which lies most inside to get the inside intersection if not no_intersection(pts1, pts2): if point_inside_quad(pts1[0], pts2): pts3[0] = pts1[0] elif point_inside_quad(pts2[0], pts1): pts3[0] = pts2[0] else: pta = intersections(np.array([pts1[0][0], pts1[0][1], pts1[3][0], pts1[3][1]]), np.array([pts2[0][0], pts2[0][1], pts2[1][0], pts2[1][1]])) ptb = intersections(np.array([pts1[0][0], pts1[0][1], pts1[1][0], pts1[1][1]]), np.array([pts2[0][0], pts2[0][1], pts2[3][0], pts2[3][1]])) if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000: pts3[0] = ptb elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000: pts3[0] = pta elif ptb[0] > pta[0]: pts3[0] = ptb else: pts3[0] = pta if point_inside_quad(pts1[1], pts2): pts3[1] = pts1[1] elif point_inside_quad(pts2[1], pts1): pts3[1] = pts2[1] else: pta = intersections(np.array([pts1[1][0], pts1[1][1], pts1[2][0], pts1[2][1]]), np.array([pts2[0][0], pts2[0][1], pts2[1][0], pts2[1][1]])) ptb = intersections(np.array([pts1[0][0], pts1[0][1], pts1[1][0], pts1[1][1]]), np.array([pts2[1][0], pts2[1][1], pts2[2][0], pts2[2][1]])) if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000: pts3[1] = ptb elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000: pts3[1] = pta elif ptb[0] < pta[0]: pts3[1] = ptb else: pts3[1] = pta if point_inside_quad(pts1[2], pts2): pts3[2] = pts1[2] elif point_inside_quad(pts2[2], pts1): pts3[2] = pts2[2] else: pta = intersections(np.array([pts1[1][0], pts1[1][1], pts1[2][0], pts1[2][1]]), np.array([pts2[2][0], pts2[2][1], pts2[3][0], pts2[3][1]])) ptb = intersections(np.array([pts1[2][0], pts1[2][1], pts1[3][0], pts1[3][1]]), np.array([pts2[1][0], pts2[1][1], pts2[2][0], pts2[2][1]])) if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000: pts3[2] = ptb elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000: pts3[2] = pta elif ptb[0] < pta[0]: pts3[2] = ptb else: pts3[2] = pta if point_inside_quad(pts1[3], pts2): pts3[3] = pts1[3] elif point_inside_quad(pts2[3], pts1): pts3[3] = pts2[3] else: pta = intersections(np.array([pts1[3][0], pts1[3][1], pts1[0][0], pts1[0][1]]), np.array([pts2[2][0], pts2[2][1], pts2[3][0], pts2[3][1]])) ptb = intersections(np.array([pts1[2][0], pts1[2][1], pts1[3][0], pts1[3][1]]), np.array([pts2[0][0], pts2[0][1], pts2[3][0], pts2[3][1]])) if pta[0] < 0 or pta[0] > 10000 or pta[1] < 0 or pta[1] > 10000: pts3[3] = ptb elif ptb[0] < 0 or ptb[0] > 10000 or ptb[1] < 0 or ptb[1] > 10000: pts3[3] = pta elif ptb[0] > pta[0]: pts3[3] = ptb else: pts3[3] = pta return pts3 return np.array([(-1, -1), (-1, -1), (-1, -1), (-1, -1)])
def bounding_rect_2(lines, hparams, shape, theta_threshold=.1): """ Algorithm 2 Pick 4 lines which are most likely to be the edges of the painting, used for perspective correction. Returns: Numpy array of shape (4,2) containing 4 corners, ordered clockwise from the top left. """ lines_polar = [to_polar(l) for l in lines] buckets = [[] for _ in range(18)] for idx, (rho, theta) in enumerate(lines_polar): buckets[int(theta * 17.5 / math.pi)].append(idx) top = np.array([len(b) for b in buckets]).argsort()[-2:] first = list(sorted(buckets[top[0]], key=lambda idx: lines_polar[idx][0])) second = list(sorted(buckets[top[1]], key=lambda idx: lines_polar[idx][0])) indices = [0, 0, len(first)-1, len(second)-1] bad_ratio = True while(bad_ratio and indices[0] < indices[2] and indices[1] < indices[3]): line1 = first[indices[0]] line2 = second[indices[1]] line3 = first[indices[2]] line4 = second[indices[3]] width = abs(lines_polar[line1][0] - lines_polar[line3][0]) height = abs(lines_polar[line2][0] - lines_polar[line4][0]) if out_of_ratio(width, height, hparams['ratio']): if width > height: indices[0] += 1 else: indices[1] += 1 else: corners = np.int32([ intersections(lines[line1], lines[line2]), intersections(lines[line2], lines[line3]), intersections(lines[line3], lines[line4]), intersections(lines[line4], lines[line1]), ]) out_of_frame = False for idx, (x, y) in enumerate(corners): if not (0 <= x < shape[1] and 0 <= y < shape[0]): out_of_frame = True if idx <= 1: indices[idx] += 1 else: indices[idx] -= 1 break bad_ratio = out_of_frame line1 = first[indices[0]] line2 = second[indices[1]] line3 = first[indices[2]] line4 = second[indices[3]] width = abs(lines_polar[line1][0] - lines_polar[line3][0]) height = abs(lines_polar[line2][0] - lines_polar[line4][0]) if out_of_ratio(width, height, hparams['ratio']): # Ratio is still bad, so there is no good bounding rectangle logging.warning('Perspective transform: Bad aspect ratio (%f) ', min(width / height, height/width)) return [] elif width < hparams['min_rect_size'] or height < hparams['min_rect_size']: # Rectangle is too small logging.warning('Perspective transform: Detected rectangle too small (%f) ', min(width, height)) return [] corners = np.int32([ intersections(lines[line1], lines[line2]), intersections(lines[line2], lines[line3]), intersections(lines[line3], lines[line4]), intersections(lines[line4], lines[line1]), ]) corners = perspective.order_points(corners) return corners
def bounding_rect(lines, hparams, theta_threshold=.1): """ Algorithm 1 Pick 4 lines which are most likely to be the edges of the painting, used for perspective correction Returns: Numpy array of shape (4,2) containing 4 corners, ordered clockwise from the top left. """ ratio = hparams['ratio'] if len(lines) < 4: logging.warning( 'Perspective transform: Not enough lines found (%d).', len(lines)) return [] straight = np.pi / 2 parallel = [] perpendicular = [] best = 0 # Foreach line, find +/- parallel and +/- perpendicular lines for i in range(len(lines)): parallel.append([]) perpendicular.append([]) r1, theta1 = to_polar(lines[i]) for j in range(len(lines)): r2, theta2 = to_polar(lines[j]) if i != j: angle_diff = min(abs(theta1 - theta2), abs(theta2 - theta1)) if angle_diff < theta_threshold: parallel[i].append((j, abs(r1-r2))) elif straight - theta_threshold < angle_diff < straight + theta_threshold: perpendicular[i].append((j, r2)) if len(perpendicular[i]) + len(parallel[i]) > len(perpendicular[best]) + len(parallel[best]) \ and len(perpendicular[i]) >= 2 \ and len(parallel[i]) >= 1: best = i # Sort the lines by rho-difference parallel[best].sort(key=lambda x: x[1], reverse=True) perpendicular[best].sort(key=lambda x: x[1], reverse=True) if len(parallel[best]) < 1 or len(perpendicular[best]) < 2: logging.warning( 'Perspective transform: Not enough parallel/perpendicular lines found.') return [] # Initialize indices of the bounding rectangle par = 0 # highest Δrho perp1 = 0 # highest Δrho perp2 = -1 # lowest Δrho # While the ratio of the bounding rectangle is not realistic for a # painting, try to decrease te size and get a better ratio good_ratio = False while not good_ratio: # Get 3 corners of the rectangle p1 = intersections( lines[best], lines[perpendicular[best][perp1][0]]) p2 = intersections( lines[best], lines[perpendicular[best][perp2][0]]) p3 = intersections( lines[parallel[best][par][0]], lines[perpendicular[best][perp1][0]]) # Not really width and height, it can be the other way around as well width = euclid_dist(p1, p2) height = euclid_dist(p1, p3) if out_of_ratio(width, height, ratio): # Bad ratio if width > height: # Look for maximum distance to remove (perp1 or perp2) # Calculate new intersections of best with perp1 + 1 and with perp2 -1 and the corresponding distances p = intersections( lines[best], lines[perpendicular[best][perp1+1][0]]) dist_perp1 = euclid_dist(p2, p) p = intersections( lines[best], lines[perpendicular[best][perp2-1][0]]) dist_perp2 = euclid_dist(p1, p) # change index with biggest distance from previous line if dist_perp1 > dist_perp2 and perp1 + 1 < len(perpendicular[best]) + perp2: perp1 += 1 elif len(perpendicular[best]) - 1 + perp2 > perp1 + 1: perp2 -= 1 else: # It is not possible to make the bounding rectangle smaller good_ratio = True elif par + 1 < len(parallel[best]) - 1: par += 1 else: # It is impossible to make the bounding rectangle smaller good_ratio = True else: good_ratio = True # Calculate final intersections p1 = intersections( lines[best], lines[perpendicular[best][perp1][0]]) p2 = intersections( lines[best], lines[perpendicular[best][perp2][0]]) p3 = intersections( lines[parallel[best][par][0]], lines[perpendicular[best][perp1][0]]) width = euclid_dist(p1, p2) height = euclid_dist(p1, p3) # Check if the ratio is now good if out_of_ratio(width, height, ratio): # Ratio is still bad, so there is no good bounding rectangle logging.warning('Perspective transform: Bad aspect ratio (%f) ', min(width / height, height/width)) return [] elif width < hparams['min_rect_size'] or height < hparams['min_rect_size']: # Rectangle is too small logging.warning('Perspective transform: Detected rectangle too small (%f) ', min(width, height)) return [] else: # The ratio is good, return the bounding rectangle l1 = lines[best] l2 = lines[perpendicular[best][perp1][0]] l3 = lines[parallel[best][par][0]] l4 = lines[perpendicular[best][perp2][0]] corners = np.int32([ intersections(l1, l2), intersections(l2, l3), intersections(l3, l4), intersections(l4, l1), ]) corners = perspective.order_points(corners) return corners