def objective_function(x, *args): cur_vectors = unpack_x(x) cur_points = vectors_to_points(photo, image, cur_vectors) # line-segment errors residuals = [ weights[i_l, i_p] * line_residual(all_lines[i_l], p) for i_p, p in enumerate(cur_points) for i_l in np.flatnonzero(weights[:, i_p]) ] # penalize deviations from 45 or 90 degree angles if lambda_perp: residuals += [ lambda_perp * math.sin(4 * math.acos(abs_dot(v, w))) for i_v, v in enumerate(cur_vectors) for w in cur_vectors[:i_v] ] return residuals
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False): """ Return (uvnb, num_vanishing_points) """ print 'estimate_uvnb for shape_id: %s' % shape.id # local import to avoid cyclic dependencies from shapes.utils import parse_vertices, parse_triangles, \ parse_segments, bbox_vertices # load photo photo = shape.photo if not photo.vanishing_lines or not photo.vanishing_points: raise ValueError("Vanishing points not computed") if not photo.focal_y: raise ValueError("Photo does not have focal_y") vlines = json.loads(photo.vanishing_lines) vpoints = json.loads(photo.vanishing_points) vvectors = copy.copy(photo.vanishing_vectors()) if len(vlines) != len(vpoints): raise ValueError("Invalid vanishing points data structure") # add any missing vanishing points vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75) # find vanishing lines inside shape vertices = parse_vertices(shape.vertices) segments = parse_segments(shape.triangles) triangles = parse_triangles(shape.triangles) # intersect shapes with segments counts = [] for idx in xrange(len(vlines)): # re-pack for geom routines query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]] from common.geom import triangles_segments_intersections_only n = len( triangles_segments_intersections_only(vertices, segments, triangles, query_segments)) if n >= 5: counts.append((n, idx)) counts.sort(key=lambda x: x[0], reverse=True) # function to judge normals: its vanishing line can't intersect the shape. def auto_normal_acceptable(n): sign = None for (x, y) in vertices: # vanishing line line = (n[0], n[1], n[2] * photo.focal_y) # signed distance d = ((x - 0.5) * photo.aspect_ratio * line[0] + (0.5 - y) * line[1] + # flip y line[2]) if abs(d) < 0.05: return False elif sign is None: sign = (d > 0) else: if sign != (d > 0): return False return True # find coordinate frame best_n = None best_u = None method = None num_vanishing_lines = 0 # make sure shape has label if not shape.label_pos_x or not shape.label_pos_y: from shapes.utils import update_shape_label_pos update_shape_label_pos(shape) # place label in 3D b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1 b = [(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) / photo.focal_y, (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y, b_z] # estimate closest human normal human_labels = list( ShapeRectifiedNormalLabel.objects.filter( shape=shape, automatic=False, correct_score__isnull=False, ).order_by('-correct_score')) human_labels += list( ShapeRectifiedNormalLabel.objects.filter(shape=shape, automatic=False, correct_score__isnull=True)) if human_labels: for label in human_labels: human_u = label.u() human_n = label.n() b = list(label.uvnb_numpy()[0:3, 3].flat) # find best normal best_n_dot = 0.9 best_n = None for n in vvectors: d = abs_dot(human_n, n) if d > best_n_dot: best_n_dot = d best_n = n method = 'S' # if there is a match find u and quit if best_n is not None: # find best u best_u_dot = 0 best_u = None for u in vvectors: if abs_dot(u, best_n) < 0.1: d = abs_dot(human_u, u) if d > best_u_dot: best_u_dot = d best_u = u break # try using object label if best_n is None and shape.name: if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'): best_y = 0.9 for v in vvectors[0:3]: if abs(v[1]) > best_y: best_y = abs(v[1]) best_n = v method = 'O' # try fully automatic method if human normals are not good enough if (try_fully_automatic and best_n is None and len(vpoints) >= 3 and len(counts) >= 2 and (shape.substance is None or shape.substance.name != 'Painted')): # choose two dominant vanishing points best_u = vvectors[counts[0][1]] best_v = vvectors[counts[1][1]] # don't try and auto-rectify frontal surfaces if auto_normal_acceptable(normalized_cross(best_u, best_v)): num_vanishing_lines = counts[0][0] + counts[1][0] uv_dot = abs_dot(best_u, best_v) print 'u dot v = %s' % uv_dot else: best_u, best_v = None, None uv_dot = None # make sure these vectors are accurate if not uv_dot or uv_dot > 0.05: # try and find two other orthogonal vanishing points best_dot = 0.05 best_u = None best_v = None for c1, i1 in counts: for c2, i2 in counts[:i1]: d = abs_dot(vvectors[i1], vvectors[i2]) if d < best_dot: # don't try and auto-rectify frontal surfaces if auto_normal_acceptable( normalized_cross(vvectors[i1], vvectors[i2])): best_dot = d if c1 > c2: best_u = vvectors[i1] best_v = vvectors[i2] else: best_u = vvectors[i2] best_v = vvectors[i1] num_vanishing_lines = c1 + c2 if best_u is not None and best_v is not None: best_n = normalized_cross(best_u, best_v) method = 'A' # give up for some classes of objects if shape.name: name = shape.name.name.lower() if ((abs(best_n[1]) > 0.5 and name in ('wall', 'door', 'window')) or (abs(best_n[1]) < 0.5 and name in ('floor', 'ceiling', 'table', 'worktop/countertop', 'carpet/rug'))): method = best_u = best_v = best_n = None num_vanishing_lines = 0 # for walls that touch the edge of the photo, try using bbox center as a # vanishing point (i.e. assume top/bottom shapes are horizontal, side # shapes are vertical) if (try_fully_automatic and best_n is None and shape.name and (shape.substance is None or shape.substance.name != 'Painted')): if shape.name.name.lower() == 'wall': bbox = bbox_vertices(parse_vertices(shape.vertices)) if ((bbox[0] < 0.05 and bbox[2] < 0.50) or (bbox[0] > 0.50 and bbox[2] > 0.95)): bbox_n = photo.vanishing_point_to_vector( (0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5)) # find normal that best matches this fake bbox normal best_n_dot = 0.9 best_n = None for n in vvectors: if auto_normal_acceptable(n): d = abs_dot(bbox_n, n) if d > best_n_dot: best_n_dot = d best_n = n method = 'O' # find best u vector if not already found if best_n is not None and best_u is None: # first check if any in-shape vanishing points # are perpendicular to the normal best_u = most_orthogonal_vector(best_n, [vvectors[i] for __, i in counts], tolerance_dot=0.05) # else, find the best u from all vectors if best_u is None: best_u = most_orthogonal_vector(best_n, vvectors) # failure if best_u is None or best_n is None: return (None, None, 0) # ortho-normalize system uvn = construct_uvn_frame(best_n, best_u, b, flip_to_match_image=True) # form uvnb matrix, column major uvnb = (uvn[0, 0], uvn[1, 0], uvn[2, 0], 0, uvn[0, 1], uvn[1, 1], uvn[2, 1], 0, uvn[0, 2], uvn[1, 2], uvn[2, 2], 0, b[0], b[1], b[2], 1) return uvnb, method, num_vanishing_lines
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False): """ Return (uvnb, num_vanishing_points) """ print 'estimate_uvnb for shape_id: %s' % shape.id # local import to avoid cyclic dependencies from shapes.utils import parse_vertices, parse_triangles, \ parse_segments, bbox_vertices # load photo photo = shape.photo if not photo.vanishing_lines or not photo.vanishing_points: raise ValueError("Vanishing points not computed") if not photo.focal_y: raise ValueError("Photo does not have focal_y") vlines = json.loads(photo.vanishing_lines) vpoints = json.loads(photo.vanishing_points) vvectors = copy.copy(photo.vanishing_vectors()) if len(vlines) != len(vpoints): raise ValueError("Invalid vanishing points data structure") # add any missing vanishing points vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75) # find vanishing lines inside shape vertices = parse_vertices(shape.vertices) segments = parse_segments(shape.triangles) triangles = parse_triangles(shape.triangles) # intersect shapes with segments counts = [] for idx in xrange(len(vlines)): # re-pack for geom routines query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]] from common.geom import triangles_segments_intersections_only n = len(triangles_segments_intersections_only( vertices, segments, triangles, query_segments)) if n >= 5: counts.append((n, idx)) counts.sort(key=lambda x: x[0], reverse=True) # function to judge normals: its vanishing line can't intersect the shape. def auto_normal_acceptable(n): sign = None for (x, y) in vertices: # vanishing line line = (n[0], n[1], n[2] * photo.focal_y) # signed distance d = ( (x - 0.5) * photo.aspect_ratio * line[0] + (0.5 - y) * line[1] + # flip y line[2] ) if abs(d) < 0.05: return False elif sign is None: sign = (d > 0) else: if sign != (d > 0): return False return True # find coordinate frame best_n = None best_u = None method = None num_vanishing_lines = 0 # make sure shape has label if not shape.label_pos_x or not shape.label_pos_y: from shapes.utils import update_shape_label_pos update_shape_label_pos(shape) # place label in 3D b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1 b = [ (shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) / photo.focal_y, (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y, b_z ] # estimate closest human normal human_labels = list(ShapeRectifiedNormalLabel.objects.filter( shape=shape, automatic=False, correct_score__isnull=False, ).order_by('-correct_score')) human_labels += list(ShapeRectifiedNormalLabel.objects.filter( shape=shape, automatic=False, correct_score__isnull=True)) if human_labels: for label in human_labels: human_u = label.u() human_n = label.n() b = list(label.uvnb_numpy()[0:3, 3].flat) # find best normal best_n_dot = 0.9 best_n = None for n in vvectors: d = abs_dot(human_n, n) if d > best_n_dot: best_n_dot = d best_n = n method = 'S' # if there is a match find u and quit if best_n is not None: # find best u best_u_dot = 0 best_u = None for u in vvectors: if abs_dot(u, best_n) < 0.1: d = abs_dot(human_u, u) if d > best_u_dot: best_u_dot = d best_u = u break # try using object label if best_n is None and shape.name: if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'): best_y = 0.9 for v in vvectors[0:3]: if abs(v[1]) > best_y: best_y = abs(v[1]) best_n = v method = 'O' # try fully automatic method if human normals are not good enough if (try_fully_automatic and best_n is None and len(vpoints) >= 3 and len(counts) >= 2 and (shape.substance is None or shape.substance.name != 'Painted')): # choose two dominant vanishing points best_u = vvectors[counts[0][1]] best_v = vvectors[counts[1][1]] # don't try and auto-rectify frontal surfaces if auto_normal_acceptable(normalized_cross(best_u, best_v)): num_vanishing_lines = counts[0][0] + counts[1][0] uv_dot = abs_dot(best_u, best_v) print 'u dot v = %s' % uv_dot else: best_u, best_v = None, None uv_dot = None # make sure these vectors are accurate if not uv_dot or uv_dot > 0.05: # try and find two other orthogonal vanishing points best_dot = 0.05 best_u = None best_v = None for c1, i1 in counts: for c2, i2 in counts[:i1]: d = abs_dot(vvectors[i1], vvectors[i2]) if d < best_dot: # don't try and auto-rectify frontal surfaces if auto_normal_acceptable(normalized_cross( vvectors[i1], vvectors[i2])): best_dot = d if c1 > c2: best_u = vvectors[i1] best_v = vvectors[i2] else: best_u = vvectors[i2] best_v = vvectors[i1] num_vanishing_lines = c1 + c2 if best_u is not None and best_v is not None: best_n = normalized_cross(best_u, best_v) method = 'A' # give up for some classes of objects if shape.name: name = shape.name.name.lower() if ((abs(best_n[1]) > 0.5 and name in ( 'wall', 'door', 'window')) or (abs(best_n[1]) < 0.5 and name in ( 'floor', 'ceiling', 'table', 'worktop/countertop', 'carpet/rug'))): method = best_u = best_v = best_n = None num_vanishing_lines = 0 # for walls that touch the edge of the photo, try using bbox center as a # vanishing point (i.e. assume top/bottom shapes are horizontal, side # shapes are vertical) if (try_fully_automatic and best_n is None and shape.name and (shape.substance is None or shape.substance.name != 'Painted')): if shape.name.name.lower() == 'wall': bbox = bbox_vertices(parse_vertices(shape.vertices)) if ((bbox[0] < 0.05 and bbox[2] < 0.50) or (bbox[0] > 0.50 and bbox[2] > 0.95)): bbox_n = photo.vanishing_point_to_vector(( 0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5 )) # find normal that best matches this fake bbox normal best_n_dot = 0.9 best_n = None for n in vvectors: if auto_normal_acceptable(n): d = abs_dot(bbox_n, n) if d > best_n_dot: best_n_dot = d best_n = n method = 'O' # find best u vector if not already found if best_n is not None and best_u is None: # first check if any in-shape vanishing points # are perpendicular to the normal best_u = most_orthogonal_vector( best_n, [vvectors[i] for __, i in counts], tolerance_dot=0.05) # else, find the best u from all vectors if best_u is None: best_u = most_orthogonal_vector(best_n, vvectors) # failure if best_u is None or best_n is None: return (None, None, 0) # ortho-normalize system uvn = construct_uvn_frame( best_n, best_u, b, flip_to_match_image=True) # form uvnb matrix, column major uvnb = ( uvn[0, 0], uvn[1, 0], uvn[2, 0], 0, uvn[0, 1], uvn[1, 1], uvn[2, 1], 0, uvn[0, 2], uvn[1, 2], uvn[2, 2], 0, b[0], b[1], b[2], 1 ) return uvnb, method, num_vanishing_lines
def detect_vanishing_points_impl(photo, image, save=True): # algorithm parameters max_em_iter = 0 # if 0, don't do EM min_cluster_size = 10 min_line_len2 = 4.0 residual_stdev = 0.75 max_clusters = 8 outlier_weight = 0.2 weight_clamp = 0.1 lambda_perp = 1.0 verbose = False width, height = image.size print 'size: %s x %s' % (width, height) vpdetection_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'opt', 'vpdetection', 'matlab')) tmpdir = tempfile.mkdtemp() try: # save image to local tmpdir localname = os.path.join(tmpdir, 'image.jpg') with open(tmpdir + '/image.jpg', 'wb') as target: save_image(image, target, format='JPEG', options={'quality': 90}) # detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel, # J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a # False Detection Control. IEEE Transactions on Pattern Analysis and # Machine Intelligence, 32, 722.) linesname = os.path.join(tmpdir, 'lines.txt') matlab_command = ";".join([ "try", "addpath('../lsd-1.5/')", "lines = lsd(double(rgb2gray(imread('%s'))))" % localname, "save('%s', 'lines', '-ascii', '-tabs')" % linesname, "catch", "end", "quit", ]) print 'matlab command: %s' % matlab_command subprocess.check_call(args=[ 'matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r', matlab_command ], cwd=vpdetection_dir) # cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008). # Robust multiple structures estimation with J-Linkage. European # Conference on Computer Vision(ECCV), 2008.) # and (Tardif J.-P., Non-iterative Approach for Fast and Accurate # Vanishing Point Detection, 12th IEEE International Conference on # Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.) clustername = os.path.join(tmpdir, 'clusters.txt') subprocess.check_call(args=['./vpdetection', linesname, clustername], cwd=vpdetection_dir) # collect line clusters clusters_dict = {} all_lines = [] for row in open(clustername, 'r').readlines(): cols = row.split() idx = int(cols[4]) line = [float(f) for f in cols[0:4]] # discard small lines x1, y1, x2, y2 = line len2 = (x1 - x2)**2 + (y2 - y1)**2 if len2 < min_line_len2: continue if idx in clusters_dict: clusters_dict[idx].append(line) all_lines.append(line) else: clusters_dict[idx] = [line] finally: shutil.rmtree(tmpdir) # discard invalid clusters and sort by cluster length thresh = 3 if max_em_iter else min_cluster_size clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values()) clusters.sort(key=line_cluster_length, reverse=True) if max_em_iter and len(clusters) > max_clusters: clusters = clusters[:max_clusters] print "Using %s clusters and %s lines" % (len(clusters), len(all_lines)) if not clusters: print "Not enough clusters" return # Solve for optimal vanishing point using V_GS in 5.2 section of # (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf). # where "optimal" minimizes algebraic error. vectors = [] for lines in clusters: # Minimize 'algebraic' error to get an initial solution A = np.zeros((len(lines), 3)) for i in xrange(0, len(lines)): x1, y1, x2, y2 = lines[i] A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2] __, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True) if VT.shape != (3, 3): raise ValueError("Invalid SVD shape (%s)" % VT.size) x, y, w = VT[2, :] p = [x / w, y / w] v = photo.vanishing_point_to_vector((p[0] / width, p[1] / height)) vectors.append(v) # EM if max_em_iter: # complete orthonormal system if len(vectors) >= 2: vectors.append(normalized_cross(vectors[0], vectors[1])) ### EM refinement ### x0 = None x_opt = None exp_coeff = 0.5 / (residual_stdev**2) num_weights_nnz = 0 num_weights = 0 for em_iter in xrange(max_em_iter): ### E STEP ### # convert back to vanishing points points = vectors_to_points(photo, image, vectors) # last column is the outlier cluster weights = np.zeros((len(all_lines), len(vectors) + 1)) # estimate weights (assume uniform prior) for i_p, p in enumerate(points): weights[:, i_p] = [line_residual(l, p) for l in all_lines] weights = np.exp(-exp_coeff * np.square(weights)) # outlier weight weights[:, len(points)] = outlier_weight # normalize each row (each line segment) to have unit sum weights_row_sum = weights.sum(axis=1) weights /= weights_row_sum[:, np.newaxis] # add sparsity weights[weights < weight_clamp] = 0 num_weights += weights.size num_weights_nnz += np.count_nonzero(weights) # check convergence if (em_iter >= 10 and len(x0) == len(x_opt) and np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5): break # sort by weight if len(vectors) > 1: vectors_weights = [(v, weights[:, i_v].sum()) for i_v, v in enumerate(vectors)] vectors_weights.sort(key=lambda x: x[1], reverse=True) vectors = [x[0] for x in vectors_weights] ### M STEP ### # objective function to minimize def objective_function(x, *args): cur_vectors = unpack_x(x) cur_points = vectors_to_points(photo, image, cur_vectors) # line-segment errors residuals = [ weights[i_l, i_p] * line_residual(all_lines[i_l], p) for i_p, p in enumerate(cur_points) for i_l in np.flatnonzero(weights[:, i_p]) ] # penalize deviations from 45 or 90 degree angles if lambda_perp: residuals += [ lambda_perp * math.sin(4 * math.acos(abs_dot(v, w))) for i_v, v in enumerate(cur_vectors) for w in cur_vectors[:i_v] ] return residuals # slowly vary parameters t = min(1.0, em_iter / 20.0) # vary tol from 1e-2 to 1e-6 tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t) from scipy.optimize import leastsq x0 = pack_x(vectors) x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol) vectors = unpack_x(x_opt) ### BETWEEN ITERATIONS ### if verbose: print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % ( em_iter, len(vectors), 100.0 * num_weights_nnz / num_weights) print 'residual: %s' % sum(y**2 for y in objective_function(x_opt)) # complete orthonormal system if missing if len(vectors) == 2: vectors.append(normalized_cross(vectors[0], vectors[1])) # merge similar clusters cluster_merge_dot = math.cos(math.radians(t * 20.0)) vectors_merged = [] for v in vectors: if (not vectors_merged or all( abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)): vectors_merged.append(v) if verbose and len(vectors) != len(vectors_merged): print 'Merging %s --> %s vectors' % (len(vectors), len(vectors_merged)) vectors = vectors_merged residual = sum(r**2 for r in objective_function(x_opt)) print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % ( em_iter, residual, len(vectors), 100.0 * num_weights_nnz / num_weights) # final points points = vectors_to_points(photo, image, vectors) # sanity checks assert len(vectors) == len(points) # re-assign clusters clusters_points = [([], p) for p in points] line_map_cluster = np.argmax(weights, axis=1) for i_l, l in enumerate(all_lines): i_c = line_map_cluster[i_l] if i_c < len(points): clusters_points[i_c][0].append(l) # throw away small clusters clusters_points = filter(lambda x: len(x[0]) >= min_cluster_size, clusters_points) # reverse sort by cluster length clusters_points.sort(key=lambda x: line_cluster_length(x[0]), reverse=True) # split into two parallel arrays clusters = [cp[0] for cp in clusters_points] points = [cp[1] for cp in clusters_points] else: # no EM for i_v, lines in enumerate(clusters): def objective_function(x, *args): p = vectors_to_points(photo, image, unpack_x(x))[0] return [line_residual(l, p) for l in lines] from scipy.optimize import leastsq x0 = pack_x([vectors[i_v]]) x_opt, __ = leastsq(objective_function, x0) vectors[i_v] = unpack_x(x_opt)[0] # delete similar vectors cluster_merge_dot = math.cos(math.radians(20.0)) vectors_merged = [] clusters_merged = [] for i_v, v in enumerate(vectors): if (not vectors_merged or all( abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)): vectors_merged.append(v) clusters_merged.append(clusters[i_v]) vectors = vectors_merged clusters = clusters_merged # clamp number of vectors if len(clusters) > max_clusters: vectors = vectors[:max_clusters] clusters = clusters[:max_clusters] points = vectors_to_points(photo, image, vectors) # normalize to [0, 0], [1, 1] clusters_normalized = [[[ l[0] / width, l[1] / height, l[2] / width, l[3] / height ] for l in lines] for lines in clusters] points_normalized = [(x / width, y / height) for (x, y) in points] # save result photo.vanishing_lines = json.dumps(clusters_normalized) photo.vanishing_points = json.dumps(points_normalized) photo.vanishing_length = sum( line_cluster_length(c) for c in clusters_normalized) if save: photo.save()
def detect_vanishing_points_impl(photo, image, save=True): # algorithm parameters max_em_iter = 0 # if 0, don't do EM min_cluster_size = 10 min_line_len2 = 4.0 residual_stdev = 0.75 max_clusters = 8 outlier_weight = 0.2 weight_clamp = 0.1 lambda_perp = 1.0 verbose = False width, height = image.size print 'size: %s x %s' % (width, height) vpdetection_dir = os.path.abspath(os.path.join( os.path.dirname(__file__), os.pardir, os.pardir, 'opt', 'vpdetection', 'matlab' )) tmpdir = tempfile.mkdtemp() try: # save image to local tmpdir localname = os.path.join(tmpdir, 'image.jpg') with open(tmpdir + '/image.jpg', 'wb') as target: save_image(image, target, format='JPEG', options={'quality': 90}) # detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel, # J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a # False Detection Control. IEEE Transactions on Pattern Analysis and # Machine Intelligence, 32, 722.) linesname = os.path.join(tmpdir, 'lines.txt') matlab_command = ";".join([ "try", "addpath('../lsd-1.5/')", "lines = lsd(double(rgb2gray(imread('%s'))))" % localname, "save('%s', 'lines', '-ascii', '-tabs')" % linesname, "catch", "end", "quit", ]) print 'matlab command: %s' % matlab_command subprocess.check_call(args=[ 'matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r', matlab_command ], cwd=vpdetection_dir) # cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008). # Robust multiple structures estimation with J-Linkage. European # Conference on Computer Vision(ECCV), 2008.) # and (Tardif J.-P., Non-iterative Approach for Fast and Accurate # Vanishing Point Detection, 12th IEEE International Conference on # Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.) clustername = os.path.join(tmpdir, 'clusters.txt') subprocess.check_call( args=['./vpdetection', linesname, clustername], cwd=vpdetection_dir) # collect line clusters clusters_dict = {} all_lines = [] for row in open(clustername, 'r').readlines(): cols = row.split() idx = int(cols[4]) line = [float(f) for f in cols[0:4]] # discard small lines x1, y1, x2, y2 = line len2 = (x1 - x2) ** 2 + (y2 - y1) ** 2 if len2 < min_line_len2: continue if idx in clusters_dict: clusters_dict[idx].append(line) all_lines.append(line) else: clusters_dict[idx] = [line] finally: shutil.rmtree(tmpdir) # discard invalid clusters and sort by cluster length thresh = 3 if max_em_iter else min_cluster_size clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values()) clusters.sort(key=line_cluster_length, reverse=True) if max_em_iter and len(clusters) > max_clusters: clusters = clusters[:max_clusters] print "Using %s clusters and %s lines" % (len(clusters), len(all_lines)) if not clusters: print "Not enough clusters" return # Solve for optimal vanishing point using V_GS in 5.2 section of # (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf). # where "optimal" minimizes algebraic error. vectors = [] for lines in clusters: # Minimize 'algebraic' error to get an initial solution A = np.zeros((len(lines), 3)) for i in xrange(0, len(lines)): x1, y1, x2, y2 = lines[i] A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2] __, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True) if VT.shape != (3, 3): raise ValueError("Invalid SVD shape (%s)" % VT.size) x, y, w = VT[2, :] p = [x / w, y / w] v = photo.vanishing_point_to_vector( (p[0] / width, p[1] / height) ) vectors.append(v) # EM if max_em_iter: # complete orthonormal system if len(vectors) >= 2: vectors.append(normalized_cross(vectors[0], vectors[1])) ### EM refinement ### x0 = None x_opt = None exp_coeff = 0.5 / (residual_stdev ** 2) num_weights_nnz = 0 num_weights = 0 for em_iter in xrange(max_em_iter): ### E STEP ### # convert back to vanishing points points = vectors_to_points(photo, image, vectors) # last column is the outlier cluster weights = np.zeros((len(all_lines), len(vectors) + 1)) # estimate weights (assume uniform prior) for i_p, p in enumerate(points): weights[:, i_p] = [line_residual(l, p) for l in all_lines] weights = np.exp(-exp_coeff * np.square(weights)) # outlier weight weights[:, len(points)] = outlier_weight # normalize each row (each line segment) to have unit sum weights_row_sum = weights.sum(axis=1) weights /= weights_row_sum[:, np.newaxis] # add sparsity weights[weights < weight_clamp] = 0 num_weights += weights.size num_weights_nnz += np.count_nonzero(weights) # check convergence if (em_iter >= 10 and len(x0) == len(x_opt) and np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5): break # sort by weight if len(vectors) > 1: vectors_weights = [ (v, weights[:, i_v].sum()) for i_v, v in enumerate(vectors) ] vectors_weights.sort(key=lambda x: x[1], reverse=True) vectors = [x[0] for x in vectors_weights] ### M STEP ### # objective function to minimize def objective_function(x, *args): cur_vectors = unpack_x(x) cur_points = vectors_to_points(photo, image, cur_vectors) # line-segment errors residuals = [ weights[i_l, i_p] * line_residual(all_lines[i_l], p) for i_p, p in enumerate(cur_points) for i_l in np.flatnonzero(weights[:, i_p]) ] # penalize deviations from 45 or 90 degree angles if lambda_perp: residuals += [ lambda_perp * math.sin(4 * math.acos(abs_dot(v, w))) for i_v, v in enumerate(cur_vectors) for w in cur_vectors[:i_v] ] return residuals # slowly vary parameters t = min(1.0, em_iter / 20.0) # vary tol from 1e-2 to 1e-6 tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t) from scipy.optimize import leastsq x0 = pack_x(vectors) x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol) vectors = unpack_x(x_opt) ### BETWEEN ITERATIONS ### if verbose: print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % ( em_iter, len(vectors), 100.0 * num_weights_nnz / num_weights) print 'residual: %s' % sum(y ** 2 for y in objective_function(x_opt)) # complete orthonormal system if missing if len(vectors) == 2: vectors.append(normalized_cross(vectors[0], vectors[1])) # merge similar clusters cluster_merge_dot = math.cos(math.radians(t * 20.0)) vectors_merged = [] for v in vectors: if (not vectors_merged or all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)): vectors_merged.append(v) if verbose and len(vectors) != len(vectors_merged): print 'Merging %s --> %s vectors' % (len(vectors), len(vectors_merged)) vectors = vectors_merged residual = sum(r ** 2 for r in objective_function(x_opt)) print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % ( em_iter, residual, len(vectors), 100.0 * num_weights_nnz / num_weights) # final points points = vectors_to_points(photo, image, vectors) # sanity checks assert len(vectors) == len(points) # re-assign clusters clusters_points = [([], p) for p in points] line_map_cluster = np.argmax(weights, axis=1) for i_l, l in enumerate(all_lines): i_c = line_map_cluster[i_l] if i_c < len(points): clusters_points[i_c][0].append(l) # throw away small clusters clusters_points = filter( lambda x: len(x[0]) >= min_cluster_size, clusters_points) # reverse sort by cluster length clusters_points.sort( key=lambda x: line_cluster_length(x[0]), reverse=True) # split into two parallel arrays clusters = [cp[0] for cp in clusters_points] points = [cp[1] for cp in clusters_points] else: # no EM for i_v, lines in enumerate(clusters): def objective_function(x, *args): p = vectors_to_points(photo, image, unpack_x(x))[0] return [line_residual(l, p) for l in lines] from scipy.optimize import leastsq x0 = pack_x([vectors[i_v]]) x_opt, __ = leastsq(objective_function, x0) vectors[i_v] = unpack_x(x_opt)[0] # delete similar vectors cluster_merge_dot = math.cos(math.radians(20.0)) vectors_merged = [] clusters_merged = [] for i_v, v in enumerate(vectors): if (not vectors_merged or all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)): vectors_merged.append(v) clusters_merged.append(clusters[i_v]) vectors = vectors_merged clusters = clusters_merged # clamp number of vectors if len(clusters) > max_clusters: vectors = vectors[:max_clusters] clusters = clusters[:max_clusters] points = vectors_to_points(photo, image, vectors) # normalize to [0, 0], [1, 1] clusters_normalized = [[ [l[0] / width, l[1] / height, l[2] / width, l[3] / height] for l in lines ] for lines in clusters] points_normalized = [ (x / width, y / height) for (x, y) in points ] # save result photo.vanishing_lines = json.dumps(clusters_normalized) photo.vanishing_points = json.dumps(points_normalized) photo.vanishing_length = sum(line_cluster_length(c) for c in clusters_normalized) if save: photo.save()