Exemplo n.º 1
0
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
    """ Return (uvnb, num_vanishing_points) """

    print 'estimate_uvnb for shape_id: %s' % shape.id

    # local import to avoid cyclic dependencies
    from shapes.utils import parse_vertices, parse_triangles, \
        parse_segments, bbox_vertices

    # load photo
    photo = shape.photo

    if not photo.vanishing_lines or not photo.vanishing_points:
        raise ValueError("Vanishing points not computed")

    if not photo.focal_y:
        raise ValueError("Photo does not have focal_y")

    vlines = json.loads(photo.vanishing_lines)
    vpoints = json.loads(photo.vanishing_points)
    vvectors = copy.copy(photo.vanishing_vectors())

    if len(vlines) != len(vpoints):
        raise ValueError("Invalid vanishing points data structure")

    # add any missing vanishing points
    vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)

    # find vanishing lines inside shape
    vertices = parse_vertices(shape.vertices)
    segments = parse_segments(shape.triangles)
    triangles = parse_triangles(shape.triangles)

    # intersect shapes with segments
    counts = []
    for idx in xrange(len(vlines)):
        # re-pack for geom routines
        query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]

        from common.geom import triangles_segments_intersections_only
        n = len(
            triangles_segments_intersections_only(vertices, segments,
                                                  triangles, query_segments))
        if n >= 5:
            counts.append((n, idx))
    counts.sort(key=lambda x: x[0], reverse=True)

    # function to judge normals: its vanishing line can't intersect the shape.
    def auto_normal_acceptable(n):
        sign = None
        for (x, y) in vertices:
            # vanishing line
            line = (n[0], n[1], n[2] * photo.focal_y)
            # signed distance
            d = ((x - 0.5) * photo.aspect_ratio * line[0] +
                 (0.5 - y) * line[1] +  # flip y
                 line[2])
            if abs(d) < 0.05:
                return False
            elif sign is None:
                sign = (d > 0)
            else:
                if sign != (d > 0):
                    return False
        return True

    # find coordinate frame
    best_n = None
    best_u = None
    method = None
    num_vanishing_lines = 0

    # make sure shape has label
    if not shape.label_pos_x or not shape.label_pos_y:
        from shapes.utils import update_shape_label_pos
        update_shape_label_pos(shape)

    # place label in 3D
    b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
    b = [(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) /
         photo.focal_y, (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
         b_z]

    # estimate closest human normal
    human_labels = list(
        ShapeRectifiedNormalLabel.objects.filter(
            shape=shape,
            automatic=False,
            correct_score__isnull=False,
        ).order_by('-correct_score'))
    human_labels += list(
        ShapeRectifiedNormalLabel.objects.filter(shape=shape,
                                                 automatic=False,
                                                 correct_score__isnull=True))
    if human_labels:
        for label in human_labels:
            human_u = label.u()
            human_n = label.n()
            b = list(label.uvnb_numpy()[0:3, 3].flat)

            # find best normal
            best_n_dot = 0.9
            best_n = None
            for n in vvectors:
                d = abs_dot(human_n, n)
                if d > best_n_dot:
                    best_n_dot = d
                    best_n = n
                    method = 'S'

            # if there is a match find u and quit
            if best_n is not None:
                # find best u
                best_u_dot = 0
                best_u = None
                for u in vvectors:
                    if abs_dot(u, best_n) < 0.1:
                        d = abs_dot(human_u, u)
                        if d > best_u_dot:
                            best_u_dot = d
                            best_u = u
                break

    # try using object label
    if best_n is None and shape.name:
        if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
            best_y = 0.9
            for v in vvectors[0:3]:
                if abs(v[1]) > best_y:
                    best_y = abs(v[1])
                    best_n = v
                    method = 'O'

    # try fully automatic method if human normals are not good enough
    if (try_fully_automatic and best_n is None and len(vpoints) >= 3
            and len(counts) >= 2 and
        (shape.substance is None or shape.substance.name != 'Painted')):

        # choose two dominant vanishing points
        best_u = vvectors[counts[0][1]]
        best_v = vvectors[counts[1][1]]

        # don't try and auto-rectify frontal surfaces
        if auto_normal_acceptable(normalized_cross(best_u, best_v)):
            num_vanishing_lines = counts[0][0] + counts[1][0]
            uv_dot = abs_dot(best_u, best_v)
            print 'u dot v = %s' % uv_dot
        else:
            best_u, best_v = None, None
            uv_dot = None

        # make sure these vectors are accurate
        if not uv_dot or uv_dot > 0.05:
            # try and find two other orthogonal vanishing points
            best_dot = 0.05
            best_u = None
            best_v = None
            for c1, i1 in counts:
                for c2, i2 in counts[:i1]:
                    d = abs_dot(vvectors[i1], vvectors[i2])
                    if d < best_dot:
                        # don't try and auto-rectify frontal surfaces
                        if auto_normal_acceptable(
                                normalized_cross(vvectors[i1], vvectors[i2])):
                            best_dot = d
                            if c1 > c2:
                                best_u = vvectors[i1]
                                best_v = vvectors[i2]
                            else:
                                best_u = vvectors[i2]
                                best_v = vvectors[i1]
                            num_vanishing_lines = c1 + c2

        if best_u is not None and best_v is not None:
            best_n = normalized_cross(best_u, best_v)
            method = 'A'

            # give up for some classes of objects
            if shape.name:
                name = shape.name.name.lower()
                if ((abs(best_n[1]) > 0.5
                     and name in ('wall', 'door', 'window'))
                        or (abs(best_n[1]) < 0.5
                            and name in ('floor', 'ceiling', 'table',
                                         'worktop/countertop', 'carpet/rug'))):
                    method = best_u = best_v = best_n = None
                    num_vanishing_lines = 0

    # for walls that touch the edge of the photo, try using bbox center as a
    # vanishing point (i.e. assume top/bottom shapes are horizontal, side
    # shapes are vertical)
    if (try_fully_automatic and best_n is None and shape.name and
        (shape.substance is None or shape.substance.name != 'Painted')):
        if shape.name.name.lower() == 'wall':
            bbox = bbox_vertices(parse_vertices(shape.vertices))
            if ((bbox[0] < 0.05 and bbox[2] < 0.50)
                    or (bbox[0] > 0.50 and bbox[2] > 0.95)):

                bbox_n = photo.vanishing_point_to_vector(
                    (0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5))

                # find normal that best matches this fake bbox normal
                best_n_dot = 0.9
                best_n = None
                for n in vvectors:
                    if auto_normal_acceptable(n):
                        d = abs_dot(bbox_n, n)
                        if d > best_n_dot:
                            best_n_dot = d
                            best_n = n
                            method = 'O'

    # find best u vector if not already found
    if best_n is not None and best_u is None:
        # first check if any in-shape vanishing points
        # are perpendicular to the normal
        best_u = most_orthogonal_vector(best_n,
                                        [vvectors[i] for __, i in counts],
                                        tolerance_dot=0.05)

        # else, find the best u from all vectors
        if best_u is None:
            best_u = most_orthogonal_vector(best_n, vvectors)

    # failure
    if best_u is None or best_n is None:
        return (None, None, 0)

    # ortho-normalize system
    uvn = construct_uvn_frame(best_n, best_u, b, flip_to_match_image=True)

    # form uvnb matrix, column major
    uvnb = (uvn[0, 0], uvn[1, 0], uvn[2, 0], 0, uvn[0, 1], uvn[1, 1],
            uvn[2, 1], 0, uvn[0, 2], uvn[1, 2], uvn[2,
                                                    2], 0, b[0], b[1], b[2], 1)

    return uvnb, method, num_vanishing_lines
Exemplo n.º 2
0
def detect_vanishing_points_impl(photo, image, save=True):

    # algorithm parameters
    max_em_iter = 0  # if 0, don't do EM
    min_cluster_size = 10
    min_line_len2 = 4.0
    residual_stdev = 0.75
    max_clusters = 8
    outlier_weight = 0.2
    weight_clamp = 0.1
    lambda_perp = 1.0
    verbose = False

    width, height = image.size
    print 'size: %s x %s' % (width, height)

    vpdetection_dir = os.path.abspath(
        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'opt',
                     'vpdetection', 'matlab'))

    tmpdir = tempfile.mkdtemp()
    try:

        # save image to local tmpdir
        localname = os.path.join(tmpdir, 'image.jpg')
        with open(tmpdir + '/image.jpg', 'wb') as target:
            save_image(image, target, format='JPEG', options={'quality': 90})

        # detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel,
        # J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a
        # False Detection Control. IEEE Transactions on Pattern Analysis and
        # Machine Intelligence, 32, 722.)
        linesname = os.path.join(tmpdir, 'lines.txt')
        matlab_command = ";".join([
            "try",
            "addpath('../lsd-1.5/')",
            "lines = lsd(double(rgb2gray(imread('%s'))))" % localname,
            "save('%s', 'lines', '-ascii', '-tabs')" % linesname,
            "catch",
            "end",
            "quit",
        ])
        print 'matlab command: %s' % matlab_command
        subprocess.check_call(args=[
            'matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
            matlab_command
        ],
                              cwd=vpdetection_dir)

        # cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008).
        # Robust multiple structures estimation with J-Linkage. European
        # Conference on Computer Vision(ECCV), 2008.)
        # and (Tardif J.-P., Non-iterative Approach for Fast and Accurate
        # Vanishing Point Detection, 12th IEEE International Conference on
        # Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.)
        clustername = os.path.join(tmpdir, 'clusters.txt')
        subprocess.check_call(args=['./vpdetection', linesname, clustername],
                              cwd=vpdetection_dir)

        # collect line clusters
        clusters_dict = {}
        all_lines = []
        for row in open(clustername, 'r').readlines():
            cols = row.split()
            idx = int(cols[4])
            line = [float(f) for f in cols[0:4]]

            # discard small lines
            x1, y1, x2, y2 = line
            len2 = (x1 - x2)**2 + (y2 - y1)**2
            if len2 < min_line_len2:
                continue

            if idx in clusters_dict:
                clusters_dict[idx].append(line)
                all_lines.append(line)
            else:
                clusters_dict[idx] = [line]

    finally:
        shutil.rmtree(tmpdir)

    # discard invalid clusters and sort by cluster length
    thresh = 3 if max_em_iter else min_cluster_size
    clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values())
    clusters.sort(key=line_cluster_length, reverse=True)
    if max_em_iter and len(clusters) > max_clusters:
        clusters = clusters[:max_clusters]
    print "Using %s clusters and %s lines" % (len(clusters), len(all_lines))
    if not clusters:
        print "Not enough clusters"
        return

    # Solve for optimal vanishing point using V_GS in 5.2 section of
    # (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf).
    # where "optimal" minimizes algebraic error.
    vectors = []
    for lines in clusters:
        # Minimize 'algebraic' error to get an initial solution
        A = np.zeros((len(lines), 3))
        for i in xrange(0, len(lines)):
            x1, y1, x2, y2 = lines[i]
            A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2]
        __, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True)
        if VT.shape != (3, 3):
            raise ValueError("Invalid SVD shape (%s)" % VT.size)
        x, y, w = VT[2, :]
        p = [x / w, y / w]
        v = photo.vanishing_point_to_vector((p[0] / width, p[1] / height))
        vectors.append(v)

    # EM
    if max_em_iter:

        # complete orthonormal system
        if len(vectors) >= 2:
            vectors.append(normalized_cross(vectors[0], vectors[1]))

        ### EM refinement ###

        x0 = None
        x_opt = None
        exp_coeff = 0.5 / (residual_stdev**2)

        num_weights_nnz = 0
        num_weights = 0

        for em_iter in xrange(max_em_iter):

            ### E STEP ###

            # convert back to vanishing points
            points = vectors_to_points(photo, image, vectors)

            # last column is the outlier cluster
            weights = np.zeros((len(all_lines), len(vectors) + 1))

            # estimate weights (assume uniform prior)
            for i_p, p in enumerate(points):
                weights[:, i_p] = [line_residual(l, p) for l in all_lines]
            weights = np.exp(-exp_coeff * np.square(weights))

            # outlier weight
            weights[:, len(points)] = outlier_weight

            # normalize each row (each line segment) to have unit sum
            weights_row_sum = weights.sum(axis=1)
            weights /= weights_row_sum[:, np.newaxis]

            # add sparsity
            weights[weights < weight_clamp] = 0
            num_weights += weights.size
            num_weights_nnz += np.count_nonzero(weights)

            # check convergence
            if (em_iter >= 10 and len(x0) == len(x_opt) and
                    np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5):
                break

            # sort by weight
            if len(vectors) > 1:
                vectors_weights = [(v, weights[:, i_v].sum())
                                   for i_v, v in enumerate(vectors)]
                vectors_weights.sort(key=lambda x: x[1], reverse=True)
                vectors = [x[0] for x in vectors_weights]

            ### M STEP ###

            # objective function to minimize
            def objective_function(x, *args):
                cur_vectors = unpack_x(x)
                cur_points = vectors_to_points(photo, image, cur_vectors)

                # line-segment errors
                residuals = [
                    weights[i_l, i_p] * line_residual(all_lines[i_l], p)
                    for i_p, p in enumerate(cur_points)
                    for i_l in np.flatnonzero(weights[:, i_p])
                ]

                # penalize deviations from 45 or 90 degree angles
                if lambda_perp:
                    residuals += [
                        lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
                        for i_v, v in enumerate(cur_vectors)
                        for w in cur_vectors[:i_v]
                    ]

                return residuals

            # slowly vary parameters
            t = min(1.0, em_iter / 20.0)

            # vary tol from 1e-2 to 1e-6
            tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t)

            from scipy.optimize import leastsq
            x0 = pack_x(vectors)
            x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol)
            vectors = unpack_x(x_opt)

            ### BETWEEN ITERATIONS ###

            if verbose:
                print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % (
                    em_iter, len(vectors),
                    100.0 * num_weights_nnz / num_weights)
                print 'residual: %s' % sum(y**2
                                           for y in objective_function(x_opt))

            # complete orthonormal system if missing
            if len(vectors) == 2:
                vectors.append(normalized_cross(vectors[0], vectors[1]))

            # merge similar clusters
            cluster_merge_dot = math.cos(math.radians(t * 20.0))
            vectors_merged = []
            for v in vectors:
                if (not vectors_merged or all(
                        abs_dot(v, w) < cluster_merge_dot
                        for w in vectors_merged)):
                    vectors_merged.append(v)
            if verbose and len(vectors) != len(vectors_merged):
                print 'Merging %s --> %s vectors' % (len(vectors),
                                                     len(vectors_merged))
            vectors = vectors_merged

        residual = sum(r**2 for r in objective_function(x_opt))
        print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % (
            em_iter, residual, len(vectors),
            100.0 * num_weights_nnz / num_weights)

        # final points
        points = vectors_to_points(photo, image, vectors)

        # sanity checks
        assert len(vectors) == len(points)

        # re-assign clusters
        clusters_points = [([], p) for p in points]
        line_map_cluster = np.argmax(weights, axis=1)
        for i_l, l in enumerate(all_lines):
            i_c = line_map_cluster[i_l]
            if i_c < len(points):
                clusters_points[i_c][0].append(l)

        # throw away small clusters
        clusters_points = filter(lambda x: len(x[0]) >= min_cluster_size,
                                 clusters_points)

        # reverse sort by cluster length
        clusters_points.sort(key=lambda x: line_cluster_length(x[0]),
                             reverse=True)

        # split into two parallel arrays
        clusters = [cp[0] for cp in clusters_points]
        points = [cp[1] for cp in clusters_points]

    else:  # no EM

        for i_v, lines in enumerate(clusters):

            def objective_function(x, *args):
                p = vectors_to_points(photo, image, unpack_x(x))[0]
                return [line_residual(l, p) for l in lines]

            from scipy.optimize import leastsq
            x0 = pack_x([vectors[i_v]])
            x_opt, __ = leastsq(objective_function, x0)
            vectors[i_v] = unpack_x(x_opt)[0]

        # delete similar vectors
        cluster_merge_dot = math.cos(math.radians(20.0))
        vectors_merged = []
        clusters_merged = []
        for i_v, v in enumerate(vectors):
            if (not vectors_merged or all(
                    abs_dot(v, w) < cluster_merge_dot
                    for w in vectors_merged)):
                vectors_merged.append(v)
                clusters_merged.append(clusters[i_v])
        vectors = vectors_merged
        clusters = clusters_merged

        # clamp number of vectors
        if len(clusters) > max_clusters:
            vectors = vectors[:max_clusters]
            clusters = clusters[:max_clusters]

        points = vectors_to_points(photo, image, vectors)

    # normalize to [0, 0], [1, 1]
    clusters_normalized = [[[
        l[0] / width, l[1] / height, l[2] / width, l[3] / height
    ] for l in lines] for lines in clusters]

    points_normalized = [(x / width, y / height) for (x, y) in points]

    # save result
    photo.vanishing_lines = json.dumps(clusters_normalized)
    photo.vanishing_points = json.dumps(points_normalized)
    photo.vanishing_length = sum(
        line_cluster_length(c) for c in clusters_normalized)
    if save:
        photo.save()
Exemplo n.º 3
0
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
    """ Return (uvnb, num_vanishing_points) """

    print 'estimate_uvnb for shape_id: %s' % shape.id

    # local import to avoid cyclic dependencies
    from shapes.utils import parse_vertices, parse_triangles, \
        parse_segments, bbox_vertices

    # load photo
    photo = shape.photo

    if not photo.vanishing_lines or not photo.vanishing_points:
        raise ValueError("Vanishing points not computed")

    if not photo.focal_y:
        raise ValueError("Photo does not have focal_y")

    vlines = json.loads(photo.vanishing_lines)
    vpoints = json.loads(photo.vanishing_points)
    vvectors = copy.copy(photo.vanishing_vectors())

    if len(vlines) != len(vpoints):
        raise ValueError("Invalid vanishing points data structure")

    # add any missing vanishing points
    vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)

    # find vanishing lines inside shape
    vertices = parse_vertices(shape.vertices)
    segments = parse_segments(shape.triangles)
    triangles = parse_triangles(shape.triangles)

    # intersect shapes with segments
    counts = []
    for idx in xrange(len(vlines)):
        # re-pack for geom routines
        query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]

        from common.geom import triangles_segments_intersections_only
        n = len(triangles_segments_intersections_only(
            vertices, segments, triangles, query_segments))
        if n >= 5:
            counts.append((n, idx))
    counts.sort(key=lambda x: x[0], reverse=True)

    # function to judge normals: its vanishing line can't intersect the shape.
    def auto_normal_acceptable(n):
        sign = None
        for (x, y) in vertices:
            # vanishing line
            line = (n[0], n[1], n[2] * photo.focal_y)
            # signed distance
            d = (
                (x - 0.5) * photo.aspect_ratio * line[0] +
                (0.5 - y) * line[1] +  # flip y
                line[2]
            )
            if abs(d) < 0.05:
                return False
            elif sign is None:
                sign = (d > 0)
            else:
                if sign != (d > 0):
                    return False
        return True

    # find coordinate frame
    best_n = None
    best_u = None
    method = None
    num_vanishing_lines = 0

    # make sure shape has label
    if not shape.label_pos_x or not shape.label_pos_y:
        from shapes.utils import update_shape_label_pos
        update_shape_label_pos(shape)

    # place label in 3D
    b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
    b = [
        (shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) / photo.focal_y,
        (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
        b_z
    ]

    # estimate closest human normal
    human_labels = list(ShapeRectifiedNormalLabel.objects.filter(
        shape=shape, automatic=False, correct_score__isnull=False,
    ).order_by('-correct_score'))
    human_labels += list(ShapeRectifiedNormalLabel.objects.filter(
        shape=shape, automatic=False, correct_score__isnull=True))
    if human_labels:
        for label in human_labels:
            human_u = label.u()
            human_n = label.n()
            b = list(label.uvnb_numpy()[0:3, 3].flat)

            # find best normal
            best_n_dot = 0.9
            best_n = None
            for n in vvectors:
                d = abs_dot(human_n, n)
                if d > best_n_dot:
                    best_n_dot = d
                    best_n = n
                    method = 'S'

            # if there is a match find u and quit
            if best_n is not None:
                # find best u
                best_u_dot = 0
                best_u = None
                for u in vvectors:
                    if abs_dot(u, best_n) < 0.1:
                        d = abs_dot(human_u, u)
                        if d > best_u_dot:
                            best_u_dot = d
                            best_u = u
                break

    # try using object label
    if best_n is None and shape.name:
        if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
            best_y = 0.9
            for v in vvectors[0:3]:
                if abs(v[1]) > best_y:
                    best_y = abs(v[1])
                    best_n = v
                    method = 'O'

    # try fully automatic method if human normals are not good enough
    if (try_fully_automatic and best_n is None and len(vpoints) >= 3 and len(counts) >= 2 and
            (shape.substance is None or shape.substance.name != 'Painted')):

        # choose two dominant vanishing points
        best_u = vvectors[counts[0][1]]
        best_v = vvectors[counts[1][1]]

        # don't try and auto-rectify frontal surfaces
        if auto_normal_acceptable(normalized_cross(best_u, best_v)):
            num_vanishing_lines = counts[0][0] + counts[1][0]
            uv_dot = abs_dot(best_u, best_v)
            print 'u dot v = %s' % uv_dot
        else:
            best_u, best_v = None, None
            uv_dot = None

        # make sure these vectors are accurate
        if not uv_dot or uv_dot > 0.05:
            # try and find two other orthogonal vanishing points
            best_dot = 0.05
            best_u = None
            best_v = None
            for c1, i1 in counts:
                for c2, i2 in counts[:i1]:
                    d = abs_dot(vvectors[i1], vvectors[i2])
                    if d < best_dot:
                        # don't try and auto-rectify frontal surfaces
                        if auto_normal_acceptable(normalized_cross(
                                vvectors[i1], vvectors[i2])):
                            best_dot = d
                            if c1 > c2:
                                best_u = vvectors[i1]
                                best_v = vvectors[i2]
                            else:
                                best_u = vvectors[i2]
                                best_v = vvectors[i1]
                            num_vanishing_lines = c1 + c2

        if best_u is not None and best_v is not None:
            best_n = normalized_cross(best_u, best_v)
            method = 'A'

            # give up for some classes of objects
            if shape.name:
                name = shape.name.name.lower()
                if ((abs(best_n[1]) > 0.5 and name in (
                        'wall', 'door', 'window')) or
                    (abs(best_n[1]) < 0.5 and name in (
                        'floor', 'ceiling', 'table',
                        'worktop/countertop', 'carpet/rug'))):
                    method = best_u = best_v = best_n = None
                    num_vanishing_lines = 0

    # for walls that touch the edge of the photo, try using bbox center as a
    # vanishing point (i.e. assume top/bottom shapes are horizontal, side
    # shapes are vertical)
    if (try_fully_automatic and best_n is None and shape.name and
            (shape.substance is None or shape.substance.name != 'Painted')):
        if shape.name.name.lower() == 'wall':
            bbox = bbox_vertices(parse_vertices(shape.vertices))
            if ((bbox[0] < 0.05 and bbox[2] < 0.50) or
                    (bbox[0] > 0.50 and bbox[2] > 0.95)):

                bbox_n = photo.vanishing_point_to_vector((
                    0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5
                ))

                # find normal that best matches this fake bbox normal
                best_n_dot = 0.9
                best_n = None
                for n in vvectors:
                    if auto_normal_acceptable(n):
                        d = abs_dot(bbox_n, n)
                        if d > best_n_dot:
                            best_n_dot = d
                            best_n = n
                            method = 'O'

    # find best u vector if not already found
    if best_n is not None and best_u is None:
        # first check if any in-shape vanishing points
        # are perpendicular to the normal
        best_u = most_orthogonal_vector(
            best_n, [vvectors[i] for __, i in counts],
            tolerance_dot=0.05)

        # else, find the best u from all vectors
        if best_u is None:
            best_u = most_orthogonal_vector(best_n, vvectors)

    # failure
    if best_u is None or best_n is None:
        return (None, None, 0)

    # ortho-normalize system
    uvn = construct_uvn_frame(
        best_n, best_u, b, flip_to_match_image=True)

    # form uvnb matrix, column major
    uvnb = (
        uvn[0, 0], uvn[1, 0], uvn[2, 0], 0,
        uvn[0, 1], uvn[1, 1], uvn[2, 1], 0,
        uvn[0, 2], uvn[1, 2], uvn[2, 2], 0,
        b[0], b[1], b[2], 1
    )

    return uvnb, method, num_vanishing_lines
Exemplo n.º 4
0
def detect_vanishing_points_impl(photo, image, save=True):

    # algorithm parameters
    max_em_iter = 0  # if 0, don't do EM
    min_cluster_size = 10
    min_line_len2 = 4.0
    residual_stdev = 0.75
    max_clusters = 8
    outlier_weight = 0.2
    weight_clamp = 0.1
    lambda_perp = 1.0
    verbose = False

    width, height = image.size
    print 'size: %s x %s' % (width, height)

    vpdetection_dir = os.path.abspath(os.path.join(
        os.path.dirname(__file__), os.pardir, os.pardir, 'opt', 'vpdetection', 'matlab'
    ))

    tmpdir = tempfile.mkdtemp()
    try:

        # save image to local tmpdir
        localname = os.path.join(tmpdir, 'image.jpg')
        with open(tmpdir + '/image.jpg', 'wb') as target:
            save_image(image, target, format='JPEG', options={'quality': 90})

        # detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel,
        # J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a
        # False Detection Control. IEEE Transactions on Pattern Analysis and
        # Machine Intelligence, 32, 722.)
        linesname = os.path.join(tmpdir, 'lines.txt')
        matlab_command = ";".join([
            "try",
            "addpath('../lsd-1.5/')",
            "lines = lsd(double(rgb2gray(imread('%s'))))" % localname,
            "save('%s', 'lines', '-ascii', '-tabs')" % linesname,
            "catch",
            "end",
            "quit",
        ])
        print 'matlab command: %s' % matlab_command
        subprocess.check_call(args=[
            'matlab', '-nodesktop', '-nosplash', '-nodisplay',
            '-r', matlab_command
        ], cwd=vpdetection_dir)

        # cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008).
        # Robust multiple structures estimation with J-Linkage. European
        # Conference on Computer Vision(ECCV), 2008.)
        # and (Tardif J.-P., Non-iterative Approach for Fast and Accurate
        # Vanishing Point Detection, 12th IEEE International Conference on
        # Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.)
        clustername = os.path.join(tmpdir, 'clusters.txt')
        subprocess.check_call(
            args=['./vpdetection', linesname, clustername],
            cwd=vpdetection_dir)

        # collect line clusters
        clusters_dict = {}
        all_lines = []
        for row in open(clustername, 'r').readlines():
            cols = row.split()
            idx = int(cols[4])
            line = [float(f) for f in cols[0:4]]

            # discard small lines
            x1, y1, x2, y2 = line
            len2 = (x1 - x2) ** 2 + (y2 - y1) ** 2
            if len2 < min_line_len2:
                continue

            if idx in clusters_dict:
                clusters_dict[idx].append(line)
                all_lines.append(line)
            else:
                clusters_dict[idx] = [line]

    finally:
        shutil.rmtree(tmpdir)

    # discard invalid clusters and sort by cluster length
    thresh = 3 if max_em_iter else min_cluster_size
    clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values())
    clusters.sort(key=line_cluster_length, reverse=True)
    if max_em_iter and len(clusters) > max_clusters:
        clusters = clusters[:max_clusters]
    print "Using %s clusters and %s lines" % (len(clusters), len(all_lines))
    if not clusters:
        print "Not enough clusters"
        return

    # Solve for optimal vanishing point using V_GS in 5.2 section of
    # (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf).
    # where "optimal" minimizes algebraic error.
    vectors = []
    for lines in clusters:
        # Minimize 'algebraic' error to get an initial solution
        A = np.zeros((len(lines), 3))
        for i in xrange(0, len(lines)):
            x1, y1, x2, y2 = lines[i]
            A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2]
        __, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True)
        if VT.shape != (3, 3):
            raise ValueError("Invalid SVD shape (%s)" % VT.size)
        x, y, w = VT[2, :]
        p = [x / w, y / w]
        v = photo.vanishing_point_to_vector(
            (p[0] / width, p[1] / height)
        )
        vectors.append(v)

    # EM
    if max_em_iter:

        # complete orthonormal system
        if len(vectors) >= 2:
            vectors.append(normalized_cross(vectors[0], vectors[1]))

        ### EM refinement ###

        x0 = None
        x_opt = None
        exp_coeff = 0.5 / (residual_stdev ** 2)

        num_weights_nnz = 0
        num_weights = 0

        for em_iter in xrange(max_em_iter):

            ### E STEP ###

            # convert back to vanishing points
            points = vectors_to_points(photo, image, vectors)

            # last column is the outlier cluster
            weights = np.zeros((len(all_lines), len(vectors) + 1))

            # estimate weights (assume uniform prior)
            for i_p, p in enumerate(points):
                weights[:, i_p] = [line_residual(l, p) for l in all_lines]
            weights = np.exp(-exp_coeff * np.square(weights))

            # outlier weight
            weights[:, len(points)] = outlier_weight

            # normalize each row (each line segment) to have unit sum
            weights_row_sum = weights.sum(axis=1)
            weights /= weights_row_sum[:, np.newaxis]

            # add sparsity
            weights[weights < weight_clamp] = 0
            num_weights += weights.size
            num_weights_nnz += np.count_nonzero(weights)

            # check convergence
            if (em_iter >= 10 and len(x0) == len(x_opt) and
                    np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5):
                break

            # sort by weight
            if len(vectors) > 1:
                vectors_weights = [
                    (v, weights[:, i_v].sum()) for i_v, v in enumerate(vectors)
                ]
                vectors_weights.sort(key=lambda x: x[1], reverse=True)
                vectors = [x[0] for x in vectors_weights]

            ### M STEP ###

            # objective function to minimize
            def objective_function(x, *args):
                cur_vectors = unpack_x(x)
                cur_points = vectors_to_points(photo, image, cur_vectors)

                # line-segment errors
                residuals = [
                    weights[i_l, i_p] * line_residual(all_lines[i_l], p)
                    for i_p, p in enumerate(cur_points)
                    for i_l in np.flatnonzero(weights[:, i_p])
                ]

                # penalize deviations from 45 or 90 degree angles
                if lambda_perp:
                    residuals += [
                        lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
                        for i_v, v in enumerate(cur_vectors)
                        for w in cur_vectors[:i_v]
                    ]

                return residuals

            # slowly vary parameters
            t = min(1.0, em_iter / 20.0)

            # vary tol from 1e-2 to 1e-6
            tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t)

            from scipy.optimize import leastsq
            x0 = pack_x(vectors)
            x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol)
            vectors = unpack_x(x_opt)

            ### BETWEEN ITERATIONS ###

            if verbose:
                print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % (
                    em_iter, len(vectors), 100.0 * num_weights_nnz / num_weights)
                print 'residual: %s' % sum(y ** 2 for y in objective_function(x_opt))

            # complete orthonormal system if missing
            if len(vectors) == 2:
                vectors.append(normalized_cross(vectors[0], vectors[1]))

            # merge similar clusters
            cluster_merge_dot = math.cos(math.radians(t * 20.0))
            vectors_merged = []
            for v in vectors:
                if (not vectors_merged or
                        all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)):
                    vectors_merged.append(v)
            if verbose and len(vectors) != len(vectors_merged):
                print 'Merging %s --> %s vectors' % (len(vectors), len(vectors_merged))
            vectors = vectors_merged

        residual = sum(r ** 2 for r in objective_function(x_opt))
        print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % (
            em_iter, residual, len(vectors), 100.0 * num_weights_nnz / num_weights)

        # final points
        points = vectors_to_points(photo, image, vectors)

        # sanity checks
        assert len(vectors) == len(points)

        # re-assign clusters
        clusters_points = [([], p) for p in points]
        line_map_cluster = np.argmax(weights, axis=1)
        for i_l, l in enumerate(all_lines):
            i_c = line_map_cluster[i_l]
            if i_c < len(points):
                clusters_points[i_c][0].append(l)

        # throw away small clusters
        clusters_points = filter(
            lambda x: len(x[0]) >= min_cluster_size, clusters_points)

        # reverse sort by cluster length
        clusters_points.sort(
            key=lambda x: line_cluster_length(x[0]), reverse=True)

        # split into two parallel arrays
        clusters = [cp[0] for cp in clusters_points]
        points = [cp[1] for cp in clusters_points]

    else:  # no EM

        for i_v, lines in enumerate(clusters):
            def objective_function(x, *args):
                p = vectors_to_points(photo, image, unpack_x(x))[0]
                return [line_residual(l, p) for l in lines]
            from scipy.optimize import leastsq
            x0 = pack_x([vectors[i_v]])
            x_opt, __ = leastsq(objective_function, x0)
            vectors[i_v] = unpack_x(x_opt)[0]

        # delete similar vectors
        cluster_merge_dot = math.cos(math.radians(20.0))
        vectors_merged = []
        clusters_merged = []
        for i_v, v in enumerate(vectors):
            if (not vectors_merged or
                    all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)):
                vectors_merged.append(v)
                clusters_merged.append(clusters[i_v])
        vectors = vectors_merged
        clusters = clusters_merged

        # clamp number of vectors
        if len(clusters) > max_clusters:
            vectors = vectors[:max_clusters]
            clusters = clusters[:max_clusters]

        points = vectors_to_points(photo, image, vectors)

    # normalize to [0, 0], [1, 1]
    clusters_normalized = [[
        [l[0] / width, l[1] / height, l[2] / width, l[3] / height]
        for l in lines
    ] for lines in clusters]

    points_normalized = [
        (x / width, y / height) for (x, y) in points
    ]

    # save result
    photo.vanishing_lines = json.dumps(clusters_normalized)
    photo.vanishing_points = json.dumps(points_normalized)
    photo.vanishing_length = sum(line_cluster_length(c)
                                 for c in clusters_normalized)
    if save:
        photo.save()