def objective_function(x, *args):
cur_vectors = unpack_x(x)
cur_points = vectors_to_points(photo, image, cur_vectors)
# line-segment errors
residuals = [
weights[i_l, i_p] * line_residual(all_lines[i_l], p)
for i_p, p in enumerate(cur_points)
for i_l in np.flatnonzero(weights[:, i_p])
]
# penalize deviations from 45 or 90 degree angles
if lambda_perp:
residuals += [
lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
for i_v, v in enumerate(cur_vectors)
for w in cur_vectors[:i_v]
]
return residuals
def objective_function(x, *args):
cur_vectors = unpack_x(x)
cur_points = vectors_to_points(photo, image, cur_vectors)
# line-segment errors
residuals = [
weights[i_l, i_p] * line_residual(all_lines[i_l], p)
for i_p, p in enumerate(cur_points)
for i_l in np.flatnonzero(weights[:, i_p])
]
# penalize deviations from 45 or 90 degree angles
if lambda_perp:
residuals += [
lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
for i_v, v in enumerate(cur_vectors)
for w in cur_vectors[:i_v]
]
return residuals
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
""" Return (uvnb, num_vanishing_points) """
print 'estimate_uvnb for shape_id: %s' % shape.id
# local import to avoid cyclic dependencies
from shapes.utils import parse_vertices, parse_triangles, \
parse_segments, bbox_vertices
# load photo
photo = shape.photo
if not photo.vanishing_lines or not photo.vanishing_points:
raise ValueError("Vanishing points not computed")
if not photo.focal_y:
raise ValueError("Photo does not have focal_y")
vlines = json.loads(photo.vanishing_lines)
vpoints = json.loads(photo.vanishing_points)
vvectors = copy.copy(photo.vanishing_vectors())
if len(vlines) != len(vpoints):
raise ValueError("Invalid vanishing points data structure")
# add any missing vanishing points
vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)
# find vanishing lines inside shape
vertices = parse_vertices(shape.vertices)
segments = parse_segments(shape.triangles)
triangles = parse_triangles(shape.triangles)
# intersect shapes with segments
counts = []
for idx in xrange(len(vlines)):
# re-pack for geom routines
query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]
from common.geom import triangles_segments_intersections_only
n = len(
triangles_segments_intersections_only(vertices, segments,
triangles, query_segments))
if n >= 5:
counts.append((n, idx))
counts.sort(key=lambda x: x[0], reverse=True)
# function to judge normals: its vanishing line can't intersect the shape.
def auto_normal_acceptable(n):
sign = None
for (x, y) in vertices:
# vanishing line
line = (n[0], n[1], n[2] * photo.focal_y)
# signed distance
d = ((x - 0.5) * photo.aspect_ratio * line[0] +
(0.5 - y) * line[1] + # flip y
line[2])
if abs(d) < 0.05:
return False
elif sign is None:
sign = (d > 0)
else:
if sign != (d > 0):
return False
return True
# find coordinate frame
best_n = None
best_u = None
method = None
num_vanishing_lines = 0
# make sure shape has label
if not shape.label_pos_x or not shape.label_pos_y:
from shapes.utils import update_shape_label_pos
update_shape_label_pos(shape)
# place label in 3D
b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
b = [(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) /
photo.focal_y, (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
b_z]
# estimate closest human normal
human_labels = list(
ShapeRectifiedNormalLabel.objects.filter(
shape=shape,
automatic=False,
correct_score__isnull=False,
).order_by('-correct_score'))
human_labels += list(
ShapeRectifiedNormalLabel.objects.filter(shape=shape,
automatic=False,
correct_score__isnull=True))
if human_labels:
for label in human_labels:
human_u = label.u()
human_n = label.n()
b = list(label.uvnb_numpy()[0:3, 3].flat)
# find best normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
d = abs_dot(human_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'S'
# if there is a match find u and quit
if best_n is not None:
# find best u
best_u_dot = 0
best_u = None
for u in vvectors:
if abs_dot(u, best_n) < 0.1:
d = abs_dot(human_u, u)
if d > best_u_dot:
best_u_dot = d
best_u = u
break
# try using object label
if best_n is None and shape.name:
if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
best_y = 0.9
for v in vvectors[0:3]:
if abs(v[1]) > best_y:
best_y = abs(v[1])
best_n = v
method = 'O'
# try fully automatic method if human normals are not good enough
if (try_fully_automatic and best_n is None and len(vpoints) >= 3
and len(counts) >= 2 and
(shape.substance is None or shape.substance.name != 'Painted')):
# choose two dominant vanishing points
best_u = vvectors[counts[0][1]]
best_v = vvectors[counts[1][1]]
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(best_u, best_v)):
num_vanishing_lines = counts[0][0] + counts[1][0]
uv_dot = abs_dot(best_u, best_v)
print 'u dot v = %s' % uv_dot
else:
best_u, best_v = None, None
uv_dot = None
# make sure these vectors are accurate
if not uv_dot or uv_dot > 0.05:
# try and find two other orthogonal vanishing points
best_dot = 0.05
best_u = None
best_v = None
for c1, i1 in counts:
for c2, i2 in counts[:i1]:
d = abs_dot(vvectors[i1], vvectors[i2])
if d < best_dot:
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(
normalized_cross(vvectors[i1], vvectors[i2])):
best_dot = d
if c1 > c2:
best_u = vvectors[i1]
best_v = vvectors[i2]
else:
best_u = vvectors[i2]
best_v = vvectors[i1]
num_vanishing_lines = c1 + c2
if best_u is not None and best_v is not None:
best_n = normalized_cross(best_u, best_v)
method = 'A'
# give up for some classes of objects
if shape.name:
name = shape.name.name.lower()
if ((abs(best_n[1]) > 0.5
and name in ('wall', 'door', 'window'))
or (abs(best_n[1]) < 0.5
and name in ('floor', 'ceiling', 'table',
'worktop/countertop', 'carpet/rug'))):
method = best_u = best_v = best_n = None
num_vanishing_lines = 0
# for walls that touch the edge of the photo, try using bbox center as a
# vanishing point (i.e. assume top/bottom shapes are horizontal, side
# shapes are vertical)
if (try_fully_automatic and best_n is None and shape.name and
(shape.substance is None or shape.substance.name != 'Painted')):
if shape.name.name.lower() == 'wall':
bbox = bbox_vertices(parse_vertices(shape.vertices))
if ((bbox[0] < 0.05 and bbox[2] < 0.50)
or (bbox[0] > 0.50 and bbox[2] > 0.95)):
bbox_n = photo.vanishing_point_to_vector(
(0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5))
# find normal that best matches this fake bbox normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
if auto_normal_acceptable(n):
d = abs_dot(bbox_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'O'
# find best u vector if not already found
if best_n is not None and best_u is None:
# first check if any in-shape vanishing points
# are perpendicular to the normal
best_u = most_orthogonal_vector(best_n,
[vvectors[i] for __, i in counts],
tolerance_dot=0.05)
# else, find the best u from all vectors
if best_u is None:
best_u = most_orthogonal_vector(best_n, vvectors)
# failure
if best_u is None or best_n is None:
return (None, None, 0)
# ortho-normalize system
uvn = construct_uvn_frame(best_n, best_u, b, flip_to_match_image=True)
# form uvnb matrix, column major
uvnb = (uvn[0, 0], uvn[1, 0], uvn[2, 0], 0, uvn[0, 1], uvn[1, 1],
uvn[2, 1], 0, uvn[0, 2], uvn[1, 2], uvn[2,
2], 0, b[0], b[1], b[2], 1)
return uvnb, method, num_vanishing_lines
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
""" Return (uvnb, num_vanishing_points) """
print 'estimate_uvnb for shape_id: %s' % shape.id
# local import to avoid cyclic dependencies
from shapes.utils import parse_vertices, parse_triangles, \
parse_segments, bbox_vertices
# load photo
photo = shape.photo
if not photo.vanishing_lines or not photo.vanishing_points:
raise ValueError("Vanishing points not computed")
if not photo.focal_y:
raise ValueError("Photo does not have focal_y")
vlines = json.loads(photo.vanishing_lines)
vpoints = json.loads(photo.vanishing_points)
vvectors = copy.copy(photo.vanishing_vectors())
if len(vlines) != len(vpoints):
raise ValueError("Invalid vanishing points data structure")
# add any missing vanishing points
vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)
# find vanishing lines inside shape
vertices = parse_vertices(shape.vertices)
segments = parse_segments(shape.triangles)
triangles = parse_triangles(shape.triangles)
# intersect shapes with segments
counts = []
for idx in xrange(len(vlines)):
# re-pack for geom routines
query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]
from common.geom import triangles_segments_intersections_only
n = len(triangles_segments_intersections_only(
vertices, segments, triangles, query_segments))
if n >= 5:
counts.append((n, idx))
counts.sort(key=lambda x: x[0], reverse=True)
# function to judge normals: its vanishing line can't intersect the shape.
def auto_normal_acceptable(n):
sign = None
for (x, y) in vertices:
# vanishing line
line = (n[0], n[1], n[2] * photo.focal_y)
# signed distance
d = (
(x - 0.5) * photo.aspect_ratio * line[0] +
(0.5 - y) * line[1] + # flip y
line[2]
)
if abs(d) < 0.05:
return False
elif sign is None:
sign = (d > 0)
else:
if sign != (d > 0):
return False
return True
# find coordinate frame
best_n = None
best_u = None
method = None
num_vanishing_lines = 0
# make sure shape has label
if not shape.label_pos_x or not shape.label_pos_y:
from shapes.utils import update_shape_label_pos
update_shape_label_pos(shape)
# place label in 3D
b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
b = [
(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) / photo.focal_y,
(0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
b_z
]
# estimate closest human normal
human_labels = list(ShapeRectifiedNormalLabel.objects.filter(
shape=shape, automatic=False, correct_score__isnull=False,
).order_by('-correct_score'))
human_labels += list(ShapeRectifiedNormalLabel.objects.filter(
shape=shape, automatic=False, correct_score__isnull=True))
if human_labels:
for label in human_labels:
human_u = label.u()
human_n = label.n()
b = list(label.uvnb_numpy()[0:3, 3].flat)
# find best normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
d = abs_dot(human_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'S'
# if there is a match find u and quit
if best_n is not None:
# find best u
best_u_dot = 0
best_u = None
for u in vvectors:
if abs_dot(u, best_n) < 0.1:
d = abs_dot(human_u, u)
if d > best_u_dot:
best_u_dot = d
best_u = u
break
# try using object label
if best_n is None and shape.name:
if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
best_y = 0.9
for v in vvectors[0:3]:
if abs(v[1]) > best_y:
best_y = abs(v[1])
best_n = v
method = 'O'
# try fully automatic method if human normals are not good enough
if (try_fully_automatic and best_n is None and len(vpoints) >= 3 and len(counts) >= 2 and
(shape.substance is None or shape.substance.name != 'Painted')):
# choose two dominant vanishing points
best_u = vvectors[counts[0][1]]
best_v = vvectors[counts[1][1]]
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(best_u, best_v)):
num_vanishing_lines = counts[0][0] + counts[1][0]
uv_dot = abs_dot(best_u, best_v)
print 'u dot v = %s' % uv_dot
else:
best_u, best_v = None, None
uv_dot = None
# make sure these vectors are accurate
if not uv_dot or uv_dot > 0.05:
# try and find two other orthogonal vanishing points
best_dot = 0.05
best_u = None
best_v = None
for c1, i1 in counts:
for c2, i2 in counts[:i1]:
d = abs_dot(vvectors[i1], vvectors[i2])
if d < best_dot:
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(
vvectors[i1], vvectors[i2])):
best_dot = d
if c1 > c2:
best_u = vvectors[i1]
best_v = vvectors[i2]
else:
best_u = vvectors[i2]
best_v = vvectors[i1]
num_vanishing_lines = c1 + c2
if best_u is not None and best_v is not None:
best_n = normalized_cross(best_u, best_v)
method = 'A'
# give up for some classes of objects
if shape.name:
name = shape.name.name.lower()
if ((abs(best_n[1]) > 0.5 and name in (
'wall', 'door', 'window')) or
(abs(best_n[1]) < 0.5 and name in (
'floor', 'ceiling', 'table',
'worktop/countertop', 'carpet/rug'))):
method = best_u = best_v = best_n = None
num_vanishing_lines = 0
# for walls that touch the edge of the photo, try using bbox center as a
# vanishing point (i.e. assume top/bottom shapes are horizontal, side
# shapes are vertical)
if (try_fully_automatic and best_n is None and shape.name and
(shape.substance is None or shape.substance.name != 'Painted')):
if shape.name.name.lower() == 'wall':
bbox = bbox_vertices(parse_vertices(shape.vertices))
if ((bbox[0] < 0.05 and bbox[2] < 0.50) or
(bbox[0] > 0.50 and bbox[2] > 0.95)):
bbox_n = photo.vanishing_point_to_vector((
0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5
))
# find normal that best matches this fake bbox normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
if auto_normal_acceptable(n):
d = abs_dot(bbox_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'O'
# find best u vector if not already found
if best_n is not None and best_u is None:
# first check if any in-shape vanishing points
# are perpendicular to the normal
best_u = most_orthogonal_vector(
best_n, [vvectors[i] for __, i in counts],
tolerance_dot=0.05)
# else, find the best u from all vectors
if best_u is None:
best_u = most_orthogonal_vector(best_n, vvectors)
# failure
if best_u is None or best_n is None:
return (None, None, 0)
# ortho-normalize system
uvn = construct_uvn_frame(
best_n, best_u, b, flip_to_match_image=True)
# form uvnb matrix, column major
uvnb = (
uvn[0, 0], uvn[1, 0], uvn[2, 0], 0,
uvn[0, 1], uvn[1, 1], uvn[2, 1], 0,
uvn[0, 2], uvn[1, 2], uvn[2, 2], 0,
b[0], b[1], b[2], 1
)
return uvnb, method, num_vanishing_lines
def detect_vanishing_points_impl(photo, image, save=True):
# algorithm parameters
max_em_iter = 0 # if 0, don't do EM
min_cluster_size = 10
min_line_len2 = 4.0
residual_stdev = 0.75
max_clusters = 8
outlier_weight = 0.2
weight_clamp = 0.1
lambda_perp = 1.0
verbose = False
width, height = image.size
print 'size: %s x %s' % (width, height)
vpdetection_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'opt',
'vpdetection', 'matlab'))
tmpdir = tempfile.mkdtemp()
try:
# save image to local tmpdir
localname = os.path.join(tmpdir, 'image.jpg')
with open(tmpdir + '/image.jpg', 'wb') as target:
save_image(image, target, format='JPEG', options={'quality': 90})
# detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel,
# J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a
# False Detection Control. IEEE Transactions on Pattern Analysis and
# Machine Intelligence, 32, 722.)
linesname = os.path.join(tmpdir, 'lines.txt')
matlab_command = ";".join([
"try",
"addpath('../lsd-1.5/')",
"lines = lsd(double(rgb2gray(imread('%s'))))" % localname,
"save('%s', 'lines', '-ascii', '-tabs')" % linesname,
"catch",
"end",
"quit",
])
print 'matlab command: %s' % matlab_command
subprocess.check_call(args=[
'matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
matlab_command
],
cwd=vpdetection_dir)
# cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008).
# Robust multiple structures estimation with J-Linkage. European
# Conference on Computer Vision(ECCV), 2008.)
# and (Tardif J.-P., Non-iterative Approach for Fast and Accurate
# Vanishing Point Detection, 12th IEEE International Conference on
# Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.)
clustername = os.path.join(tmpdir, 'clusters.txt')
subprocess.check_call(args=['./vpdetection', linesname, clustername],
cwd=vpdetection_dir)
# collect line clusters
clusters_dict = {}
all_lines = []
for row in open(clustername, 'r').readlines():
cols = row.split()
idx = int(cols[4])
line = [float(f) for f in cols[0:4]]
# discard small lines
x1, y1, x2, y2 = line
len2 = (x1 - x2)**2 + (y2 - y1)**2
if len2 < min_line_len2:
continue
if idx in clusters_dict:
clusters_dict[idx].append(line)
all_lines.append(line)
else:
clusters_dict[idx] = [line]
finally:
shutil.rmtree(tmpdir)
# discard invalid clusters and sort by cluster length
thresh = 3 if max_em_iter else min_cluster_size
clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values())
clusters.sort(key=line_cluster_length, reverse=True)
if max_em_iter and len(clusters) > max_clusters:
clusters = clusters[:max_clusters]
print "Using %s clusters and %s lines" % (len(clusters), len(all_lines))
if not clusters:
print "Not enough clusters"
return
# Solve for optimal vanishing point using V_GS in 5.2 section of
# (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf).
# where "optimal" minimizes algebraic error.
vectors = []
for lines in clusters:
# Minimize 'algebraic' error to get an initial solution
A = np.zeros((len(lines), 3))
for i in xrange(0, len(lines)):
x1, y1, x2, y2 = lines[i]
A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2]
__, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True)
if VT.shape != (3, 3):
raise ValueError("Invalid SVD shape (%s)" % VT.size)
x, y, w = VT[2, :]
p = [x / w, y / w]
v = photo.vanishing_point_to_vector((p[0] / width, p[1] / height))
vectors.append(v)
# EM
if max_em_iter:
# complete orthonormal system
if len(vectors) >= 2:
vectors.append(normalized_cross(vectors[0], vectors[1]))
### EM refinement ###
x0 = None
x_opt = None
exp_coeff = 0.5 / (residual_stdev**2)
num_weights_nnz = 0
num_weights = 0
for em_iter in xrange(max_em_iter):
### E STEP ###
# convert back to vanishing points
points = vectors_to_points(photo, image, vectors)
# last column is the outlier cluster
weights = np.zeros((len(all_lines), len(vectors) + 1))
# estimate weights (assume uniform prior)
for i_p, p in enumerate(points):
weights[:, i_p] = [line_residual(l, p) for l in all_lines]
weights = np.exp(-exp_coeff * np.square(weights))
# outlier weight
weights[:, len(points)] = outlier_weight
# normalize each row (each line segment) to have unit sum
weights_row_sum = weights.sum(axis=1)
weights /= weights_row_sum[:, np.newaxis]
# add sparsity
weights[weights < weight_clamp] = 0
num_weights += weights.size
num_weights_nnz += np.count_nonzero(weights)
# check convergence
if (em_iter >= 10 and len(x0) == len(x_opt) and
np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5):
break
# sort by weight
if len(vectors) > 1:
vectors_weights = [(v, weights[:, i_v].sum())
for i_v, v in enumerate(vectors)]
vectors_weights.sort(key=lambda x: x[1], reverse=True)
vectors = [x[0] for x in vectors_weights]
### M STEP ###
# objective function to minimize
def objective_function(x, *args):
cur_vectors = unpack_x(x)
cur_points = vectors_to_points(photo, image, cur_vectors)
# line-segment errors
residuals = [
weights[i_l, i_p] * line_residual(all_lines[i_l], p)
for i_p, p in enumerate(cur_points)
for i_l in np.flatnonzero(weights[:, i_p])
]
# penalize deviations from 45 or 90 degree angles
if lambda_perp:
residuals += [
lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
for i_v, v in enumerate(cur_vectors)
for w in cur_vectors[:i_v]
]
return residuals
# slowly vary parameters
t = min(1.0, em_iter / 20.0)
# vary tol from 1e-2 to 1e-6
tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t)
from scipy.optimize import leastsq
x0 = pack_x(vectors)
x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol)
vectors = unpack_x(x_opt)
### BETWEEN ITERATIONS ###
if verbose:
print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % (
em_iter, len(vectors),
100.0 * num_weights_nnz / num_weights)
print 'residual: %s' % sum(y**2
for y in objective_function(x_opt))
# complete orthonormal system if missing
if len(vectors) == 2:
vectors.append(normalized_cross(vectors[0], vectors[1]))
# merge similar clusters
cluster_merge_dot = math.cos(math.radians(t * 20.0))
vectors_merged = []
for v in vectors:
if (not vectors_merged or all(
abs_dot(v, w) < cluster_merge_dot
for w in vectors_merged)):
vectors_merged.append(v)
if verbose and len(vectors) != len(vectors_merged):
print 'Merging %s --> %s vectors' % (len(vectors),
len(vectors_merged))
vectors = vectors_merged
residual = sum(r**2 for r in objective_function(x_opt))
print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % (
em_iter, residual, len(vectors),
100.0 * num_weights_nnz / num_weights)
# final points
points = vectors_to_points(photo, image, vectors)
# sanity checks
assert len(vectors) == len(points)
# re-assign clusters
clusters_points = [([], p) for p in points]
line_map_cluster = np.argmax(weights, axis=1)
for i_l, l in enumerate(all_lines):
i_c = line_map_cluster[i_l]
if i_c < len(points):
clusters_points[i_c][0].append(l)
# throw away small clusters
clusters_points = filter(lambda x: len(x[0]) >= min_cluster_size,
clusters_points)
# reverse sort by cluster length
clusters_points.sort(key=lambda x: line_cluster_length(x[0]),
reverse=True)
# split into two parallel arrays
clusters = [cp[0] for cp in clusters_points]
points = [cp[1] for cp in clusters_points]
else: # no EM
for i_v, lines in enumerate(clusters):
def objective_function(x, *args):
p = vectors_to_points(photo, image, unpack_x(x))[0]
return [line_residual(l, p) for l in lines]
from scipy.optimize import leastsq
x0 = pack_x([vectors[i_v]])
x_opt, __ = leastsq(objective_function, x0)
vectors[i_v] = unpack_x(x_opt)[0]
# delete similar vectors
cluster_merge_dot = math.cos(math.radians(20.0))
vectors_merged = []
clusters_merged = []
for i_v, v in enumerate(vectors):
if (not vectors_merged or all(
abs_dot(v, w) < cluster_merge_dot
for w in vectors_merged)):
vectors_merged.append(v)
clusters_merged.append(clusters[i_v])
vectors = vectors_merged
clusters = clusters_merged
# clamp number of vectors
if len(clusters) > max_clusters:
vectors = vectors[:max_clusters]
clusters = clusters[:max_clusters]
points = vectors_to_points(photo, image, vectors)
# normalize to [0, 0], [1, 1]
clusters_normalized = [[[
l[0] / width, l[1] / height, l[2] / width, l[3] / height
] for l in lines] for lines in clusters]
points_normalized = [(x / width, y / height) for (x, y) in points]
# save result
photo.vanishing_lines = json.dumps(clusters_normalized)
photo.vanishing_points = json.dumps(points_normalized)
photo.vanishing_length = sum(
line_cluster_length(c) for c in clusters_normalized)
if save:
photo.save()
def detect_vanishing_points_impl(photo, image, save=True):
# algorithm parameters
max_em_iter = 0 # if 0, don't do EM
min_cluster_size = 10
min_line_len2 = 4.0
residual_stdev = 0.75
max_clusters = 8
outlier_weight = 0.2
weight_clamp = 0.1
lambda_perp = 1.0
verbose = False
width, height = image.size
print 'size: %s x %s' % (width, height)
vpdetection_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), os.pardir, os.pardir, 'opt', 'vpdetection', 'matlab'
))
tmpdir = tempfile.mkdtemp()
try:
# save image to local tmpdir
localname = os.path.join(tmpdir, 'image.jpg')
with open(tmpdir + '/image.jpg', 'wb') as target:
save_image(image, target, format='JPEG', options={'quality': 90})
# detect line segments using LSD (Grompone, G., Jakubowicz, J., Morel,
# J. and Randall, G. (2010). LSD: A Fast Line Segment Detector with a
# False Detection Control. IEEE Transactions on Pattern Analysis and
# Machine Intelligence, 32, 722.)
linesname = os.path.join(tmpdir, 'lines.txt')
matlab_command = ";".join([
"try",
"addpath('../lsd-1.5/')",
"lines = lsd(double(rgb2gray(imread('%s'))))" % localname,
"save('%s', 'lines', '-ascii', '-tabs')" % linesname,
"catch",
"end",
"quit",
])
print 'matlab command: %s' % matlab_command
subprocess.check_call(args=[
'matlab', '-nodesktop', '-nosplash', '-nodisplay',
'-r', matlab_command
], cwd=vpdetection_dir)
# cluster lines using J-linkage (Toldo, R. and Fusiello, A. (2008).
# Robust multiple structures estimation with J-Linkage. European
# Conference on Computer Vision(ECCV), 2008.)
# and (Tardif J.-P., Non-iterative Approach for Fast and Accurate
# Vanishing Point Detection, 12th IEEE International Conference on
# Computer Vision, Kyoto, Japan, September 27 - October 4, 2009.)
clustername = os.path.join(tmpdir, 'clusters.txt')
subprocess.check_call(
args=['./vpdetection', linesname, clustername],
cwd=vpdetection_dir)
# collect line clusters
clusters_dict = {}
all_lines = []
for row in open(clustername, 'r').readlines():
cols = row.split()
idx = int(cols[4])
line = [float(f) for f in cols[0:4]]
# discard small lines
x1, y1, x2, y2 = line
len2 = (x1 - x2) ** 2 + (y2 - y1) ** 2
if len2 < min_line_len2:
continue
if idx in clusters_dict:
clusters_dict[idx].append(line)
all_lines.append(line)
else:
clusters_dict[idx] = [line]
finally:
shutil.rmtree(tmpdir)
# discard invalid clusters and sort by cluster length
thresh = 3 if max_em_iter else min_cluster_size
clusters = filter(lambda x: len(x) >= thresh, clusters_dict.values())
clusters.sort(key=line_cluster_length, reverse=True)
if max_em_iter and len(clusters) > max_clusters:
clusters = clusters[:max_clusters]
print "Using %s clusters and %s lines" % (len(clusters), len(all_lines))
if not clusters:
print "Not enough clusters"
return
# Solve for optimal vanishing point using V_GS in 5.2 section of
# (http://www-etud.iro.umontreal.ca/~tardif/fichiers/Tardif_ICCV2009.pdf).
# where "optimal" minimizes algebraic error.
vectors = []
for lines in clusters:
# Minimize 'algebraic' error to get an initial solution
A = np.zeros((len(lines), 3))
for i in xrange(0, len(lines)):
x1, y1, x2, y2 = lines[i]
A[i, :] = [y1 - y2, x2 - x1, x1 * y2 - y1 * x2]
__, __, VT = np.linalg.svd(A, full_matrices=False, compute_uv=True)
if VT.shape != (3, 3):
raise ValueError("Invalid SVD shape (%s)" % VT.size)
x, y, w = VT[2, :]
p = [x / w, y / w]
v = photo.vanishing_point_to_vector(
(p[0] / width, p[1] / height)
)
vectors.append(v)
# EM
if max_em_iter:
# complete orthonormal system
if len(vectors) >= 2:
vectors.append(normalized_cross(vectors[0], vectors[1]))
### EM refinement ###
x0 = None
x_opt = None
exp_coeff = 0.5 / (residual_stdev ** 2)
num_weights_nnz = 0
num_weights = 0
for em_iter in xrange(max_em_iter):
### E STEP ###
# convert back to vanishing points
points = vectors_to_points(photo, image, vectors)
# last column is the outlier cluster
weights = np.zeros((len(all_lines), len(vectors) + 1))
# estimate weights (assume uniform prior)
for i_p, p in enumerate(points):
weights[:, i_p] = [line_residual(l, p) for l in all_lines]
weights = np.exp(-exp_coeff * np.square(weights))
# outlier weight
weights[:, len(points)] = outlier_weight
# normalize each row (each line segment) to have unit sum
weights_row_sum = weights.sum(axis=1)
weights /= weights_row_sum[:, np.newaxis]
# add sparsity
weights[weights < weight_clamp] = 0
num_weights += weights.size
num_weights_nnz += np.count_nonzero(weights)
# check convergence
if (em_iter >= 10 and len(x0) == len(x_opt) and
np.linalg.norm(np.array(x0) - np.array(x_opt)) <= 1e-5):
break
# sort by weight
if len(vectors) > 1:
vectors_weights = [
(v, weights[:, i_v].sum()) for i_v, v in enumerate(vectors)
]
vectors_weights.sort(key=lambda x: x[1], reverse=True)
vectors = [x[0] for x in vectors_weights]
### M STEP ###
# objective function to minimize
def objective_function(x, *args):
cur_vectors = unpack_x(x)
cur_points = vectors_to_points(photo, image, cur_vectors)
# line-segment errors
residuals = [
weights[i_l, i_p] * line_residual(all_lines[i_l], p)
for i_p, p in enumerate(cur_points)
for i_l in np.flatnonzero(weights[:, i_p])
]
# penalize deviations from 45 or 90 degree angles
if lambda_perp:
residuals += [
lambda_perp * math.sin(4 * math.acos(abs_dot(v, w)))
for i_v, v in enumerate(cur_vectors)
for w in cur_vectors[:i_v]
]
return residuals
# slowly vary parameters
t = min(1.0, em_iter / 20.0)
# vary tol from 1e-2 to 1e-6
tol = math.exp(math.log(1e-2) * (1 - t) + math.log(1e-6) * t)
from scipy.optimize import leastsq
x0 = pack_x(vectors)
x_opt, __ = leastsq(objective_function, x0, ftol=tol, xtol=tol)
vectors = unpack_x(x_opt)
### BETWEEN ITERATIONS ###
if verbose:
print 'EM: %s iters, %s clusters, weight sparsity: %s%%' % (
em_iter, len(vectors), 100.0 * num_weights_nnz / num_weights)
print 'residual: %s' % sum(y ** 2 for y in objective_function(x_opt))
# complete orthonormal system if missing
if len(vectors) == 2:
vectors.append(normalized_cross(vectors[0], vectors[1]))
# merge similar clusters
cluster_merge_dot = math.cos(math.radians(t * 20.0))
vectors_merged = []
for v in vectors:
if (not vectors_merged or
all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)):
vectors_merged.append(v)
if verbose and len(vectors) != len(vectors_merged):
print 'Merging %s --> %s vectors' % (len(vectors), len(vectors_merged))
vectors = vectors_merged
residual = sum(r ** 2 for r in objective_function(x_opt))
print 'EM: %s iters, residual: %s, %s clusters, weight sparsity: %s%%' % (
em_iter, residual, len(vectors), 100.0 * num_weights_nnz / num_weights)
# final points
points = vectors_to_points(photo, image, vectors)
# sanity checks
assert len(vectors) == len(points)
# re-assign clusters
clusters_points = [([], p) for p in points]
line_map_cluster = np.argmax(weights, axis=1)
for i_l, l in enumerate(all_lines):
i_c = line_map_cluster[i_l]
if i_c < len(points):
clusters_points[i_c][0].append(l)
# throw away small clusters
clusters_points = filter(
lambda x: len(x[0]) >= min_cluster_size, clusters_points)
# reverse sort by cluster length
clusters_points.sort(
key=lambda x: line_cluster_length(x[0]), reverse=True)
# split into two parallel arrays
clusters = [cp[0] for cp in clusters_points]
points = [cp[1] for cp in clusters_points]
else: # no EM
for i_v, lines in enumerate(clusters):
def objective_function(x, *args):
p = vectors_to_points(photo, image, unpack_x(x))[0]
return [line_residual(l, p) for l in lines]
from scipy.optimize import leastsq
x0 = pack_x([vectors[i_v]])
x_opt, __ = leastsq(objective_function, x0)
vectors[i_v] = unpack_x(x_opt)[0]
# delete similar vectors
cluster_merge_dot = math.cos(math.radians(20.0))
vectors_merged = []
clusters_merged = []
for i_v, v in enumerate(vectors):
if (not vectors_merged or
all(abs_dot(v, w) < cluster_merge_dot for w in vectors_merged)):
vectors_merged.append(v)
clusters_merged.append(clusters[i_v])
vectors = vectors_merged
clusters = clusters_merged
# clamp number of vectors
if len(clusters) > max_clusters:
vectors = vectors[:max_clusters]
clusters = clusters[:max_clusters]
points = vectors_to_points(photo, image, vectors)
# normalize to [0, 0], [1, 1]
clusters_normalized = [[
[l[0] / width, l[1] / height, l[2] / width, l[3] / height]
for l in lines
] for lines in clusters]
points_normalized = [
(x / width, y / height) for (x, y) in points
]
# save result
photo.vanishing_lines = json.dumps(clusters_normalized)
photo.vanishing_points = json.dumps(points_normalized)
photo.vanishing_length = sum(line_cluster_length(c)
for c in clusters_normalized)
if save:
photo.save()
