def photo_to_label_image_task(photo_id,
color_map,
attr='substance',
larger_dimension=320,
filename=None,
format=None,
next_task=None):
""" Returns a PIL image where each pixel corresponds to a label.
filename: if specified, save the result to this filename with the specified format
(instead of returning it since PIL objects often can't be pickled)
next_task: task to start when this finishes
"""
from shapes.models import MaterialShape
from shapes.utils import parse_vertices, parse_triangles
photo = Photo.objects.get(id=photo_id)
image = open_image(photo.image_orig)
w, h = image.size
if w > h:
size = (larger_dimension, larger_dimension * h / w)
else:
size = (larger_dimension * w / h, larger_dimension)
label_image = Image.new(mode='RGB', size=size, color=(0, 0, 0))
drawer = ImageDraw.Draw(label_image)
shapes = MaterialShape.objects \
.filter(**{
'photo_id': photo.id,
attr + '__isnull': False,
}) \
.filter(**MaterialShape.DEFAULT_FILTERS) \
.order_by('-area')
has_labels = False
for shape in shapes:
vertices = parse_vertices(shape.vertices)
vertices = [(int(x * size[0]), int(y * size[1]))
for (x, y) in vertices]
for tri in parse_triangles(shape.triangles):
points = [vertices[tri[t]] for t in (0, 1, 2)]
val = getattr(shape, attr + '_id')
if val in color_map:
color = color_map[val]
drawer.polygon(points, fill=color, outline=None)
has_labels = True
if not has_labels:
return None
if filename:
label_image.save(filename, format=format)
if next_task:
next_task()
else:
return label_image
def photo_to_label_image_task(photo_id, color_map, attr='substance', larger_dimension=320,
filename=None, format=None, next_task=None):
""" Returns a PIL image where each pixel corresponds to a label.
filename: if specified, save the result to this filename with the specified format
(instead of returning it since PIL objects often can't be pickled)
next_task: task to start when this finishes
"""
from shapes.models import MaterialShape
from shapes.utils import parse_vertices, parse_triangles
photo = Photo.objects.get(id=photo_id)
image = open_image(photo.image_orig)
w, h = image.size
if w > h:
size = (larger_dimension, larger_dimension * h / w)
else:
size = (larger_dimension * w / h, larger_dimension)
label_image = Image.new(mode='RGB', size=size, color=(0, 0, 0))
drawer = ImageDraw.Draw(label_image)
shapes = MaterialShape.objects \
.filter(**{
'photo_id': photo.id,
attr + '__isnull': False,
}) \
.filter(**MaterialShape.DEFAULT_FILTERS) \
.order_by('-area')
has_labels = False
for shape in shapes:
vertices = parse_vertices(shape.vertices)
vertices = [(int(x * size[0]), int(y * size[1])) for (x, y) in vertices]
for tri in parse_triangles(shape.triangles):
points = [vertices[tri[t]] for t in (0, 1, 2)]
val = getattr(shape, attr + '_id')
if val in color_map:
color = color_map[val]
drawer.polygon(points, fill=color, outline=None)
has_labels = True
if not has_labels:
return None
if filename:
label_image.save(filename, format=format)
if next_task:
next_task()
else:
return label_image
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
""" Return (uvnb, num_vanishing_points) """
print 'estimate_uvnb for shape_id: %s' % shape.id
# local import to avoid cyclic dependencies
from shapes.utils import parse_vertices, parse_triangles, \
parse_segments, bbox_vertices
# load photo
photo = shape.photo
if not photo.vanishing_lines or not photo.vanishing_points:
raise ValueError("Vanishing points not computed")
if not photo.focal_y:
raise ValueError("Photo does not have focal_y")
vlines = json.loads(photo.vanishing_lines)
vpoints = json.loads(photo.vanishing_points)
vvectors = copy.copy(photo.vanishing_vectors())
if len(vlines) != len(vpoints):
raise ValueError("Invalid vanishing points data structure")
# add any missing vanishing points
vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)
# find vanishing lines inside shape
vertices = parse_vertices(shape.vertices)
segments = parse_segments(shape.triangles)
triangles = parse_triangles(shape.triangles)
# intersect shapes with segments
counts = []
for idx in xrange(len(vlines)):
# re-pack for geom routines
query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]
from common.geom import triangles_segments_intersections_only
n = len(
triangles_segments_intersections_only(vertices, segments,
triangles, query_segments))
if n >= 5:
counts.append((n, idx))
counts.sort(key=lambda x: x[0], reverse=True)
# function to judge normals: its vanishing line can't intersect the shape.
def auto_normal_acceptable(n):
sign = None
for (x, y) in vertices:
# vanishing line
line = (n[0], n[1], n[2] * photo.focal_y)
# signed distance
d = ((x - 0.5) * photo.aspect_ratio * line[0] +
(0.5 - y) * line[1] + # flip y
line[2])
if abs(d) < 0.05:
return False
elif sign is None:
sign = (d > 0)
else:
if sign != (d > 0):
return False
return True
# find coordinate frame
best_n = None
best_u = None
method = None
num_vanishing_lines = 0
# make sure shape has label
if not shape.label_pos_x or not shape.label_pos_y:
from shapes.utils import update_shape_label_pos
update_shape_label_pos(shape)
# place label in 3D
b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
b = [(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) /
photo.focal_y, (0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
b_z]
# estimate closest human normal
human_labels = list(
ShapeRectifiedNormalLabel.objects.filter(
shape=shape,
automatic=False,
correct_score__isnull=False,
).order_by('-correct_score'))
human_labels += list(
ShapeRectifiedNormalLabel.objects.filter(shape=shape,
automatic=False,
correct_score__isnull=True))
if human_labels:
for label in human_labels:
human_u = label.u()
human_n = label.n()
b = list(label.uvnb_numpy()[0:3, 3].flat)
# find best normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
d = abs_dot(human_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'S'
# if there is a match find u and quit
if best_n is not None:
# find best u
best_u_dot = 0
best_u = None
for u in vvectors:
if abs_dot(u, best_n) < 0.1:
d = abs_dot(human_u, u)
if d > best_u_dot:
best_u_dot = d
best_u = u
break
# try using object label
if best_n is None and shape.name:
if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
best_y = 0.9
for v in vvectors[0:3]:
if abs(v[1]) > best_y:
best_y = abs(v[1])
best_n = v
method = 'O'
# try fully automatic method if human normals are not good enough
if (try_fully_automatic and best_n is None and len(vpoints) >= 3
and len(counts) >= 2 and
(shape.substance is None or shape.substance.name != 'Painted')):
# choose two dominant vanishing points
best_u = vvectors[counts[0][1]]
best_v = vvectors[counts[1][1]]
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(best_u, best_v)):
num_vanishing_lines = counts[0][0] + counts[1][0]
uv_dot = abs_dot(best_u, best_v)
print 'u dot v = %s' % uv_dot
else:
best_u, best_v = None, None
uv_dot = None
# make sure these vectors are accurate
if not uv_dot or uv_dot > 0.05:
# try and find two other orthogonal vanishing points
best_dot = 0.05
best_u = None
best_v = None
for c1, i1 in counts:
for c2, i2 in counts[:i1]:
d = abs_dot(vvectors[i1], vvectors[i2])
if d < best_dot:
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(
normalized_cross(vvectors[i1], vvectors[i2])):
best_dot = d
if c1 > c2:
best_u = vvectors[i1]
best_v = vvectors[i2]
else:
best_u = vvectors[i2]
best_v = vvectors[i1]
num_vanishing_lines = c1 + c2
if best_u is not None and best_v is not None:
best_n = normalized_cross(best_u, best_v)
method = 'A'
# give up for some classes of objects
if shape.name:
name = shape.name.name.lower()
if ((abs(best_n[1]) > 0.5
and name in ('wall', 'door', 'window'))
or (abs(best_n[1]) < 0.5
and name in ('floor', 'ceiling', 'table',
'worktop/countertop', 'carpet/rug'))):
method = best_u = best_v = best_n = None
num_vanishing_lines = 0
# for walls that touch the edge of the photo, try using bbox center as a
# vanishing point (i.e. assume top/bottom shapes are horizontal, side
# shapes are vertical)
if (try_fully_automatic and best_n is None and shape.name and
(shape.substance is None or shape.substance.name != 'Painted')):
if shape.name.name.lower() == 'wall':
bbox = bbox_vertices(parse_vertices(shape.vertices))
if ((bbox[0] < 0.05 and bbox[2] < 0.50)
or (bbox[0] > 0.50 and bbox[2] > 0.95)):
bbox_n = photo.vanishing_point_to_vector(
(0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5))
# find normal that best matches this fake bbox normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
if auto_normal_acceptable(n):
d = abs_dot(bbox_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'O'
# find best u vector if not already found
if best_n is not None and best_u is None:
# first check if any in-shape vanishing points
# are perpendicular to the normal
best_u = most_orthogonal_vector(best_n,
[vvectors[i] for __, i in counts],
tolerance_dot=0.05)
# else, find the best u from all vectors
if best_u is None:
best_u = most_orthogonal_vector(best_n, vvectors)
# failure
if best_u is None or best_n is None:
return (None, None, 0)
# ortho-normalize system
uvn = construct_uvn_frame(best_n, best_u, b, flip_to_match_image=True)
# form uvnb matrix, column major
uvnb = (uvn[0, 0], uvn[1, 0], uvn[2, 0], 0, uvn[0, 1], uvn[1, 1],
uvn[2, 1], 0, uvn[0, 2], uvn[1, 2], uvn[2,
2], 0, b[0], b[1], b[2], 1)
return uvnb, method, num_vanishing_lines
def estimate_uvnb_from_vanishing_points(shape, try_fully_automatic=False):
""" Return (uvnb, num_vanishing_points) """
print 'estimate_uvnb for shape_id: %s' % shape.id
# local import to avoid cyclic dependencies
from shapes.utils import parse_vertices, parse_triangles, \
parse_segments, bbox_vertices
# load photo
photo = shape.photo
if not photo.vanishing_lines or not photo.vanishing_points:
raise ValueError("Vanishing points not computed")
if not photo.focal_y:
raise ValueError("Photo does not have focal_y")
vlines = json.loads(photo.vanishing_lines)
vpoints = json.loads(photo.vanishing_points)
vvectors = copy.copy(photo.vanishing_vectors())
if len(vlines) != len(vpoints):
raise ValueError("Invalid vanishing points data structure")
# add any missing vanishing points
vvectors = complete_vector_triplets(vvectors, tolerance_dot=0.75)
# find vanishing lines inside shape
vertices = parse_vertices(shape.vertices)
segments = parse_segments(shape.triangles)
triangles = parse_triangles(shape.triangles)
# intersect shapes with segments
counts = []
for idx in xrange(len(vlines)):
# re-pack for geom routines
query_segments = [((l[0], l[1]), (l[2], l[3])) for l in vlines[idx]]
from common.geom import triangles_segments_intersections_only
n = len(triangles_segments_intersections_only(
vertices, segments, triangles, query_segments))
if n >= 5:
counts.append((n, idx))
counts.sort(key=lambda x: x[0], reverse=True)
# function to judge normals: its vanishing line can't intersect the shape.
def auto_normal_acceptable(n):
sign = None
for (x, y) in vertices:
# vanishing line
line = (n[0], n[1], n[2] * photo.focal_y)
# signed distance
d = (
(x - 0.5) * photo.aspect_ratio * line[0] +
(0.5 - y) * line[1] + # flip y
line[2]
)
if abs(d) < 0.05:
return False
elif sign is None:
sign = (d > 0)
else:
if sign != (d > 0):
return False
return True
# find coordinate frame
best_n = None
best_u = None
method = None
num_vanishing_lines = 0
# make sure shape has label
if not shape.label_pos_x or not shape.label_pos_y:
from shapes.utils import update_shape_label_pos
update_shape_label_pos(shape)
# place label in 3D
b_z = -(photo.focal_y / photo.aspect_ratio) / 0.1
b = [
(shape.label_pos_x - 0.5) * photo.aspect_ratio * (-b_z) / photo.focal_y,
(0.5 - shape.label_pos_y) * (-b_z) / photo.focal_y,
b_z
]
# estimate closest human normal
human_labels = list(ShapeRectifiedNormalLabel.objects.filter(
shape=shape, automatic=False, correct_score__isnull=False,
).order_by('-correct_score'))
human_labels += list(ShapeRectifiedNormalLabel.objects.filter(
shape=shape, automatic=False, correct_score__isnull=True))
if human_labels:
for label in human_labels:
human_u = label.u()
human_n = label.n()
b = list(label.uvnb_numpy()[0:3, 3].flat)
# find best normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
d = abs_dot(human_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'S'
# if there is a match find u and quit
if best_n is not None:
# find best u
best_u_dot = 0
best_u = None
for u in vvectors:
if abs_dot(u, best_n) < 0.1:
d = abs_dot(human_u, u)
if d > best_u_dot:
best_u_dot = d
best_u = u
break
# try using object label
if best_n is None and shape.name:
if shape.name.name.lower() in ('floor', 'carpet/rug', 'ceiling'):
best_y = 0.9
for v in vvectors[0:3]:
if abs(v[1]) > best_y:
best_y = abs(v[1])
best_n = v
method = 'O'
# try fully automatic method if human normals are not good enough
if (try_fully_automatic and best_n is None and len(vpoints) >= 3 and len(counts) >= 2 and
(shape.substance is None or shape.substance.name != 'Painted')):
# choose two dominant vanishing points
best_u = vvectors[counts[0][1]]
best_v = vvectors[counts[1][1]]
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(best_u, best_v)):
num_vanishing_lines = counts[0][0] + counts[1][0]
uv_dot = abs_dot(best_u, best_v)
print 'u dot v = %s' % uv_dot
else:
best_u, best_v = None, None
uv_dot = None
# make sure these vectors are accurate
if not uv_dot or uv_dot > 0.05:
# try and find two other orthogonal vanishing points
best_dot = 0.05
best_u = None
best_v = None
for c1, i1 in counts:
for c2, i2 in counts[:i1]:
d = abs_dot(vvectors[i1], vvectors[i2])
if d < best_dot:
# don't try and auto-rectify frontal surfaces
if auto_normal_acceptable(normalized_cross(
vvectors[i1], vvectors[i2])):
best_dot = d
if c1 > c2:
best_u = vvectors[i1]
best_v = vvectors[i2]
else:
best_u = vvectors[i2]
best_v = vvectors[i1]
num_vanishing_lines = c1 + c2
if best_u is not None and best_v is not None:
best_n = normalized_cross(best_u, best_v)
method = 'A'
# give up for some classes of objects
if shape.name:
name = shape.name.name.lower()
if ((abs(best_n[1]) > 0.5 and name in (
'wall', 'door', 'window')) or
(abs(best_n[1]) < 0.5 and name in (
'floor', 'ceiling', 'table',
'worktop/countertop', 'carpet/rug'))):
method = best_u = best_v = best_n = None
num_vanishing_lines = 0
# for walls that touch the edge of the photo, try using bbox center as a
# vanishing point (i.e. assume top/bottom shapes are horizontal, side
# shapes are vertical)
if (try_fully_automatic and best_n is None and shape.name and
(shape.substance is None or shape.substance.name != 'Painted')):
if shape.name.name.lower() == 'wall':
bbox = bbox_vertices(parse_vertices(shape.vertices))
if ((bbox[0] < 0.05 and bbox[2] < 0.50) or
(bbox[0] > 0.50 and bbox[2] > 0.95)):
bbox_n = photo.vanishing_point_to_vector((
0.5 + 10 * (bbox[0] + bbox[2] - 1), 0.5
))
# find normal that best matches this fake bbox normal
best_n_dot = 0.9
best_n = None
for n in vvectors:
if auto_normal_acceptable(n):
d = abs_dot(bbox_n, n)
if d > best_n_dot:
best_n_dot = d
best_n = n
method = 'O'
# find best u vector if not already found
if best_n is not None and best_u is None:
# first check if any in-shape vanishing points
# are perpendicular to the normal
best_u = most_orthogonal_vector(
best_n, [vvectors[i] for __, i in counts],
tolerance_dot=0.05)
# else, find the best u from all vectors
if best_u is None:
best_u = most_orthogonal_vector(best_n, vvectors)
# failure
if best_u is None or best_n is None:
return (None, None, 0)
# ortho-normalize system
uvn = construct_uvn_frame(
best_n, best_u, b, flip_to_match_image=True)
# form uvnb matrix, column major
uvnb = (
uvn[0, 0], uvn[1, 0], uvn[2, 0], 0,
uvn[0, 1], uvn[1, 1], uvn[2, 1], 0,
uvn[0, 2], uvn[1, 2], uvn[2, 2], 0,
b[0], b[1], b[2], 1
)
return uvnb, method, num_vanishing_lines