def _save_image(img_file, filename, format, dimensions):
import shutil
from PIL import Image
from imagekit.utils import open_image
from pilkit.utils import extension_to_format
from cnntools.common_utils import resize_mindim
# Skip if we already downloaded the image
if os.path.exists(filename):
return
parent_dir = os.path.dirname(filename)
ensuredir(parent_dir)
if not dimensions and not format:
img_file.seek(0)
with open(filename, 'wb') as f:
shutil.copyfileobj(img_file, f)
else:
if dimensions and len(dimensions) == 2:
image = open_image(img_file)
if image.size != tuple(dimensions):
image = image.resize(dimensions, Image.ANTIALIAS)
elif dimensions and len(dimensions) == 1:
# Here we specified the minimum dimension
image = open_image(img_file)
image = resize_mindim(image, dimensions[0])
else:
image = open_image(img_file)
if not format:
format = extension_to_format(os.path.splitext(filename)[1].lower())
image.save(filename, format)
def _process_content(self, filename, content):
img = open_image(content)
original_format = img.format
img = self.field.process(img, self)
# Determine the format.
format = self.field.format
if not format:
if callable(getattr(self.field, self.field._upload_to_attr)):
# The extension is explicit, so assume they want the matching format.
extension = os.path.splitext(filename)[1].lower()
# Try to guess the format from the extension.
try:
format = extension_to_format(extension)
except UnknownExtensionError:
pass
format = format or img.format or original_format or 'JPEG'
if format != 'JPEG':
imgfile = img_to_fobj(img, format)
else:
imgfile = img_to_fobj(img, format,
quality=int(self.field.quality),
optimize=True)
content = ContentFile(imgfile.read())
return img, content
def update_shape_dominant_rgb(shape, save=True):
""" Updates the dominant_* fields in a shape instance """
h = []
try:
h = get_dominant_image_colors(open_image(shape.image_crop))
except:
print 'Could not find dominant colors for: %s' % shape
traceback.print_exc()
return
if len(h) < 1:
return
shape.dominant_rgb0 = h[0]
shape.dominant_rgb1 = h[1] if len(h) > 1 else ''
shape.dominant_rgb2 = h[2] if len(h) > 2 else ''
shape.dominant_rgb3 = h[3] if len(h) > 3 else ''
shape.dominant_r = float(int(h[0][1:3], 16)) / 255.0
shape.dominant_g = float(int(h[0][3:5], 16)) / 255.0
shape.dominant_b = float(int(h[0][5:7], 16)) / 255.0
print '%s: dominant colors: %s %s %s %s' % (
shape, shape.dominant_rgb0, shape.dominant_rgb1, shape.dominant_rgb2, shape.dominant_rgb3)
if save:
shape.save()
def create_shape_image_sample(shape, sample_width=256, sample_height=256):
""" Return a centered image sample of the shape, or None if a centered
box intersects the shape edge """
from shapes.models import ShapeImageSample
from common.wavelets import compute_wavelet_feature_vector
if ShapeImageSample.objects.filter(shape=shape).exists():
return
image = open_image(shape.photo.image_2048)
image_width, image_height = image.size
triangles = parse_triangles(shape.triangles)
vertices = [(x * image_width, y * image_height)
for (x, y) in parse_vertices(shape.vertices)]
b = bbox_vertices(vertices)
# make sure box is big enough
if b[2] - b[0] < sample_width or b[3] - b[1] < sample_height:
return None
# try random boxes
filled = False
for iters in xrange(1000):
x = int(random.uniform(b[0] + sample_width / 2, b[2] - sample_width / 2))
y = int(random.uniform(b[1] + sample_height / 2, b[3] - sample_height / 2))
box = (x - sample_width / 2, y - sample_height / 2,
x + sample_width / 2, y + sample_height / 2)
box = [int(f) for f in box]
# make sure box is filled entirely
poly = Image.new(mode='1', size=(box[2] - box[0], box[3] - box[1]), color=0)
draw = ImageDraw.Draw(poly)
vertices_shifted = [(x - box[0], y - box[1]) for (x, y) in vertices]
for tri in triangles:
points = [vertices_shifted[tri[t]] for t in (0, 1, 2)]
draw.polygon(points, fill=1)
del draw
filled = True
for p in poly.getdata():
if p != 1:
filled = False
break
if filled:
break
if not filled:
return None
# create sample
sample, created = ShapeImageSample.objects.get_or_create(
shape=shape, defaults={'bbox': json.dumps(box)})
if created:
sample_image = image.crop(box)
sample.feature_vector = \
compute_wavelet_feature_vector(sample_image)
save_obj_attr_image(
sample, attr='image', img=sample_image, format='jpg', save=True)
return sample
def update_shape_dominant_rgb(shape, save=True):
""" Updates the dominant_* fields in a shape instance """
h = []
try:
h = get_dominant_image_colors(open_image(shape.image_crop))
except:
print 'Could not find dominant colors for: %s' % shape
traceback.print_exc()
return
if len(h) < 1:
return
shape.dominant_rgb0 = h[0]
shape.dominant_rgb1 = h[1] if len(h) > 1 else ''
shape.dominant_rgb2 = h[2] if len(h) > 2 else ''
shape.dominant_rgb3 = h[3] if len(h) > 3 else ''
shape.dominant_r = float(int(h[0][1:3], 16)) / 255.0
shape.dominant_g = float(int(h[0][3:5], 16)) / 255.0
shape.dominant_b = float(int(h[0][5:7], 16)) / 255.0
print '%s: dominant colors: %s %s %s %s' % (
shape, shape.dominant_rgb0, shape.dominant_rgb1, shape.dominant_rgb2, shape.dominant_rgb3)
if save:
shape.save()
def _process_content(self, filename, content):
img = open_image(content)
original_format = img.format
img = self.field.process(img, self)
# Determine the format.
format = self.field.format
if not format:
if callable(getattr(self.field, self.field._upload_to_attr)):
# The extension is explicit, so assume they want the matching format.
extension = os.path.splitext(filename)[1].lower()
# Try to guess the format from the extension.
try:
format = extension_to_format(extension)
except UnknownExtensionError:
pass
format = format or img.format or original_format or 'JPEG'
if format == 'JPEG':
img_to_fobj_kwargs = dict(quality=int(self.field.quality),
optimize=True)
else:
img_to_fobj_kwargs = {}
# Run the AutoConvert processor
if getattr(self.field, 'autoconvert', True):
autoconvert_processor = AutoConvert(format)
img = autoconvert_processor.process(img)
img_to_fobj_kwargs.update(autoconvert_processor.save_kwargs)
imgfile = img_to_fobj(img, format, **img_to_fobj_kwargs)
content = ContentFile(imgfile.read())
return img, content
def _process_content(self, filename, content):
img = open_image(content)
original_format = img.format
img = self.field.process(img, self)
# Determine the format.
format = self.field.format
if not format:
if callable(getattr(self.field, self.field._upload_to_attr)):
# The extension is explicit, so assume they want the matching format.
extension = os.path.splitext(filename)[1].lower()
# Try to guess the format from the extension.
try:
format = extension_to_format(extension)
except UnknownExtensionError:
pass
format = format or img.format or original_format or 'JPEG'
if format == 'JPEG':
img_to_fobj_kwargs = dict(quality=int(self.field.quality),
optimize=True)
else:
img_to_fobj_kwargs = {}
# Run the AutoConvert processor
if getattr(self.field, 'autoconvert', True):
autoconvert_processor = AutoConvert(format)
img = autoconvert_processor.process(img)
img_to_fobj_kwargs.update(autoconvert_processor.save_kwargs)
imgfile = img_to_fobj(img, format, **img_to_fobj_kwargs)
content = ContentFile(imgfile.read())
return img, content
def correct_segmentation_(self, photo_name, all_tasks, part=None):
tasks = all_tasks.filter(
# full body segmentation
part=part,
# at least one person thinks this is correct
responses__qualities__correct=True,
)
if not tasks:
return None
# gather all those good segmentations
segmentations = [
response for task in tasks
for response in task.responses.filter(qualities__correct=True)
]
# inform the user if we have more than one good solution
if len(segmentations) > 1:
print('The photo: {} has more than one good segmentation'.format(
photo_name))
segmentation_file = choice(segmentations).segmentation
segmentation_image = open_image(segmentation_file)
segmentation = np.asarray(segmentation_image)
segmentation_image.close()
segmentation_file.close()
return segmentation
def photo_to_label_image_task(photo_id,
color_map,
attr='substance',
larger_dimension=320,
filename=None,
format=None,
next_task=None):
""" Returns a PIL image where each pixel corresponds to a label.
filename: if specified, save the result to this filename with the specified format
(instead of returning it since PIL objects often can't be pickled)
next_task: task to start when this finishes
"""
from shapes.models import MaterialShape
from shapes.utils import parse_vertices, parse_triangles
photo = Photo.objects.get(id=photo_id)
image = open_image(photo.image_orig)
w, h = image.size
if w > h:
size = (larger_dimension, larger_dimension * h / w)
else:
size = (larger_dimension * w / h, larger_dimension)
label_image = Image.new(mode='RGB', size=size, color=(0, 0, 0))
drawer = ImageDraw.Draw(label_image)
shapes = MaterialShape.objects \
.filter(**{
'photo_id': photo.id,
attr + '__isnull': False,
}) \
.filter(**MaterialShape.DEFAULT_FILTERS) \
.order_by('-area')
has_labels = False
for shape in shapes:
vertices = parse_vertices(shape.vertices)
vertices = [(int(x * size[0]), int(y * size[1]))
for (x, y) in vertices]
for tri in parse_triangles(shape.triangles):
points = [vertices[tri[t]] for t in (0, 1, 2)]
val = getattr(shape, attr + '_id')
if val in color_map:
color = color_map[val]
drawer.polygon(points, fill=color, outline=None)
has_labels = True
if not has_labels:
return None
if filename:
label_image.save(filename, format=format)
if next_task:
next_task()
else:
return label_image
def open_image(self, width='orig'):
""" Fetch the image at a given size (see the image_<width> fields) """
cache_attr = '_cache_image_%s' % width
if hasattr(self, cache_attr):
return getattr(self, cache_attr)
pil = open_image(getattr(self, 'image_%s' % width))
setattr(self, cache_attr, pil)
return pil
def handle(self, *args, **options):
photos = Photo.objects.filter(id__in=[95686, 97532, 116625, 85877, 69122, 104870])
for p in photos:
decomp = IntrinsicImagesDecomposition.objects.get(photo_id=p.id, algorithm_id=1141)
img_i = p.open_image(width='orig')
img_r = open_image(decomp.reflectance_image)
img_s = open_image(decomp.shading_image)
if not os.path.exists('example-intrinsic-segments/%s' % p.id):
os.makedirs('example-intrinsic-segments/%s' % p.id)
img_i.save('example-intrinsic-segments/%s/image.jpg' % p.id)
img_r.save('example-intrinsic-segments/%s/reflectance.png' % p.id)
img_s.save('example-intrinsic-segments/%s/shading.png' % p.id)
for s in progress_bar(p.material_shapes.all()):
mask_complex_polygon(img_i, s.vertices, s.triangles)[0].save('example-intrinsic-segments/%s/shape-%s-image.png' % (p.id, s.id))
mask_complex_polygon(img_r, s.vertices, s.triangles)[0].save('example-intrinsic-segments/%s/shape-%s-reflectance.png' % (p.id, s.id))
mask_complex_polygon(img_s, s.vertices, s.triangles)[0].save('example-intrinsic-segments/%s/shape-%s-shading.png' % (p.id, s.id))
def fill_in_bbox_task(shape):
""" Helper to fill in the potentially empty image_bbox field """
image_bbox = mask_complex_polygon(
image=open_image(shape.photo.image_orig),
vertices=shape.vertices,
triangles=shape.triangles,
bbox_only=True)
save_obj_attr_image(shape, attr='image_bbox',
img=image_bbox,
format='jpg', save=True)
def download_photo_task(photo_id,
filename,
format=None,
larger_dimension=None):
""" Downloads a photo and stores it, potentially downsampling it and
potentially converting formats """
parent_dir = os.path.dirname(filename)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
photo = Photo.objects.get(id=photo_id)
if not larger_dimension and not format:
photo.image_orig.seek(0)
with open(filename, 'wb') as f:
shutil.copyfileobj(photo.image_orig, f)
else:
if larger_dimension <= 512:
image = open_image(photo.image_512)
elif larger_dimension <= 1024:
image = open_image(photo.image_1024)
elif larger_dimension <= 2048:
image = open_image(photo.image_2048)
else:
image = open_image(photo.image_orig)
if max(image.size) > larger_dimension:
if image.size[0] > image.size[1]:
image = image.resize(
(larger_dimension,
larger_dimension * image.size[1] / image.size[0]),
Image.ANTIALIAS)
else:
image = image.resize(
(larger_dimension * image.size[0] / image.size[1],
larger_dimension), Image.ANTIALIAS)
if not format:
format = extension_to_format(os.path.splitext(filename).lower())
image.save(filename, format)
def fill_in_bbox_task(shape):
""" Helper to fill in the potentially empty image_bbox field """
image_bbox = mask_complex_polygon(image=open_image(shape.photo.image_orig),
vertices=shape.vertices,
triangles=shape.triangles,
bbox_only=True)
save_obj_attr_image(shape,
attr='image_bbox',
img=image_bbox,
format='jpg',
save=True)
def photo_to_label_image_task(photo_id, color_map, attr='substance', larger_dimension=320,
filename=None, format=None, next_task=None):
""" Returns a PIL image where each pixel corresponds to a label.
filename: if specified, save the result to this filename with the specified format
(instead of returning it since PIL objects often can't be pickled)
next_task: task to start when this finishes
"""
from shapes.models import MaterialShape
from shapes.utils import parse_vertices, parse_triangles
photo = Photo.objects.get(id=photo_id)
image = open_image(photo.image_orig)
w, h = image.size
if w > h:
size = (larger_dimension, larger_dimension * h / w)
else:
size = (larger_dimension * w / h, larger_dimension)
label_image = Image.new(mode='RGB', size=size, color=(0, 0, 0))
drawer = ImageDraw.Draw(label_image)
shapes = MaterialShape.objects \
.filter(**{
'photo_id': photo.id,
attr + '__isnull': False,
}) \
.filter(**MaterialShape.DEFAULT_FILTERS) \
.order_by('-area')
has_labels = False
for shape in shapes:
vertices = parse_vertices(shape.vertices)
vertices = [(int(x * size[0]), int(y * size[1])) for (x, y) in vertices]
for tri in parse_triangles(shape.triangles):
points = [vertices[tri[t]] for t in (0, 1, 2)]
val = getattr(shape, attr + '_id')
if val in color_map:
color = color_map[val]
drawer.polygon(points, fill=color, outline=None)
has_labels = True
if not has_labels:
return None
if filename:
label_image.save(filename, format=format)
if next_task:
next_task()
else:
return label_image
def download_photo_task(photo_id, filename, format=None, larger_dimension=None):
""" Downloads a photo and stores it, potentially downsampling it and
potentially converting formats """
parent_dir = os.path.dirname(filename)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
photo = Photo.objects.get(id=photo_id)
if not larger_dimension and not format:
photo.image_orig.seek(0)
with open(filename, 'wb') as f:
shutil.copyfileobj(photo.image_orig, f)
else:
if larger_dimension <= 512:
image = open_image(photo.image_512)
elif larger_dimension <= 1024:
image = open_image(photo.image_1024)
elif larger_dimension <= 2048:
image = open_image(photo.image_2048)
else:
image = open_image(photo.image_orig)
if max(image.size) > larger_dimension:
if image.size[0] > image.size[1]:
image = image.resize((
larger_dimension,
larger_dimension * image.size[1] / image.size[0]), Image.ANTIALIAS)
else:
image = image.resize((
larger_dimension * image.size[0] / image.size[1],
larger_dimension), Image.ANTIALIAS)
if not format:
format = extension_to_format(os.path.splitext(filename).lower())
image.save(filename, format)
def update_shape_image_crop(shape, save=True):
""" Update the cropped image for a shape """
# compute masked image
image_crop, image_bbox = mask_complex_polygon(
image=open_image(shape.photo.image_orig),
vertices=shape.vertices,
triangles=shape.triangles)
if image_crop:
save_obj_attr_image(shape, attr='image_crop', img=image_crop, format='png', save=save)
shape.image_square_300.generate()
if image_bbox:
save_obj_attr_image(shape, attr='image_bbox', img=image_bbox, format='jpg', save=save)
shape.image_bbox_1024.generate()
def handle(self, *args, **options):
qset = PhotoWhitebalanceLabel.objects.filter(
whitebalanced=F('photo__whitebalanced')
)
out = []
for label in progress.bar(qset):
pil = open_image(label.photo.image_300)
points_list = label.points.split(',')
for idx in xrange(label.num_points):
x = float(points_list[idx * 2]) * pil.size[0]
y = float(points_list[idx * 2 + 1]) * pil.size[1]
rgb = pil.getpixel((x, y))
out.append([c / 255.0 for c in rgb])
out = np.array(out)
np.save(args[0] if args else 'whitebalance_clicks.npy', out)
def handle(self, *args, **options):
qset = PhotoWhitebalanceLabel.objects.filter(
whitebalanced=F('photo__whitebalanced')
)
out = []
for label in progress.bar(qset):
photo = label.photo.__class__.objects.get(id=label.photo.id)
pil = open_image(photo.image_300)
points_list = label.points.split(',')
for idx in xrange(label.num_points):
x = float(points_list[idx * 2]) * pil.size[0]
y = float(points_list[idx * 2 + 1]) * pil.size[1]
rgb = pil.getpixel((x, y))
out.append([c / 255.0 for c in rgb])
out = np.array(out)
np.save(args[0] if args else 'whitebalance_clicks.npy', out)
def detect_vanishing_points(photo_id, dim=800):
""" Detects vanishing points for a photo and stores it in the database in
the photo model. """
# load photo
photo = Photo.objects.get(id=photo_id)
orig_image = open_image(photo.image_2048)
old_vanishing_lines = photo.vanishing_lines
old_vanishing_points = photo.vanishing_points
old_vanishing_length = photo.vanishing_length
detect_vanishing_points_impl(photo,
ResizeToFit(dim, dim).process(orig_image),
save=False)
if old_vanishing_length > photo.vanishing_length:
photo.vanishing_lines = old_vanishing_lines
photo.vanishing_points = old_vanishing_points
photo.vanishing_length = old_vanishing_length
if photo.vanishing_length:
photo.save()
def detect_vanishing_points(photo_id, dim=800):
""" Detects vanishing points for a photo and stores it in the database in
the photo model. """
# load photo
photo = Photo.objects.get(id=photo_id)
orig_image = open_image(photo.image_2048)
old_vanishing_lines = photo.vanishing_lines
old_vanishing_points = photo.vanishing_points
old_vanishing_length = photo.vanishing_length
detect_vanishing_points_impl(
photo, ResizeToFit(dim, dim).process(orig_image),
save=False)
if old_vanishing_length > photo.vanishing_length:
photo.vanishing_lines = old_vanishing_lines
photo.vanishing_points = old_vanishing_points
photo.vanishing_length = old_vanishing_length
if photo.vanishing_length:
photo.save()
def evaluate_decomposition(decomposition_id,
delete_failed_open=False,
**evaluate_kwargs):
"""
Evaluate a decomposition and return the fields to be updated on
the IntrinsicImagesDecomposition instance.
:param delete_failed_open: if ``True``, delete decompositions that generate
an error upon opening. Warning: if there is an internet problem,
enabling this will cause otherwise valid decompositions to be deleted.
"""
from intrinsic.evaluation import evaluate_error
# fetch reflectance image
decomp = None
try:
decomp = IntrinsicImagesDecomposition.objects.get(id=decomposition_id)
reflectance_image = open_image(decomp.reflectance_image)
except KeyboardInterrupt:
return
except Exception as e:
print e
if delete_failed_open:
print "Deleting: id %s (algorithm: %s %s)" % tuple(
IntrinsicImagesDecomposition.objects.filter(
id=decomposition_id).values_list('id', 'algorithm__slug',
'algorithm_id').get())
IntrinsicImagesDecomposition.objects.filter(
id=decomposition_id).delete()
return {k: None for k in IntrinsicImagesDecomposition.ERROR_ATTRS}
# evaluate score
update_kwargs = evaluate_error(decomp.photo_id, reflectance_image,
**evaluate_kwargs)
return update_kwargs
def evaluate_decomposition(
decomposition_id, delete_failed_open=False, **evaluate_kwargs):
"""
Evaluate a decomposition and return the fields to be updated on
the IntrinsicImagesDecomposition instance.
:param delete_failed_open: if ``True``, delete decompositions that generate
an error upon opening. Warning: if there is an internet problem,
enabling this will cause otherwise valid decompositions to be deleted.
"""
from intrinsic.evaluation import evaluate_error
# fetch reflectance image
decomp = None
try:
decomp = IntrinsicImagesDecomposition.objects.get(id=decomposition_id)
reflectance_image = open_image(decomp.reflectance_image)
except KeyboardInterrupt:
return
except Exception as e:
print e
if delete_failed_open:
print "Deleting: id %s (algorithm: %s %s)" % tuple(
IntrinsicImagesDecomposition.objects
.filter(id=decomposition_id).values_list('id', 'algorithm__slug', 'algorithm_id').get()
)
IntrinsicImagesDecomposition.objects.filter(id=decomposition_id).delete()
return {
k: None for k in IntrinsicImagesDecomposition.ERROR_ATTRS
}
# evaluate score
update_kwargs = evaluate_error(decomp.photo_id, reflectance_image, **evaluate_kwargs)
return update_kwargs
def calc_person_overlay_img(task, scribbles):
parse_pose = task.parse_pose
bounding_box = task.person.bounding_box
# check if the person already has a segmentation that we can use to improve
# the quality
segmentation = None
if task.part:
tasks = task.person.segmentation_tasks.filter(
responses__qualities__correct=True, part__isnull=True)
if tasks:
response = tasks[0].responses.filter(qualities__correct=True)[0]
segmentation = open_image(response.segmentation)
else:
print('no task')
# add other segmenations if they are correct
return calc_pose_overlay_img(task.person.photo,
scribbles,
parse_pose=parse_pose,
part=task.part,
bounding_box=bounding_box,
segmentation=segmentation)
def rectify_shape_from_uvnb(shape, rectified_normal, max_dim=None):
"""
Returns the rectified PIL image
shape: MaterialShape instance
rectified_normal: ShapeRectifiedNormalLabel instance
pq: original pixel coordinates with y down
xy: centered pixel coordinates with y up
uv: in-plane coordinates (arbitrary) with y up
st: rescaled and shifted plane coordinates (fits inside [0,1]x[0,1] but
with correct aspect ratio) with y down
ij: scaled final pixel plane coordinates with y down
"""
# helper function that applies a homography
def transform(H, points):
proj = projection_function(H)
return [proj(p) for p in points]
# grab original photo info
w = shape.photo.image_orig.width
h = shape.photo.image_orig.height
focal_pixels = 0.5 * max(w, h) / math.tan(
math.radians(0.5 * shape.photo.fov))
# uvnb: [u v n b] matrix arranged in column-major order
uvnb = [float(f) for f in json.loads(rectified_normal.uvnb)]
# mapping from plane coords to image plane
M_uv_to_xy = np.matrix([
[focal_pixels, 0, 0], [0, focal_pixels, 0], [0, 0, -1]
]) * np.matrix([[uvnb[0], uvnb[4], uvnb[12]], [uvnb[1], uvnb[5], uvnb[13]],
[uvnb[2], uvnb[6], uvnb[14]]])
M_xy_to_uv = linalg.inv(M_uv_to_xy)
M_pq_to_xy = np.matrix([
[1, 0, -0.5 * w],
[0, -1, 0.5 * h],
[0, 0, 1],
])
verts_pq = [(v[0] * w, v[1] * h) for v in parse_vertices(shape.vertices)]
#print 'verts_pq:', verts_pq
# estimate rough resolution from original bbox
if not max_dim:
min_p, min_q, max_p, max_q = bbox_vertices(verts_pq)
max_dim = max(max_p - min_p, max_q - min_q)
#print 'max_dim:', max_dim
# transform
verts_xy = transform(M_pq_to_xy, verts_pq)
#print 'verts_xy:', verts_pq
verts_uv = transform(M_xy_to_uv, verts_xy)
#print 'verts_uv:', verts_uv
# compute bbox in uv plane
min_u, min_v, max_u, max_v = bbox_vertices(verts_uv)
max_uv_range = float(max(max_u - min_u, max_v - min_v))
#print 'max_uv_range:', max_uv_range
# scale so that st fits inside [0, 1] x [0, 1]
# (but with the correct aspect ratio)
M_uv_to_st = np.matrix([[1, 0, -min_u], [0, -1, max_v],
[0, 0, max_uv_range]])
verts_st = transform(M_uv_to_st, verts_uv)
#print 'verts_st:', verts_st
M_st_to_ij = np.matrix([[max_dim, 0, 0], [0, max_dim, 0], [0, 0, 1]])
verts_ij = transform(M_st_to_ij, verts_st)
#print 'verts_ij:', verts_ij
# find final bbox
min_i, min_j, max_i, max_j = bbox_vertices(verts_ij)
size = (int(math.ceil(max_i)), int(math.ceil(max_j)))
#print 'i: %s to %s, j: %s to %s' % (min_i, max_i, min_j, max_j)
#print 'size:', size
# homography for final pixels to original pixels (ij --> pq)
M_pq_to_ij = M_st_to_ij * M_uv_to_st * M_xy_to_uv * M_pq_to_xy
M_ij_to_pq = linalg.inv(M_pq_to_ij)
M_ij_to_pq /= M_ij_to_pq[2, 2] # NORMALIZE!
data = M_ij_to_pq.ravel().tolist()[0]
image = open_image(shape.photo.image_orig)
rectified = image.transform(size=size,
method=Image.PERSPECTIVE,
data=data,
resample=Image.BICUBIC)
# crop to polygon
verts_ij_normalized = [(v[0] / size[0], v[1] / size[1]) for v in verts_ij]
image_crop, image_bbox = mask_complex_polygon(rectified,
verts_ij_normalized,
shape.triangles)
return image_crop
def update_shape_image_pbox(shape, padding=0.25, save=True):
""" Update the pbox image for a shape """
# load image
photo = shape.photo.__class__.objects.get(id=shape.photo.id)
image = open_image(photo.image_orig)
w, h = image.size
# compute bbox
vertices = [(v[0] * w, v[1] * h) for v in parse_vertices(shape.vertices)]
bbox = bbox_vertices(vertices)
if (len(vertices) < 3 or bbox[0] >= bbox[2] or bbox[1] >= bbox[3]):
return None
# compute pbox
padding_x = (bbox[2] - bbox[0]) * padding
padding_y = (bbox[3] - bbox[1]) * padding
pbox = [
max(0, bbox[0] - padding_x),
max(0, bbox[1] - padding_y),
min(w, bbox[2] + padding_x),
min(h, bbox[3] + padding_y),
]
# make square
if pbox[3] - pbox[1] > pbox[2] - pbox[0]:
mid_x = 0.5 * (pbox[2] + pbox[0])
half_h = 0.5 * (pbox[3] - pbox[1])
pbox[0] = mid_x - half_h
pbox[2] = mid_x + half_h
if pbox[0] < 0:
pbox[2] = min(w, pbox[2] - pbox[0])
pbox[0] = 0
elif pbox[2] > w:
pbox[0] = max(0, pbox[0] - (pbox[2] - w))
pbox[2] = w
else:
mid_y = 0.5 * (pbox[3] + pbox[1])
half_w = 0.5 * (pbox[2] - pbox[0])
pbox[1] = mid_y - half_w
pbox[3] = mid_y + half_w
if pbox[1] < 0:
pbox[3] = min(h, pbox[3] - pbox[1])
pbox[1] = 0
elif pbox[3] > h:
pbox[1] = max(0, pbox[1] - (pbox[3] - h))
pbox[3] = h
# crop image
image_pbox = image.crop([int(v) for v in pbox])
if image_pbox:
shape.pbox = json.dumps([
float(pbox[0]) / w,
float(pbox[1]) / h,
float(pbox[2]) / w,
float(pbox[3]) / h,
])
# pbox is in units of pixels, so this gives the pixel ratio
shape.pbox_aspect_ratio = float(pbox[2] - pbox[0]) / float(pbox[3] - pbox[1])
#print 'bbox: %s, pbox: %s' % (bbox, pbox)
save_obj_attr_image(shape, attr='image_pbox',
img=image_pbox, format='jpg', save=save)
shape.image_pbox_300.generate()
shape.image_pbox_512.generate()
def rectify_shape_from_uvnb(shape, rectified_normal, max_dim=None):
"""
Returns the rectified PIL image
shape: MaterialShape instance
rectified_normal: ShapeRectifiedNormalLabel instance
pq: original pixel coordinates with y down
xy: centered pixel coordinates with y up
uv: in-plane coordinates (arbitrary) with y up
st: rescaled and shifted plane coordinates (fits inside [0,1]x[0,1] but
with correct aspect ratio) with y down
ij: scaled final pixel plane coordinates with y down
"""
# helper function that applies a homography
def transform(H, points):
proj = projection_function(H)
return [proj(p) for p in points]
# grab original photo info
w = shape.photo.image_orig.width
h = shape.photo.image_orig.height
focal_pixels = 0.5 * max(w, h) / math.tan(math.radians(
0.5 * shape.photo.fov))
# uvnb: [u v n b] matrix arranged in column-major order
uvnb = [float(f) for f in json.loads(rectified_normal.uvnb)]
# mapping from plane coords to image plane
M_uv_to_xy = np.matrix([
[focal_pixels, 0, 0],
[0, focal_pixels, 0],
[0, 0, -1]
]) * np.matrix([
[uvnb[0], uvnb[4], uvnb[12]],
[uvnb[1], uvnb[5], uvnb[13]],
[uvnb[2], uvnb[6], uvnb[14]]
])
M_xy_to_uv = linalg.inv(M_uv_to_xy)
M_pq_to_xy = np.matrix([
[1, 0, -0.5 * w],
[0, -1, 0.5 * h],
[0, 0, 1],
])
verts_pq = [(v[0] * w, v[1] * h) for v in parse_vertices(shape.vertices)]
#print 'verts_pq:', verts_pq
# estimate rough resolution from original bbox
if not max_dim:
min_p, min_q, max_p, max_q = bbox_vertices(verts_pq)
max_dim = max(max_p - min_p, max_q - min_q)
#print 'max_dim:', max_dim
# transform
verts_xy = transform(M_pq_to_xy, verts_pq)
#print 'verts_xy:', verts_pq
verts_uv = transform(M_xy_to_uv, verts_xy)
#print 'verts_uv:', verts_uv
# compute bbox in uv plane
min_u, min_v, max_u, max_v = bbox_vertices(verts_uv)
max_uv_range = float(max(max_u - min_u, max_v - min_v))
#print 'max_uv_range:', max_uv_range
# scale so that st fits inside [0, 1] x [0, 1]
# (but with the correct aspect ratio)
M_uv_to_st = np.matrix([
[1, 0, -min_u],
[0, -1, max_v],
[0, 0, max_uv_range]
])
verts_st = transform(M_uv_to_st, verts_uv)
#print 'verts_st:', verts_st
M_st_to_ij = np.matrix([
[max_dim, 0, 0],
[0, max_dim, 0],
[0, 0, 1]
])
verts_ij = transform(M_st_to_ij, verts_st)
#print 'verts_ij:', verts_ij
# find final bbox
min_i, min_j, max_i, max_j = bbox_vertices(verts_ij)
size = (int(math.ceil(max_i)), int(math.ceil(max_j)))
#print 'i: %s to %s, j: %s to %s' % (min_i, max_i, min_j, max_j)
#print 'size:', size
# homography for final pixels to original pixels (ij --> pq)
M_pq_to_ij = M_st_to_ij * M_uv_to_st * M_xy_to_uv * M_pq_to_xy
M_ij_to_pq = linalg.inv(M_pq_to_ij)
M_ij_to_pq /= M_ij_to_pq[2, 2] # NORMALIZE!
data = M_ij_to_pq.ravel().tolist()[0]
image = open_image(shape.photo.image_orig)
rectified = image.transform(size=size, method=Image.PERSPECTIVE,
data=data, resample=Image.BICUBIC)
# crop to polygon
verts_ij_normalized = [(v[0] / size[0], v[1] / size[1]) for v in verts_ij]
image_crop, image_bbox = mask_complex_polygon(
rectified, verts_ij_normalized, shape.triangles)
return image_crop
def export_dataset_photo_task(photo_id, out_dir):
photo = Photo.objects.filter(id=photo_id).select_related(
'license',
'intrinsic_points',
'intrinsic_points__opacities',
'intrinsic_comparisons'
'intrinsic_comparisons__responses',
)[0]
image_filename = '%s.png' % photo_id
json_filename = '%s.json' % photo_id
intrinsic_points = photo.intrinsic_points.all()
intrinsic_comparisons = photo.intrinsic_comparisons.all()
intrinsic_points = intrinsic_points.filter(opaque__isnull=False, )
intrinsic_comparisons = intrinsic_comparisons.filter(
point1__opaque=True,
point2__opaque=True,
darker__isnull=False,
darker__in=("1", "2", "E"),
darker_score__isnull=False,
darker_score__gt=0)
intrinsic_points = [{
'id':
p.id,
'x':
p.x,
'y':
p.y,
'sRGB':
p.sRGB,
'min_separation':
float(p.min_separation),
'opaque':
p.opaque,
'opaque_score':
p.opaque_score,
'opaque_method':
p.opaque_method,
'opaque_responses': [{
'id': r.id,
'mturk_worker_id': r.user.mturk_worker_id,
'opaque': r.opaque,
'time_ms': r.time_ms,
'time_active_ms': r.time_active_ms,
} for r in p.opacities.filter(invalid=False).order_by('id')]
} for p in intrinsic_points]
intrinsic_comparisons = [{
'id':
c.id,
'point1':
c.point1_id,
'point2':
c.point2_id,
'min_separation':
float(min(c.point1.min_separation, c.point2.min_separation)),
'darker':
c.darker,
'darker_score':
c.darker_score,
'darker_method':
c.darker_method,
'darker_responses': [{
'id': r.id,
'mturk_worker_id': r.user.mturk_worker_id,
'darker': r.darker,
'confidence': r.confidence,
'time_ms': r.time_ms,
'time_active_ms': r.time_active_ms,
} for r in c.responses.filter(invalid=False).order_by('id')]
} for c in intrinsic_comparisons]
if photo.flickr_user_id:
attribution_name = photo.flickr_user.display_name
attribution_url = photo.get_flickr_url()
else:
attribution_name = photo.attribution_name
attribution_url = photo.attribution_url
data = {
'photo': photo_id,
'image_filename': image_filename,
'aspect_ratio': photo.aspect_ratio,
'flickr_user':
photo.flickr_user.username if photo.flickr_user_id else None,
'flickr_id': photo.flickr_id,
'license': {
'name': photo.license.name,
'url': photo.license.url,
'cc': photo.license.creative_commons,
},
'light_stack': photo.light_stack_id,
'attribution_name': attribution_name,
'attribution_url': attribution_url,
'intrinsic_points': intrinsic_points,
'intrinsic_comparisons': intrinsic_comparisons,
}
if not os.path.exists(out_dir):
try:
os.makedirs(out_dir)
except:
# this could be multiple attempts to create the same directory.
# the file-write step will fail if this is a true fail.
pass
with open(os.path.join(out_dir, json_filename), 'w') as f:
f.write(json.dumps(data, indent=2, sort_keys=True))
image = ResizeToFit(512, 512).process(open_image(photo.image_orig))
image.save(os.path.join(out_dir, image_filename))
def update_shape_image_pbox(shape, padding=0.25, save=True):
""" Update the pbox image for a shape """
# load image
image = open_image(shape.photo.image_orig)
w, h = image.size
# compute bbox
vertices = [(v[0] * w, v[1] * h) for v in parse_vertices(shape.vertices)]
bbox = bbox_vertices(vertices)
if (len(vertices) < 3 or bbox[0] >= bbox[2] or bbox[1] >= bbox[3]):
return None
# compute pbox
padding_x = (bbox[2] - bbox[0]) * padding
padding_y = (bbox[3] - bbox[1]) * padding
pbox = [
max(0, bbox[0] - padding_x),
max(0, bbox[1] - padding_y),
min(w, bbox[2] + padding_x),
min(h, bbox[3] + padding_y),
]
# make square
if pbox[3] - pbox[1] > pbox[2] - pbox[0]:
mid_x = 0.5 * (pbox[2] + pbox[0])
half_h = 0.5 * (pbox[3] - pbox[1])
pbox[0] = mid_x - half_h
pbox[2] = mid_x + half_h
if pbox[0] < 0:
pbox[2] = min(w, pbox[2] - pbox[0])
pbox[0] = 0
elif pbox[2] > w:
pbox[0] = max(0, pbox[0] - (pbox[2] - w))
pbox[2] = w
else:
mid_y = 0.5 * (pbox[3] + pbox[1])
half_w = 0.5 * (pbox[2] - pbox[0])
pbox[1] = mid_y - half_w
pbox[3] = mid_y + half_w
if pbox[1] < 0:
pbox[3] = min(h, pbox[3] - pbox[1])
pbox[1] = 0
elif pbox[3] > h:
pbox[1] = max(0, pbox[1] - (pbox[3] - h))
pbox[3] = h
# crop image
image_pbox = image.crop([int(v) for v in pbox])
if image_pbox:
shape.pbox = json.dumps([
float(pbox[0]) / w,
float(pbox[1]) / h,
float(pbox[2]) / w,
float(pbox[3]) / h,
])
# pbox is in units of pixels, so this gives the pixel ratio
shape.pbox_aspect_ratio = float(pbox[2] - pbox[0]) / float(pbox[3] - pbox[1])
#print 'bbox: %s, pbox: %s' % (bbox, pbox)
save_obj_attr_image(shape, attr='image_pbox',
img=image_pbox, format='jpg', save=save)
shape.image_pbox_300.generate()
shape.image_pbox_512.generate()
def handle(self, *args, **options):
algorithm_ids = [1141, 709, 1217, 426, 522, 633]
photo_ids = IntrinsicImagesDecomposition.objects.filter(algorithm_id=1141) \
.filter(mean_sum_error__isnull=False,
photo__stylized=False,
photo__rotated=False,
photo__synthetic=False,
photo__license__publishable=True,
photo__num_intrinsic_comparisons__gte=20,
#photo__aspect_ratio__lt=1,
) \
.order_by('-photo__num_intrinsic_comparisons') \
.values_list('photo_id', flat=True)[:100]
if not os.path.exists('visual-comparison'):
os.makedirs('visual-comparison')
with open('supplemental-comparisons.tex', 'w') as f:
for photo_num, photo_id in enumerate(progress_bar(photo_ids)):
Photo.objects.get(id=photo_id).open_image(width=512).save(
'visual-comparison/photo-%s.jpg' % photo_id)
decomps = [
IntrinsicImagesDecomposition.objects.get(
algorithm_id=algorithm_id, photo_id=photo_id)
for algorithm_id in algorithm_ids
]
for d in decomps:
open_image(d.reflectance_image).save('visual-comparison/decomp-%s-r.jpg' % d.id)
open_image(d.shading_image).save('visual-comparison/decomp-%s-s.jpg' % d.id)
print >>f, """
\\begin{figure*}[tb]
\centering
\\begin{tabular}{@{}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}c@{\hskip 0.3em}}
\\gfxw{0.135}{visual-comparison/photo-%s.jpg} &
\\rotatebox{90}{\small{Reflectance $\mathbf{R}$}} &
""".strip() % photo_id
for i, d in enumerate(decomps):
print >>f, r"\gfxw{0.135}{visual-comparison/decomp-%s-r.jpg}".strip() % d.id
if i < len(decomps) - 1:
print >>f, "&"
print >>f, r"\\ & \rotatebox{90}{\small{Shading $S$}} &"
for i, d in enumerate(decomps):
print >>f, r"\gfxw{0.135}{visual-comparison/decomp-%s-s.jpg}".strip() % d.id
if i < len(decomps) - 1:
print >>f, "&"
print >>f, r"\\ & &"
for i, d in enumerate(decomps):
if i == 0:
print >>f, r'\MetricName{} = %.1f\%%' % (d.mean_error * 100.0)
else:
print >>f, r'%.1f\%%' % (d.mean_error * 100.0)
if i < len(decomps) - 1:
print >>f, "&"
print >>f, """
\\\\
Image $\\mathbf{I}$ &
&
Our algorithm &
\\cite{zhao-PAMI2012} &
\\cite{garces2012} &
\\cite{shen-CVPR2011b} &
Retinex (gray) &
Retinex (color)
\\end{tabular}
\\vspace{-6pt}
\\caption{\\new{%%
Visual comparison of our algorithm against several recent open-source
algorithms. Each algorithm uses the best parameters found from training
(i.e., minimizes mean \\MetricNameDelta{} across all photos).
OpenSurfaces Photo ID: %s.
}}
\\label{fig:visual-comparison-%s}
\\vspace{-6pt}
\\end{figure*}
""".strip() % (photo_id, photo_id)
if (photo_num + 1) % 3 == 0:
print >>f, r"\clearpage"
