def auto_rectify_shape(shape_id): """ Attempt to automatically rectify the shape based on vanishing points """ if ShapeRectifiedNormalLabel.objects.filter(shape_id=shape_id, automatic=True).exists(): print "shape already automatically rectified" return shape = MaterialShape.objects.get(id=shape_id) from normals.perspective import estimate_uvnb_from_vanishing_points uvnb, method, num_vanishing_lines = estimate_uvnb_from_vanishing_points( shape) if not uvnb: print "Could not estimate uvnb matrix" return admin_user = User.objects.get_or_create(username='******')[0].get_profile() print 'method: %s, uvnb: %s' % (method, uvnb) obj = ShapeRectifiedNormalLabel.objects.create( user=admin_user, shape=shape, uvnb=json.dumps(uvnb), automatic=True, method=method, num_vanishing_lines=num_vanishing_lines) from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay([get_content_tuple(obj)])
def auto_rectify_shape(shape_id): """ Attempt to automatically rectify the shape based on vanishing points """ if ShapeRectifiedNormalLabel.objects.filter(shape_id=shape_id, automatic=True).exists(): print "shape already automatically rectified" return shape = MaterialShape.objects.get(id=shape_id) from normals.perspective import estimate_uvnb_from_vanishing_points uvnb, method, num_vanishing_lines = estimate_uvnb_from_vanishing_points( shape) if not uvnb: print "Could not estimate uvnb matrix" return admin_user = User.objects.get_or_create( username='******')[0].get_profile() print 'method: %s, uvnb: %s' % (method, uvnb) obj = ShapeRectifiedNormalLabel.objects.create( user=admin_user, shape=shape, uvnb=json.dumps(uvnb), automatic=True, method=method, num_vanishing_lines=num_vanishing_lines ) from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay([get_content_tuple(obj)])
def add_photo(path, must_have_fov=False, must_have_exif=False, **args): """ Add a photo to the database """ if not os.path.exists(path): raise ValueError("File does not exist") if 'license' not in args: args['license'] = License.objects.get_or_create( name='All Rights Reserved')[0] # md5: check for duplicates md5 = md5sum(path) duplicate = True try: photo = Photo.objects.get(md5=md5) except Photo.DoesNotExist: duplicate = False except Photo.MultipleObjectsReturned: duplicates = Photo.objects.filter(md5=md5).order_by('id') for d in duplicates[1:]: d.delete() if duplicate: raise ValueError("Duplicate photo import: '%s'" % path) # parse exif if 'exif' not in args: print 'Obtaining EXIF...' exif = get_exif(path) if exif: args['exif'] = exif elif must_have_exif: raise ValueError("Photo has no EXIF: %s" % path) if 'fov' not in args: print 'Obtaining FOV...' fov = get_fov(args['exif']) if fov: args['fov'] = fov elif must_have_fov: raise ValueError("Could not obtain photo FOV: %s" % path) photo = None # use a transaction so that it is only committed to the database # after save() returns. otherwise, there's a small time betwee # when the photo is added and it has an image attached. with transaction.atomic(): with open(path, 'rb') as f: print 'Uploading photo...' photo = Photo.objects.create(image_orig=ImageFile(f), md5=md5, **args) from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay([get_content_tuple(photo)]) return photo
def add_photo(path, must_have_fov=False, must_have_exif=False, **args): """ Add a photo to the database """ if not os.path.exists(path): raise ValueError("File does not exist") if "license" not in args: args["license"] = License.objects.get_or_create(name="All Rights Reserved")[0] # md5: check for duplicates md5 = md5sum(path) duplicate = True try: photo = Photo.objects.get(md5=md5) except Photo.DoesNotExist: duplicate = False except Photo.MultipleObjectsReturned: duplicates = Photo.objects.filter(md5=md5).order_by("id") for d in duplicates[1:]: d.delete() if duplicate: raise ValueError("Duplicate photo import: '%s'" % path) # parse exif if "exif" not in args: print "Obtaining EXIF..." exif = get_exif(path) if exif: args["exif"] = exif elif must_have_exif: raise ValueError("Photo has no EXIF: %s" % path) if "fov" not in args: print "Obtaining FOV..." fov = get_fov(args["exif"]) if fov: args["fov"] = fov elif must_have_fov: raise ValueError("Could not obtain photo FOV: %s" % path) photo = None # use a transaction so that it is only committed to the database # after save() returns. otherwise, there's a small time betwee # when the photo is added and it has an image attached. with transaction.atomic(): with open(path, "rb") as f: print "Uploading photo..." photo = Photo.objects.create(image_orig=ImageFile(f), md5=md5, **args) from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay([get_content_tuple(photo)]) return photo
def triangulate_submitted_shapes_impl(photo, user, mturk_assignment, shape_model, submitted_shapes): if not submitted_shapes: return if not os.path.isfile(settings.TRIANGULATE_BIN): raise RuntimeError( "ERROR: '%s' (settings.TRIANGULATE_BIN) does not exist -- " "check that it is compiled" % settings.TRIANGULATE_BIN) input_lines = [ ('%s ' % s.id) + ' '.join(filter(None, s.vertices.split(','))) for s in submitted_shapes ] input_txt = '\n'.join(input_lines) + '\nEND' process = None try: process = subprocess.Popen(args=settings.TRIANGULATE_BIN, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) output_txt, errors_txt = process.communicate(input_txt) except: if process: process.kill() process.wait() raise if not output_txt: raise ValueError( "Error with triangulate. Bin:%s\nInput:\n%s\n\nOutput:\n%s\n\nErrors:\n%s" % (settings.TRIANGULATE_BIN, input_txt, output_txt, errors_txt)) if errors_txt: print errors_txt #print("Bin:%s\nInput:\n%s\n\nOutput:\n%s\n\nErrors:\n%s" % ( #settings.TRIANGULATE_BIN, input_txt, output_txt, errors_txt)) new_content_tuples = [] output_lines = output_txt.split('\n') with transaction.atomic(): for line in output_lines: line = line.strip() if not line: continue fields = line.split('|') if len(fields) != 4: raise ValueError("Invalid output: %s" % repr(output_txt)) ids = [int(f) for f in filter(None, fields[0].split(' '))] if not ids: print 'Discarding shape not contained in input' continue verts, tris, segs = [ ','.join(filter(None, f.split(' '))) for f in fields[1:4] ] # compute polygon area and discard small polygons area = complex_polygon_area(verts, tris) # 0.0002 is roughly a 32x32 patch for a 2400x2400 image if area < 0.0001: print 'Discarding: verts: "%s", tris: "%s", segs: "%s", area: %s' % ( verts, tris, segs, area) continue # convert area to pixels pixel_area = area * photo.image_orig.width * \ photo.image_orig.height # extract segmentation times time_ms_list = [] ss_list = [] for ss in submitted_shapes: if int(ss.id) in ids: ss_list.append(ss) time_ms_list.append(ss.time_ms) if not ss_list or not time_ms_list: print 'Discarding shape not mapping to input shapes' # use the average time of the submitted shapes time_ms = sum(time_ms_list) / float(len(time_ms_list)) # auto-grant high quality for users with qualifications quality_method = None correct = None if pixel_area >= 12000: from mturk.models import MtQualificationAssignment try: correct = bool( MtQualificationAssignment.objects.get( worker=user, qualification__slug="mat_seg").value) if correct: quality_method = 'Q' except MtQualificationAssignment.DoesNotExist: correct = False new_obj, created = shape_model.objects.get_or_create( photo=photo, user=user, mturk_assignment=mturk_assignment, vertices=verts, triangles=tris, segments=segs, area=area, pixel_area=pixel_area, time_ms=time_ms, defaults={ 'added': ss_list[0].added, 'correct': correct, 'quality_method': quality_method, }) if created: for ss in ss_list: new_obj.submitted_shapes.add(ss) new_content_tuples.append(get_content_tuple(new_obj)) # these are created outside of the mturk view response, so we need to # manually add them to the pending objects queue # (imported here to avoid circular imports) for (ct_id, obj_id) in new_content_tuples: mturk_assignment.submitted_contents.get_or_create( content_type=ContentType.objects.get_for_id(ct_id), object_id=obj_id, ) # update photo shape count synchronously from photos.tasks import update_photos_num_shapes update_photos_num_shapes([photo.id]) new_content_tuples.append(get_content_tuple(photo)) # new pending objects from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay(new_content_tuples)
def triangulate_submitted_shapes_impl( photo, user, mturk_assignment, shape_model, submitted_shapes): if not submitted_shapes: return if not os.path.isfile(settings.TRIANGULATE_BIN): raise RuntimeError("ERROR: '%s' (settings.TRIANGULATE_BIN) does not exist -- " "check that it is compiled" % settings.TRIANGULATE_BIN) input_lines = [('%s ' % s.id) + ' '.join( filter(None, s.vertices.split(','))) for s in submitted_shapes] input_txt = '\n'.join(input_lines) + '\nEND' process = None try: process = subprocess.Popen( args=settings.TRIANGULATE_BIN, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) output_txt, errors_txt = process.communicate(input_txt) except: if process: process.kill() process.wait() raise if not output_txt: raise ValueError( "Error with triangulate. Bin:%s\nInput:\n%s\n\nOutput:\n%s\n\nErrors:\n%s" % ( settings.TRIANGULATE_BIN, input_txt, output_txt, errors_txt) ) if errors_txt: print errors_txt #print("Bin:%s\nInput:\n%s\n\nOutput:\n%s\n\nErrors:\n%s" % ( #settings.TRIANGULATE_BIN, input_txt, output_txt, errors_txt)) new_content_tuples = [] output_lines = output_txt.split('\n') with transaction.atomic(): for line in output_lines: line = line.strip() if not line: continue fields = line.split('|') if len(fields) != 4: raise ValueError("Invalid output: %s" % repr(output_txt)) ids = [int(f) for f in filter(None, fields[0].split(' '))] if not ids: print 'Discarding shape not contained in input' continue verts, tris, segs = [','.join(filter(None, f.split(' '))) for f in fields[1:4]] # compute polygon area and discard small polygons area = complex_polygon_area(verts, tris) # 0.0002 is roughly a 32x32 patch for a 2400x2400 image if area < 0.0001: print 'Discarding: verts: "%s", tris: "%s", segs: "%s", area: %s' % ( verts, tris, segs, area) continue # convert area to pixels pixel_area = area * photo.image_orig.width * \ photo.image_orig.height # extract segmentation times time_ms_list = [] ss_list = [] for ss in submitted_shapes: if int(ss.id) in ids: ss_list.append(ss) time_ms_list.append(ss.time_ms) if not ss_list or not time_ms_list: print 'Discarding shape not mapping to input shapes' # use the average time of the submitted shapes time_ms = sum(time_ms_list) / float(len(time_ms_list)) # auto-grant high quality for users with qualifications quality_method = None correct = None if pixel_area >= 12000: from mturk.models import MtQualificationAssignment try: correct = bool(MtQualificationAssignment.objects.get( worker=user, qualification__slug="mat_seg").value) if correct: quality_method = 'Q' except MtQualificationAssignment.DoesNotExist: correct = False new_obj, created = shape_model.objects.get_or_create( photo=photo, user=user, mturk_assignment=mturk_assignment, vertices=verts, triangles=tris, segments=segs, area=area, pixel_area=pixel_area, time_ms=time_ms, defaults={ 'added': ss_list[0].added, 'correct': correct, 'quality_method': quality_method, } ) if created: for ss in ss_list: new_obj.submitted_shapes.add(ss) new_content_tuples.append(get_content_tuple(new_obj)) # these are created outside of the mturk view response, so we need to # manually add them to the pending objects queue # (imported here to avoid circular imports) for (ct_id, obj_id) in new_content_tuples: mturk_assignment.submitted_contents.get_or_create( content_type=ContentType.objects.get_for_id(ct_id), object_id=obj_id, ) # update photo shape count synchronously from photos.tasks import update_photos_num_shapes update_photos_num_shapes([photo.id]) new_content_tuples.append(get_content_tuple(photo)) # new pending objects from mturk.tasks import add_pending_objects_task add_pending_objects_task.delay(new_content_tuples)