def geo_rescore(pid, model, method):
"""Apply geographic rescoring."""
logging.info(str((pid, model, method)))
session = SESSION()
try:
numpy.seterr(all='raise')
session.query(Model) \
.filter_by(filename=model) \
.one()
nms_method = scores.METHODS[method]
# pylint: disable-msg=E1101
detections = session.query(Detection) \
.join(Model) \
.filter(Detection.pid == pid) \
.filter(Model.filename == model) \
.filter(or_(*[m == None for m in nms_method.inputs]))
# pylint: enable-msg=E1101
nms_method = scores.METHODS[method]
for method_input in nms_method.inputs:
score_name = str(method_input).split('.')[-1]
score = scores.SCORES[score_name]
if score.compute is None:
continue
for detection in detections:
value = score.compute(session, detection)
existing = getattr(detection, score_name)
if existing is not None:
if not math.fabs(existing - value) < 1e-8:
assert False
setattr(detection, score_name, value)
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
return pid
def geo_rescore(pid, model, method):
"""Apply geographic rescoring."""
logging.info(str((pid, model, method)))
session = SESSION()
try:
numpy.seterr(all='raise')
session.query(Model) \
.filter_by(filename=model) \
.one()
nms_method = scores.METHODS[method]
# pylint: disable-msg=E1101
detections = session.query(Detection) \
.join(Model) \
.filter(Detection.pid == pid) \
.filter(Model.filename == model) \
.filter(or_(*[m == None for m in nms_method.inputs]))
# pylint: enable-msg=E1101
nms_method = scores.METHODS[method]
for method_input in nms_method.inputs:
score_name = str(method_input).split('.')[-1]
score = scores.SCORES[score_name]
if score.compute is None:
continue
for detection in detections:
value = score.compute(session, detection)
existing = getattr(detection, score_name)
if existing is not None:
if not math.fabs(existing - value) < 1e-8:
assert False
setattr(detection, score_name, value)
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
return pid
def test(model, remote, methods, dataset_id):
"""Executes the testing protocol."""
session = SESSION()
try:
test_set = session.query(Photo) \
.filter_by(test=True, dataset_id=dataset_id)
session.query(Model) \
.filter_by(filename=model) \
.one()
for photo in test_set:
logging.info(photo.id)
celery_list = [detect.s(photo.id, model)]
for method in methods:
celery_list += [geo_rescore.s(model, method)]
for method in methods:
celery_list += [nms.s(model, method)]
celery_task = celery.chain(celery_list)
if remote:
celery_task.apply_async()
else:
celery_task.apply()
except:
session.rollback()
raise
finally:
session.close()
def test(model, remote, methods, dataset_id):
"""Executes the testing protocol."""
session = SESSION()
try:
test_set = session.query(Photo) \
.filter_by(test=True, dataset_id=dataset_id)
session.query(Model) \
.filter_by(filename=model) \
.one()
for photo in test_set:
logging.info(photo.id)
celery_list = [detect.s(photo.id, model)]
for method in methods:
celery_list += [geo_rescore.s(model, method)]
for method in methods:
celery_list += [nms.s(model, method)]
celery_task = celery.chain(celery_list)
if remote:
celery_task.apply_async()
else:
celery_task.apply()
except:
session.rollback()
raise
finally:
session.close()
def nms(pid, model, method):
"""Preforms NMS on detections."""
session = SESSION()
logging.info((pid, model, method))
try:
scoring_method = scores.METHODS[method]
set_nms = str(scoring_method.output).split('.')[-1]
# pylint: disable-msg=E1101
mid, = session.query(Model.id) \
.filter_by(filename=model) \
.one()
todo, = session.query(func.count(Detection.id)) \
.filter(Detection.pid == pid) \
.filter(or_(*[m == None for m in scoring_method.inputs])) \
.filter(Detection.mid == mid) \
.one()
# pylint: enable-msg=E1101
if todo > 0:
raise Exception('Some input was not yet computed')
while True:
# pylint: disable-msg=E1101
result = session.query(Detection) \
.filter(Detection.pid == pid) \
.filter(scoring_method.output == None) \
.filter(Detection.mid == mid)
# pylint: enable-msg=E1101
result = result \
.order_by(desc(scoring_method.score)) \
.first()
if result is None:
break
setattr(result, set_nms, True)
overlap = query_utils.overlap(result, Detection)
covered = overlap > 0.3
# pylint: disable-msg=E1101
blacklist = session.query(Detection) \
.filter(Detection.pid == pid) \
.filter(scoring_method.output == None) \
.filter(Detection.mid == mid) \
.filter(covered)
# pylint: enable-msg=E1101
for elt in blacklist:
setattr(elt, set_nms, False)
session.commit()
return pid
except Exception:
session.rollback()
raise
finally:
session.close()
def detect(pid, model_filename):
"""Runs DPM and computes 3D pose."""
logger = logging.getLogger('detect')
logger.info((pid, model_filename))
session = SESSION()
try:
# pylint: disable-msg=E1101
num_detections, = session.query(func.count(Detection.id)) \
.join(Model) \
.filter(Detection.pid == pid) \
.filter(Model.filename == model_filename) \
.one()
if num_detections > 0:
logger.info('Already computed')
return pid
model = session.query(Model) \
.filter_by(filename=model_filename) \
.one()
photo = session.query(Photo) \
.options(joinedload('dataset')) \
.filter_by(id=pid) \
.one()
vehicle_types = session.query(VehicleType) \
.filter(VehicleType.id.in_([202, 8, 150, 63, 123, 16]))
pydro_model = pydro.io.LoadModel(model.filename)
image = scipy.misc.imread(
os.path.join(IMAGE_DIR, photo.filename))
pyramid = pydro.features.BuildPyramid(image, model=pydro_model)
filtered_model = pydro_model.Filter(pyramid)
parse_trees = list(filtered_model.Parse(model.thresh))
# make sure we use at least one entry so we know we tried
if len(parse_trees) == 0:
parse_trees = list(
itertools.islice(filtered_model.Parse(-numpy.inf), 1))
assert len(parse_trees) > 0
bbox_tuple = namedtuple('bbox_tuple', 'x1,x2,y1,y2')
for tree in parse_trees:
bbox = bbox_tuple(
x1=tree.x1 / image.shape[1],
x2=tree.x2 / image.shape[1],
y1=tree.y1 / image.shape[0],
y2=tree.y2 / image.shape[0],
)
score = tree.s
angle = tree.child.rule.metadata.get('angle', None)
if bbox.x1 > bbox.x2 or bbox.y1 > bbox.y2:
continue
car_pose_generator = compute_car_pose(
photo,
bbox,
angle,
vehicle_types
)
for lla, geom, vehicle_type, world_angle in car_pose_generator:
det = Detection(
photo=photo,
x1=float(bbox.x1),
y1=float(bbox.y1),
x2=float(bbox.x2),
y2=float(bbox.y2),
score=float(score),
prob=float(
1.0 / (1.0 + math.exp(model.a * score + model.b))),
model=model,
angle=angle,
lla=lla,
geom=geom,
world_angle=float(world_angle),
vehicle_type=vehicle_type,
)
session.add(det)
session.commit()
return pid
except Exception:
session.rollback()
raise
finally:
session.close()
def select_evaluation():
"""Selects the vehicles that will appear in NYC3DCars."""
nyc3dcars_session = SESSION()
labeler_session = labeler.SESSION()
try:
# Reset photos
photos = nyc3dcars_session.query(Photo) \
.options(joinedload('dataset'))
for photo in photos:
photo.daynight = None
# Reset vehicles
vehicles = nyc3dcars_session.query(Vehicle)
for vehicle in vehicles:
nyc3dcars_session.delete(vehicle)
# Turn photos back on if they have at least 1 final revision
# pylint: disable-msg=E1101
photos = labeler_session.query(labeler.Photo) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.options(joinedload('daynights')) \
.options(joinedload('annotations.flags')) \
.options(joinedload('annotations')) \
.filter(labeler.Revision.final == True) \
.filter(labeler.User.trust == True) \
.distinct()
# pylint: enable-msg=E1101
photos = list(photos)
num_photos = len(photos)
num_test = 0
num_train = 0
num_flagged = 0
print('Checking for new photos')
for labeler_photo in photos:
# Do not consider photos that have been flagged
num_flags = sum(
len(annotation.flags)
for annotation in labeler_photo.annotations
if annotation.user.trust
)
if num_flags > 0:
num_flagged += 1
continue
days = 0
nights = 0
for daynight in labeler_photo.daynights:
if not daynight.user.trust:
continue
if daynight.daynight == 'day':
days += 1
else:
nights += 1
if days + nights == 0:
print('Need Day/Night for photo: %d' % labeler_photo.id)
continue
nyc3dcars_photo = nyc3dcars_session.query(Photo) \
.filter_by(id=labeler_photo.id) \
.one()
if nyc3dcars_photo.test == True:
num_test += 1
elif nyc3dcars_photo.test == False:
num_train += 1
else:
if num_train > num_test:
print('Test: %d' % labeler_photo.id)
nyc3dcars_photo.test = True
num_test += 1
else:
print('Train: %d' % labeler_photo.id)
nyc3dcars_photo.test = False
num_train += 1
if days > nights:
nyc3dcars_photo.daynight = 'day'
else:
nyc3dcars_photo.daynight = 'night'
print('New photos done')
print('%d photos' % num_photos)
print('%d flagged' % num_flagged)
print('%d test' % num_test)
print('%d train' % num_train)
# pylint: disable-msg=E1101
good_pids = nyc3dcars_session.query(Photo.id) \
.filter(Photo.test != None) \
.all()
# pylint: enable-msg=E1101
# get photos with 1 and 2 users
# pylint: disable-msg=E1101
photos_one_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) == 1)
photos_two_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) == 2)
photos_more_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) > 2)
for photo in photos_more_user:
print(photo.id)
for photo, in photos_one_user:
vehicles = labeler_session.query(labeler.Vehicle) \
.select_from(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.options(joinedload('revision')) \
.options(joinedload('revision.annotation')) \
.options(joinedload('revision.annotation.user')) \
.options(joinedload('occlusionrankings')) \
.options(joinedload('occlusionrankings.occlusion_session')) \
.options(joinedload('bbox_sessions')) \
.filter(labeler.User.trust == True) \
.filter(labeler.Annotation.pid == photo) \
.filter(labeler.Revision.final == True) \
.distinct()
for vehicle in vehicles:
convert_vehicle(nyc3dcars_session, vehicle)
# get good vehicles for 2 user case
for photo, in photos_two_user:
annotations = labeler_session.query(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Annotation.pid == photo) \
.filter(labeler.Revision.final == True) \
.all()
assert len(annotations) == 2
vehicles1 = labeler_session.query(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.aid == annotations[0].id) \
.filter(labeler.Revision.final == True) \
.all()
vehicles2 = labeler_session.query(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.aid == annotations[1].id) \
.filter(labeler.Revision.final == True) \
.all()
if len(vehicles1) > len(vehicles2):
vehicles = vehicles1
else:
vehicles = vehicles2
for vehicle in vehicles:
print(vehicle.id)
convert_vehicle(nyc3dcars_session, vehicle)
num_vehicles, = nyc3dcars_session.query(
func.count(Vehicle.id)) \
.one()
photo_test, = nyc3dcars_session.query(
func.count(Photo.id)) \
.filter(Photo.test == True) \
.one()
photo_train, = nyc3dcars_session.query(
func.count(Photo.id)) \
.filter(Photo.test == False) \
.one()
# pylint: enable-msg=E1101
print('%d vehicles in dataset' % num_vehicles)
print('%d images for training' % photo_train)
print('%d images for testing' % photo_test)
nyc3dcars_session.commit()
except:
nyc3dcars_session.rollback()
labeler_session.rollback()
raise
finally:
nyc3dcars_session.close()
labeler_session.close()
def nms(pid, model, method):
"""Preforms NMS on detections."""
session = SESSION()
logging.info((pid, model, method))
try:
scoring_method = scores.METHODS[method]
set_nms = str(scoring_method.output).split('.')[-1]
# pylint: disable-msg=E1101
mid, = session.query(Model.id) \
.filter_by(filename=model) \
.one()
todo, = session.query(func.count(Detection.id)) \
.filter(Detection.pid == pid) \
.filter(or_(*[m == None for m in scoring_method.inputs])) \
.filter(Detection.mid == mid) \
.one()
# pylint: enable-msg=E1101
if todo > 0:
raise Exception('Some input was not yet computed')
while True:
# pylint: disable-msg=E1101
result = session.query(Detection) \
.filter(Detection.pid == pid) \
.filter(scoring_method.output == None) \
.filter(Detection.mid == mid)
# pylint: enable-msg=E1101
result = result \
.order_by(desc(scoring_method.score)) \
.first()
if result is None:
break
setattr(result, set_nms, True)
overlap = query_utils.overlap(result, Detection)
covered = overlap > 0.3
# pylint: disable-msg=E1101
blacklist = session.query(Detection) \
.filter(Detection.pid == pid) \
.filter(scoring_method.output == None) \
.filter(Detection.mid == mid) \
.filter(covered)
# pylint: enable-msg=E1101
for elt in blacklist:
setattr(elt, set_nms, False)
session.commit()
return pid
except Exception:
session.rollback()
raise
finally:
session.close()
def select_evaluation():
"""Selects the vehicles that will appear in NYC3DCars."""
nyc3dcars_session = SESSION()
labeler_session = labeler.SESSION()
try:
# Reset photos
photos = nyc3dcars_session.query(Photo) \
.options(joinedload('dataset'))
for photo in photos:
photo.daynight = None
# Reset vehicles
vehicles = nyc3dcars_session.query(Vehicle)
for vehicle in vehicles:
nyc3dcars_session.delete(vehicle)
# Turn photos back on if they have at least 1 final revision
# pylint: disable-msg=E1101
photos = labeler_session.query(labeler.Photo) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.options(joinedload('daynights')) \
.options(joinedload('annotations.flags')) \
.options(joinedload('annotations')) \
.filter(labeler.Revision.final == True) \
.filter(labeler.User.trust == True) \
.distinct()
# pylint: enable-msg=E1101
photos = list(photos)
num_photos = len(photos)
num_test = 0
num_train = 0
num_flagged = 0
print('Checking for new photos')
for labeler_photo in photos:
# Do not consider photos that have been flagged
num_flags = sum(
len(annotation.flags)
for annotation in labeler_photo.annotations
if annotation.user.trust)
if num_flags > 0:
num_flagged += 1
continue
days = 0
nights = 0
for daynight in labeler_photo.daynights:
if not daynight.user.trust:
continue
if daynight.daynight == 'day':
days += 1
else:
nights += 1
if days + nights == 0:
print('Need Day/Night for photo: %d' % labeler_photo.id)
continue
nyc3dcars_photo = nyc3dcars_session.query(Photo) \
.filter_by(id=labeler_photo.id) \
.one()
if nyc3dcars_photo.test == True:
num_test += 1
elif nyc3dcars_photo.test == False:
num_train += 1
else:
if num_train > num_test:
print('Test: %d' % labeler_photo.id)
nyc3dcars_photo.test = True
num_test += 1
else:
print('Train: %d' % labeler_photo.id)
nyc3dcars_photo.test = False
num_train += 1
if days > nights:
nyc3dcars_photo.daynight = 'day'
else:
nyc3dcars_photo.daynight = 'night'
print('New photos done')
print('%d photos' % num_photos)
print('%d flagged' % num_flagged)
print('%d test' % num_test)
print('%d train' % num_train)
# pylint: disable-msg=E1101
good_pids = nyc3dcars_session.query(Photo.id) \
.filter(Photo.test != None) \
.all()
# pylint: enable-msg=E1101
# get photos with 1 and 2 users
# pylint: disable-msg=E1101
photos_one_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) == 1)
photos_two_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) == 2)
photos_more_user = labeler_session.query(labeler.Photo.id) \
.select_from(labeler.Photo) \
.join(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.final == True) \
.filter(labeler.Photo.id.in_(good_pids)) \
.group_by(labeler.Photo.id) \
.having(func.count(labeler.Revision.id) > 2)
for photo in photos_more_user:
print(photo.id)
for photo, in photos_one_user:
vehicles = labeler_session.query(labeler.Vehicle) \
.select_from(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.options(joinedload('revision')) \
.options(joinedload('revision.annotation')) \
.options(joinedload('revision.annotation.user')) \
.options(joinedload('occlusionrankings')) \
.options(joinedload('occlusionrankings.occlusion_session')) \
.options(joinedload('bbox_sessions')) \
.filter(labeler.User.trust == True) \
.filter(labeler.Annotation.pid == photo) \
.filter(labeler.Revision.final == True) \
.distinct()
for vehicle in vehicles:
convert_vehicle(nyc3dcars_session, vehicle)
# get good vehicles for 2 user case
for photo, in photos_two_user:
annotations = labeler_session.query(labeler.Annotation) \
.join(labeler.User) \
.join(labeler.Revision) \
.filter(labeler.User.trust == True) \
.filter(labeler.Annotation.pid == photo) \
.filter(labeler.Revision.final == True) \
.all()
assert len(annotations) == 2
vehicles1 = labeler_session.query(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.aid == annotations[0].id) \
.filter(labeler.Revision.final == True) \
.all()
vehicles2 = labeler_session.query(labeler.Vehicle) \
.join(labeler.Revision) \
.join(labeler.Annotation) \
.join(labeler.User) \
.filter(labeler.User.trust == True) \
.filter(labeler.Revision.aid == annotations[1].id) \
.filter(labeler.Revision.final == True) \
.all()
if len(vehicles1) > len(vehicles2):
vehicles = vehicles1
else:
vehicles = vehicles2
for vehicle in vehicles:
print(vehicle.id)
convert_vehicle(nyc3dcars_session, vehicle)
num_vehicles, = nyc3dcars_session.query(
func.count(Vehicle.id)) \
.one()
photo_test, = nyc3dcars_session.query(
func.count(Photo.id)) \
.filter(Photo.test == True) \
.one()
photo_train, = nyc3dcars_session.query(
func.count(Photo.id)) \
.filter(Photo.test == False) \
.one()
# pylint: enable-msg=E1101
print('%d vehicles in dataset' % num_vehicles)
print('%d images for training' % photo_train)
print('%d images for testing' % photo_test)
nyc3dcars_session.commit()
except:
nyc3dcars_session.rollback()
labeler_session.rollback()
raise
finally:
nyc3dcars_session.close()
labeler_session.close()