def on_notification(self, msg):
game_time = msg.timestamp.coordinates[0]
if not self._last_notification:
self._last_notification = game_time
return
else:
self._sim_interval = (game_time - self._last_notification)
self._last_notification = game_time
while len(self._detector_start_end_times) > 0:
(end_time, start_time) = self._detector_start_end_times[0]
# We can compute mAP if the endtime is not greater than the ground
# time.
if end_time <= game_time:
# This is the closest ground bounding box to the end time.
heapq.heappop(self._detector_start_end_times)
end_bboxes = self.__get_ground_obstacles_at(end_time)
if self._flags.detection_eval_use_accuracy_model:
# Not using the detector's outputs => get ground bboxes.
start_bboxes = self.__get_ground_obstacles_at(start_time)
if (len(start_bboxes) > 0 or len(end_bboxes) > 0):
precisions = []
for iou in self._iou_thresholds:
(precision, _) = get_precision_recall_at_iou(
end_bboxes, start_bboxes, iou)
precisions.append(precision)
avg_precision = (float(sum(precisions)) /
len(precisions))
self._logger.info('precision-IoU is: {}'.format(
avg_precision))
self._csv_logger.info('{},{},{},{}'.format(
time_epoch_ms(), self.name, 'precision-IoU',
avg_precision))
else:
# Get detector output obstacles.
det_objs = self.__get_obstacles_at(start_time)
if (len(det_objs) > 0 or len(end_bboxes) > 0):
mAP = get_pedestrian_mAP(end_bboxes, det_objs)
self._logger.info('mAP is: {}'.format(mAP))
self._csv_logger.info('{},{},{},{}'.format(
time_epoch_ms(), self.name, 'mAP', mAP))
self._logger.info('Computing accuracy for {} {}'.format(
end_time, start_time))
else:
# The remaining entries require newer ground bboxes.
break
self.__garbage_collect_obstacles()
def on_ground_obstacles(self, msg, map_stream):
# Ignore the first several seconds of the simulation because the car is
# not moving at the beginning.
assert len(msg.timestamp.coordinates) == 1
game_time = msg.timestamp.coordinates[0]
bboxes = []
# Select the person bounding boxes.
for obstacle in msg.obstacles:
if obstacle.label == 'person':
bboxes.append(obstacle.bounding_box)
# Remove the buffered bboxes that are too old.
while (len(self._ground_bboxes) > 0
and game_time - self._ground_bboxes[0][0] >
self._flags.decay_max_latency):
self._ground_bboxes.popleft()
sim_time = msg.timestamp.coordinates[0]
for (old_game_time, old_bboxes) in self._ground_bboxes:
# Ideally, we would like to take multiple precision values at
# different recalls and average them, but we can't vary model
# confidence, so we just return the actual precision.
if (len(bboxes) > 0 or len(old_bboxes) > 0):
latency = game_time - old_game_time
precisions = []
for iou in self._iou_thresholds:
(precision,
_) = get_precision_recall_at_iou(bboxes, old_bboxes, iou)
precisions.append(precision)
self._logger.info("Precision {}".format(precisions))
avg_precision = float(sum(precisions)) / len(precisions)
self._logger.info(
"The latency is {} and the average precision is {}".format(
latency, avg_precision))
self._csv_logger.info('{},{},{},{},{:.4f}'.format(
time_epoch_ms(), sim_time, self.config.name, latency,
avg_precision))
map_stream.send(
erdos.Message(msg.timestamp, (latency, avg_precision)))
# Buffer the new bounding boxes.
self._ground_bboxes.append((game_time, bboxes))
def compute_and_log_map(current_pedestrians,
current_timestamp,
csv,
deadline=210,
base_iou=0.5,
step=0.05):
""" Computes the AP from the given IOU for the detected objects. Note that,
since we use a perfect detector, our confidence values for each detection
is 1.0 and so we can't vary the recall. Thus, we cannot calculate the
area under the precision-recall curve, and default to using AP50 as our
metric for mAP.
Args:
current_pedestrians: List of bboxes for the detected pedestrians
current_timestamp: The timestamp associated with the current frame.
csv: The csv file to write the results to.
deadline: The oldest frame to compare the results to.
base_iou: The IOU to start from.
step: The step to take from the base_IOU to reach 1.0
"""
SAVED_DETECTIONS.append((current_timestamp, current_pedestrians))
# Remove data older than the deadline that we don't need anymore.
while (current_timestamp - SAVED_DETECTIONS[0][0]) * 1000 > deadline:
SAVED_DETECTIONS.popleft()
# Go over each of the saved frames, compute the difference in the
# timestamp, the AP at the given IOU and log them.
for old_timestamp, old_detections in SAVED_DETECTIONS:
for iou in np.arange(base_iou, 1.0, step):
precision, _ = get_precision_recall_at_iou(current_pedestrians,
old_detections, iou)
time_diff = current_timestamp - old_timestamp
# Format of the CSV file: (latency_in_ms, AP{IOU}, {IOU})
csv.writerow([time_diff * 1000, precision, iou])