def score_run(gt_dir, pred_dir):
gt_files = sorted(glob.glob(os.path.join(gt_dir, 'masks', '*.png')))
pred_files = sorted(glob.glob(os.path.join(pred_dir, '*.png')))
iou = 0
error1 = 0
error2 = 0
n_valid_detections = 0
n_invalid_detections = 0
for e, gt_file in enumerate(gt_files):
gt_mask = misc.imread(gt_file)[:, :, 1].clip(0, 1)
pred_mask = (misc.imread(pred_files[e])[:, :, 1] > 127).astype(np.int)
iou += scoring_utils.intersection_over_union(pred_mask, gt_mask)
# only compute centroid distance if it found some object
if pred_mask.sum() > 3:
if gt_mask.sum() > 3:
gt_centroid = scoring_utils.get_centroid_largest_blob(gt_mask)
pred_centroid = scoring_utils.get_centroid_largest_blob(
pred_mask)
error1 += scoring_utils.average_squared_distance(
pred_centroid, gt_centroid)
error2 += scoring_utils.average_squared_log_distance(
pred_centroid, gt_centroid)
n_valid_detections += 1
else:
error1 += 1
error2 += 1
n_invalid_detections += 1
return iou, error1, error2, len(
gt_files), n_valid_detections, n_invalid_detections
def on_sensor_frame(self, data):
rgb_image = Image.open(BytesIO(base64.b64decode(data['rgb_image'])))
rgb_image = np.asarray(rgb_image)
if rgb_image.shape != (256, 256, 3):
print('image shape not 256, 256, 3')
return None
if rgb_image.shape[0] != self.image_hw:
rgb_image = misc.imresize(rgb_image, (self.image_hw, self.image_hw, 3))
rgb_image = data_iterator.preprocess_input(rgb_image)
pred = np.squeeze(model.predict(np.expand_dims(rgb_image, 0)))
if self.pred_viz_enabled:
self.queue.put([rgb_image, pred])
target_mask = pred[:, :, 2] > 0.5
# reduce the number of false positives by requiring more pixels to be identified as containing the target
if target_mask.sum() > 10:
centroid = scoring_utils.get_centroid_largest_blob(target_mask)
# scale the centroid from the nn image size to the original image size
centroid = centroid.astype(np.int).tolist()
# Obtain 3D world point from centroid pixel
depth_img = get_depth_image(data['depth_image'])
# Get XYZ coordinates for specific pixel
pixel_depth = depth_img[centroid[0]][centroid[1]][0]*50/255.0
point_3d = get_xyz_from_image(centroid[0], centroid[1], pixel_depth, self.image_hw)
point_3d.append(1)
# Get cam_pose from sensor_frame (ROS convention)
cam_pose = get_ros_pose(data['gimbal_pose'])
# Calculate xyz-world coordinates of the point corresponding to the pixel
# Transformation Matrix
R = euler2mat(math.radians(cam_pose[3]), math.radians(cam_pose[4]), math.radians(cam_pose[5]))
T = np.c_[R, cam_pose[:3]]
T = np.vstack([T, [0,0,0,1]]) # transformation matrix from world to quad
# 3D point in ROS coordinates
ros_point = np.dot(T, point_3d)
sio.emit('object_detected', {'coords': [ros_point[0], ros_point[1], ros_point[2]]})
if not self.target_found:
print('Target found!')
self.target_found = True
self.num_no_see = 0
# Publish Hero Marker
marker_pos = [ros_point[0],ros_point[1], ros_point[2]] + [0, 0, 0]
marker_msg = sio_msgs.create_box_marker_msg(np.random.randint(99999), marker_pos)
sio.emit('create_box_marker', marker_msg)
elif self.target_found:
self.num_no_see += 1
# print(self.num_no_see)
if self.target_found and self.num_no_see > 6:
print('Target lost!')
sio.emit('object_lost', {'data': ''})
self.target_found = False
self.num_no_see = 0
def on_sensor_data(self, data):
rgb_image = Image.open(BytesIO(base64.b64decode(data['rgb_image'])))
rgb_image = np.asarray(rgb_image)
if rgb_image.shape != (256, 256, 3):
print('image shape not 256, 256, 3')
return None
rgb_image = data_iterator.preprocess_input(rgb_image)
pred = np.squeeze(model.predict(np.expand_dims(rgb_image, 0)))
target_mask = pred[:, :, 1] > 0.5
if self.pred_viz_enabled:
overlay_viz(rgb_image, pred)
# reduce the number of false positives by requiring more pixels to be identified as containing the target
if target_mask.sum() > 10:
# Temporary move so we only get the overlays with positive identification
if self.image_save_dir is not None:
if time.time() - self.last_time_saved > 1:
result = visualization.overlay_predictions(rgb_image, pred, None, 0.5, 1)
out_file = os.path.join(self.image_save_dir, 'overlay_' + str(time.time()) + '.png')
misc.imsave(out_file, result)
self.last_time_saved = time.time()
centroid = scoring_utils.get_centroid_largest_blob(target_mask)
# scale the centroid from the nn image size to the original image size
centroid = centroid.astype(np.int).tolist()
# Obtain 3D world point from centroid pixel
depth_img = get_depth_image(data['depth_image'])
# Get XYZ coordinates for specific pixel
pixel_depth = depth_img[centroid[0]][centroid[1]][0]*100/255.0
point_3d = get_xyz_from_image(centroid[0], centroid[1], pixel_depth)
point_3d.append(1)
#print("Pixel Depth: ", pixel_depth)
#print ("Hit point: ", point_3d)
# Get cam_pose from sensor_frame (ROS convention)
cam_pose = get_ros_pose(data['gimbal_pose'])
#print ("Quad Pose: ", cam_pose)
# Calculate xyz-world coordinates of the point corresponding to the pixel
# Transformation Matrix
R = euler2mat(math.radians(cam_pose[3]), math.radians(cam_pose[4]), math.radians(cam_pose[5]))
T = np.c_[R, cam_pose[:3]]
T = np.vstack([T, [0,0,0,1]]) # transformation matrix from world to quad
# 3D point in ROS coordinates
ros_point = np.dot(T, point_3d)
socketIO.emit('object_detected', {'coords': [ros_point[0], ros_point[1], ros_point[2]]})
self.target_found = True
self.num_no_see = 0
# Publish Hero Marker
marker_pos = [ros_point[0],ros_point[1], ros_point[2]] + [0, 0, 0]
marker_msg = sio_msgs.create_box_marker_msg(np.random.randint(99999), marker_pos)
socketIO.emit('create_box_marker', marker_msg)
# 3D point in Unity coordinates
#unity_point = get_unity_pose_from_ros(ros_point)
#print ros_point.shape
# with the depth image, and centroid from prediction we can compute
# the x,y,z coordinates where the ray intersects an object
# ray = ray_casting.cast_ray(data, [centroid[1], centroid[0]])
# pose = np.array(list(map(float, data['gimbal_pose'].split(','))))
# TODO add rotation of the camera with respect to the gimbal
# create the rotation matrix to rotate the sensors frame of reference to the world frame
# rot = transformations.euler_matrix(to_radians(pose[3]),
# to_radians(pose[4]),
# to_radians(pose[5]))[:3, :3]
# rotate array
# ray = np.dot(rot, np.array(ray))
elif self.target_found:
self.num_no_see += 1
if self.target_found and self.num_no_see > 8:
socketIO.emit('object_lost', {'data': ''})
self.target_found = False
self.num_no_see = 0
def on_sensor_frame(self, data):
rgb_image = Image.open(BytesIO(base64.b64decode(data['rgb_image'])))
rgb_image = np.asarray(rgb_image)
if rgb_image.shape != (256, 256, 3):
print('image shape not 256, 256, 3')
return None
if rgb_image.shape[0] != self.image_hw:
rgb_image = misc.imresize(rgb_image, (self.image_hw, self.image_hw, 3))
rgb_image = data_iterator.preprocess_input(rgb_image)
pred = np.squeeze(model.predict(np.expand_dims(rgb_image, 0)))
if self.pred_viz_enabled:
self.queue.put([rgb_image, pred])
target_mask = pred[:, :, 2] > 0.5
# reduce the number of false positives by requiring more pixels to be identified as containing the target
if target_mask.sum() > 10:
centroid = scoring_utils.get_centroid_largest_blob(target_mask)
# scale the centroid from the nn image size to the original image size
centroid = centroid.astype(np.int).tolist()
# Obtain 3D world point from centroid pixel
depth_img = get_depth_image(data['depth_image'])
# Get XYZ coordinates for specific pixel
pixel_depth = depth_img[centroid[0]][centroid[1]][0]*50/255.0
point_3d = get_xyz_from_image(centroid[0], centroid[1], pixel_depth, self.image_hw)
point_3d.append(1)
# Get cam_pose from sensor_frame (ROS convention)
cam_pose = get_ros_pose(data['gimbal_pose'])
# Calculate xyz-world coordinates of the point corresponding to the pixel
# Transformation Matrix
R = euler2mat(math.radians(cam_pose[3]), math.radians(cam_pose[4]), math.radians(cam_pose[5]))
T = np.c_[R, cam_pose[:3]]
T = np.vstack([T, [0,0,0,1]]) # transformation matrix from world to quad
# 3D point in ROS coordinates
ros_point = np.dot(T, point_3d)
sio.emit('object_detected', {'coords': [ros_point[0], ros_point[1], ros_point[2]]})
if not self.target_found:
print('Target found!')
self.target_found = True
self.num_no_see = 0
# Publish Hero Marker
marker_pos = [ros_point[0],ros_point[1], ros_point[2]] + [0, 0, 0]
marker_msg = sio_msgs.create_box_marker_msg(np.random.randint(99999), marker_pos)
sio.emit('create_box_marker', marker_msg)
elif self.target_found:
self.num_no_see += 1
# print(self.num_no_see)
if self.target_found and self.num_no_see > 6:
print('Target lost!')
sio.emit('object_lost', {'data': ''})
self.target_found = False
self.num_no_see = 0