def _apply(self, bof_msg, ch_msg):
target_names = bof_msg.target_names
assert target_names == ch_msg.target_names
N_label = len(target_names)
bof_proba = np.array(bof_msg.probabilities).reshape((-1, N_label))
ch_proba = np.array(ch_msg.probabilities).reshape((-1, N_label))
bof_weight = np.array([self.weight[n]['bof'] for n in target_names])
ch_weight = np.array([self.weight[n]['color'] for n in target_names])
y_proba = (bof_weight * bof_proba) + (ch_weight * ch_proba)
# verification result for debug
y_pred = np.argmax(y_proba, axis=-1)
target_names = np.array(target_names)
label_proba = [p[i] for p, i in zip(y_proba, y_pred)]
# compose msg
msg = ClassificationResult()
msg.header = bof_msg.header
msg.labels = y_pred
msg.label_names = target_names[y_pred]
msg.label_proba = label_proba
msg.probabilities = y_proba.reshape(-1)
msg.classifier = '<jsk_2015_05_baxter_apc.BoostObjectRecognition>'
msg.target_names = target_names
self.pub.publish(msg)
def _apply(self, bof_msg, ch_msg):
target_names = bof_msg.target_names
assert target_names == ch_msg.target_names
N_label = len(target_names)
bof_proba = np.array(bof_msg.probabilities).reshape((-1, N_label))
ch_proba = np.array(ch_msg.probabilities).reshape((-1, N_label))
bof_weight = np.array([self.weight[n]['bof'] for n in target_names])
ch_weight = np.array([self.weight[n]['color'] for n in target_names])
y_proba = (bof_weight * bof_proba) + (ch_weight * ch_proba)
# verification result for debug
y_pred = np.argmax(y_proba, axis=-1)
target_names = np.array(target_names)
label_proba = [p[i] for p, i in zip(y_proba, y_pred)]
# compose msg
msg = ClassificationResult()
msg.header = bof_msg.header
msg.labels = y_pred
msg.label_names = target_names[y_pred]
msg.label_proba = label_proba
msg.probabilities = y_proba.reshape(-1)
msg.classifier = '<jsk_2015_05_baxter_apc.BoostObjectRecognition>'
msg.target_names = target_names
self.pub.publish(msg)
def _predict(self, img_msg, label_msg):
# convert image
bridge = cv_bridge.CvBridge()
input_image = bridge.imgmsg_to_cv2(img_msg, 'rgb8')
input_label = bridge.imgmsg_to_cv2(label_msg)
# predict
region_imgs = []
for l in np.unique(input_label):
if l == 0: # bg_label
continue
mask = (input_label == l)
region = jsk_recognition_utils.bounding_rect_of_mask(
input_image, mask)
region_imgs.append(region)
y_proba = self.estimator.predict(region_imgs)
target_names = np.array(jsk_apc2015_common.get_object_list())
y_pred = np.argmax(y_proba, axis=-1)
label_proba = [p[i] for p, i in zip(y_proba, y_pred)]
# prepare message
res = ClassificationResult()
res.header = img_msg.header
res.labels = y_pred
res.label_names = target_names[y_pred]
res.label_proba = label_proba
res.probabilities = y_proba.reshape(-1)
res.classifier = '<jsk_2015_05_baxter_apc.ColorHistogramFeatures>'
res.target_names = target_names
self._pub.publish(res)
def image_cb(self, msg):
img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='rgb8')
# RGB -> BGR
rospy.logdebug('start readtext')
results = self.reader.readtext(img[:, :, ::-1])
rospy.logdebug('end readtext')
bboxes = []
scores = []
texts = []
rect_msg = RectArray(header=msg.header)
for result in results:
bb = result[0]
text = result[1]
score = result[2]
x_min = int(np.round(bb[0][0]))
y_min = int(np.round(bb[0][1]))
x_max = int(np.round(bb[2][0]))
y_max = int(np.round(bb[2][1]))
bboxes.append([y_min, x_min, y_max, x_max])
texts.append(text)
scores.append(score)
rect = Rect(
x=x_min, y=y_min,
width=x_max - x_min, height=y_max - y_min)
rect_msg.rects.append(rect)
bboxes = np.array(bboxes)
scores = np.array(scores)
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=texts,
labels=np.arange(len(texts)),
label_names=texts,
label_proba=scores)
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.visualize:
fig = plt.figure(
tight_layout={'pad': 0})
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.axis('off')
fig.add_axes(ax)
vis_bbox(
img.transpose((2, 0, 1)),
bboxes, np.arange(len(texts)), scores,
label_names=texts, ax=ax)
fig.canvas.draw()
w, h = fig.canvas.get_width_height()
vis_img = np.fromstring(
fig.canvas.tostring_rgb(), dtype=np.uint8)
vis_img.shape = (h, w, 3)
fig.clf()
plt.close()
vis_msg = self.bridge.cv2_to_imgmsg(vis_img, 'rgb8')
vis_msg.header = msg.header
self.pub_image.publish(vis_msg)
def cb(self, msg):
label_name = msg.label_names[0]
if label_name not in self.name_to_label:
self.name_to_label[label_name] = msg.labels[0]
# Filtering class for gesture recognition
now_time = rospy.Time.now()
if label_name != self.temporary_class:
self.last_temporary_class_time = rospy.Time.now()
self.temporary_class = label_name
# Update continous class
if (now_time - self.last_temporary_class_time).to_sec() > self.duration_thre:
self.continuous_class = self.temporary_class
# Publish continous msg
probabilities = [0.0] * len(msg.probabilities)
probabilities[0] = 1.0
pubmsg = ClassificationResult(
header=msg.header,
labels=[self.name_to_label[self.continuous_class]],
label_names=[self.continuous_class],
label_proba=[1.0],
probabilities=probabilities,
classifier=msg.classifier,
target_names=msg.target_names,
)
self.pub.publish(pubmsg)
def _recognize(self, imgmsg):
bridge = cv_bridge.CvBridge()
bgr = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
bgr = skimage.transform.resize(bgr, (224, 224), preserve_range=True)
input_msg = bridge.cv2_to_imgmsg(bgr.astype(np.uint8), encoding='bgr8')
input_msg.header = imgmsg.header
self.pub_input.publish(input_msg)
blob = (bgr - self.mean_bgr).transpose((2, 0, 1))
x_data = np.array([blob], dtype=np.float32)
if self.gpu != -1:
x_data = cuda.to_gpu(x_data, device=self.gpu)
x = Variable(x_data, volatile=True)
self.model.train = False
self.model(x)
proba = cuda.to_cpu(self.model.pred.data)[0]
label = np.argmax(proba)
label_name = self.target_names[label]
label_proba = proba[label]
cls_msg = ClassificationResult(
header=imgmsg.header,
labels=[label],
label_names=[label_name],
label_proba=[label_proba],
probabilities=proba,
classifier=self.model_name,
target_names=self.target_names,
)
self.pub.publish(cls_msg)
def _recognize(self, imgmsg, mask_msg=None):
bridge = cv_bridge.CvBridge()
bgr = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
if mask_msg is not None:
mask = bridge.imgmsg_to_cv2(mask_msg)
if mask.shape != bgr.shape[:2]:
logerr_throttle(10,
'Size of input image and mask is different')
return
elif mask.size == 0:
logerr_throttle(10, 'Size of input mask is 0')
return
bgr[mask < 128] = self.mean_bgr
input_msg = bridge.cv2_to_imgmsg(bgr.astype(np.uint8), encoding='bgr8')
input_msg.header = imgmsg.header
self.pub_input.publish(input_msg)
blob = (bgr - self.mean_bgr).transpose((2, 0, 1))
x_data = np.array([blob], dtype=np.float32)
if self.gpu != -1:
x_data = chainer.cuda.to_gpu(x_data, device=self.gpu)
x = Variable(x_data, volatile=True)
self.model.train = False
self.model(x)
proba = chainer.cuda.to_cpu(self.model.proba.data)[0]
cls_msg = ClassificationResult(header=imgmsg.header,
labels=None,
label_names=None,
label_proba=None,
probabilities=proba,
target_names=self.target_names)
self.pub.publish(cls_msg)
def image_cb(self, msg):
img_orig = self.bridge.imgmsg_to_cv2(msg, desired_encoding='rgb8')
img = np.asarray(img_orig) / 255
H, W = img.shape[:2]
h_step = self.patch_height * self.subsample
w_step = self.patch_width * self.subsample
h_idxs = np.arange(0, H, h_step)[:-1]
w_idxs = np.arange(0, W, h_step)[:-1]
results = []
for y in h_idxs:
for x in w_idxs:
sub_img = img[y:y+h_step:self.subsample, x:x+w_step:self.subsample, :][np.newaxis, ...]
self.interpreter.set_tensor(self.input_details[0]['index'], sub_img.astype(np.float32))
self.interpreter.invoke()
cls = self.interpreter.get_tensor(self.output_details[0]['index'])
cls = [np.argmax(cls), np.max(cls)]
results.append(cls)
labels = np.asarray([p[0] for p in results])
probs = np.asarray([p[1] for p in results])
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[l] for l in labels],
label_proba=probs)
self.pub_classification.publish(cls_msg)
if self.visualize:
fig = plt.figure(tight_layout={'pad': 0})
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.axis('off')
fig.add_axes(ax)
clr = np.asarray([[1,0,0], [0,1,0], [0,0,1], [0,0,0]])
idx = 0
ax = vis_image(img_orig.transpose((2, 0, 1)), ax=ax)
for y in h_idxs:
for x in w_idxs:
ax.add_patch(plt.Rectangle(
(x,y),
self.patch_width * self.subsample,
self.patch_height * self.subsample,
fill=True,
facecolor=clr[labels[idx]],
linewidth=0,
alpha=0.3))
idx += 1
fig.canvas.draw()
w, h = fig.canvas.get_width_height()
vis_img = np.fromstring(
fig.canvas.tostring_rgb(), dtype=np.uint8)
vis_img.shape = (h, w, 3)
fig.clf()
plt.close()
vis_msg = self.bridge.cv2_to_imgmsg(vis_img, 'rgb8')
# BUG: https://answers.ros.org/question/316362/sensor_msgsimage-generates-float-instead-of-int-with-python3/ # NOQA
vis_msg.step = int(vis_msg.step)
vis_msg.header = msg.header
self.pub_image.publish(vis_msg)
def _cb(self, imgmsg):
bridge = cv_bridge.CvBridge()
img = bridge.imgmsg_to_cv2(imgmsg)
label, proba = self._classify(img)
msg = ClassificationResult()
msg.header = imgmsg.header
msg.labels = [label]
msg.label_names = [self.target_names[label]]
msg.label_proba = [proba[label]]
msg.probabilities = proba
msg.classifier = self.classifier_name
msg.target_names = self.target_names
self.pub.publish(msg)
def image_cb(self, msg):
img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='bgr8')
H, W = img.shape[:2]
objs = self.engine.DetectWithImage(PIL.Image.fromarray(img),
threshold=self.score_thresh,
keep_aspect_ratio=True,
relative_coord=True,
top_k=self.top_k)
bboxes = []
scores = []
labels = []
rect_msg = RectArray(header=msg.header)
for obj in objs:
x_min, y_min, x_max, y_max = obj.bounding_box.flatten().tolist()
x_min = int(np.round(x_min * W))
y_min = int(np.round(y_min * H))
x_max = int(np.round(x_max * W))
y_max = int(np.round(y_max * H))
bboxes.append([y_min, x_min, y_max, x_max])
scores.append(obj.score)
labels.append(self.label_ids.index(int(obj.label_id)))
rect = Rect(x=x_min,
y=y_min,
width=x_max - x_min,
height=y_max - y_min)
rect_msg.rects.append(rect)
bboxes = np.array(bboxes)
scores = np.array(scores)
labels = np.array(labels)
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[l] for l in labels],
label_proba=scores)
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.visualize:
vis_img = img[:, :, ::-1].transpose(2, 0, 1)
vis_bbox(vis_img,
bboxes,
labels,
scores,
label_names=self.label_names)
fig = plt.gcf()
fig.canvas.draw()
w, h = fig.canvas.get_width_height()
vis_img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8)
fig.clf()
vis_img.shape = (h, w, 3)
plt.close()
vis_msg = self.bridge.cv2_to_imgmsg(vis_img, 'rgb8')
vis_msg.header = msg.header
self.pub_image.publish(vis_msg)
def _detect(self, imgmsg, rects_msg):
bridge = cv_bridge.CvBridge()
im_orig = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
im, im_scale = img_preprocessing(im_orig, self.pixel_means)
rects_orig = jsk_recognition_utils.rects_msg_to_ndarray(rects_msg)
if len(rects_orig) == 0:
return
rects = rects_orig * im_scale
scores, bbox_pred = self._im_detect(im, rects)
rects = RectArray(header=imgmsg.header)
labels = []
label_proba = []
for cls_id in range(1, len(self.target_names)):
_cls = scores[:, cls_id][:, np.newaxis]
_bbx = bbox_pred[:, cls_id * 4:(cls_id + 1) * 4]
dets = np.hstack((_bbx, _cls))
keep = nms(dets, self.nms_thresh)
dets = dets[keep, :]
orig_rects = cuda.cupy.asnumpy(rects_orig)[keep, :]
inds = np.where(dets[:, -1] >= self.conf_thresh)[0]
for i in inds:
_bbox = dets[i, :4]
x1, y1, x2, y2 = orig_rects[i]
width = x2 - x1
height = y2 - y1
center_x = x1 + 0.5 * width
center_y = y1 + 0.5 * height
dx, dy, dw, dh = _bbox
_center_x = dx * width + center_x
_center_y = dy * height + center_y
_width = np.exp(dw) * width
_height = np.exp(dh) * height
x1 = _center_x - 0.5 * _width
y1 = _center_y - 0.5 * _height
x2 = _center_x + 0.5 * _width
y2 = _center_y + 0.5 * _height
rect = Rect(x=x1, y=y1, width=x2 - x1, height=y2 - y1)
rects.rects.append(rect)
labels.append(cls_id)
label_proba.append(dets[:, -1][i])
# publish classification result
clss = ClassificationResult(
header=imgmsg.header,
classifier=self.classifier_name,
target_names=self.target_names,
labels=labels,
label_names=[self.target_names[l] for l in labels],
label_proba=label_proba,
)
self._pub_rects.publish(rects)
self._pub_class.publish(clss)
def _recognize(self, imgmsg):
bridge = cv_bridge.CvBridge()
mono = bridge.imgmsg_to_cv2(imgmsg)
bgr = img_jet(mono)
bgr = skimage.transform.resize(bgr, (self.insize, self.insize),
preserve_range=True)
input_msg = bridge.cv2_to_imgmsg(bgr.astype(np.uint8), encoding='bgr8')
input_msg.header = imgmsg.header
self.pub_input.publish(input_msg)
# (Height, Width, Channel) -> (Channel, Height, Width)
# ###
# print(type())
# print(rgb.shape)
# import cv2
# cv2.imwrite('/home/naoya/test.png', rgb)
# ###
rgb = bgr.transpose((2, 0, 1))[::-1, :, :]
rgb = self.dataset.process_image(rgb)
x_data = np.array([rgb], dtype=np.float32)
if self.gpu != -1:
x_data = cuda.to_gpu(x_data, device=self.gpu)
if LooseVersion(chainer.__version__) < LooseVersion('2.0.0'):
x = Variable(x_data, volatile=True)
self.model.train = False
self.model(x)
else:
with chainer.using_config('train', False), \
chainer.no_backprop_mode():
x = Variable(x_data)
self.model(x)
# swap_labels[label number in self.target_names]
# -> label number in self.target_names_ordered
swap_labels = [
self.target_names_ordered.index(name) for name in self.target_names
]
for i in range(len(swap_labels)):
if not (i in swap_labels):
rospy.logerr('Wrong target_names is given by rosparam.')
exit()
proba = cuda.to_cpu(self.model.pred.data)[0]
proba_swapped = [proba[swap_labels[i]] for i, p in enumerate(proba)]
label_swapped = np.argmax(proba_swapped)
label_name = self.target_names[label_swapped]
label_proba = proba_swapped[label_swapped]
cls_msg = ClassificationResult(
header=imgmsg.header,
labels=[label_swapped],
label_names=[label_name],
label_proba=[label_proba],
probabilities=proba_swapped,
classifier=self.model_name,
target_names=self.target_names,
)
self.pub.publish(cls_msg)
def _cb(self, imgmsg):
bridge = cv_bridge.CvBridge()
img = bridge.imgmsg_to_cv2(imgmsg)
label, proba = self._classify(img)
msg = ClassificationResult()
msg.header = imgmsg.header
msg.labels = [label]
msg.label_names = [self.target_names[label]]
msg.label_proba = [proba[label]]
msg.probabilities = proba
msg.classifier = self.classifier_name
msg.target_names = self.target_names
self.pub.publish(msg)
def publish_gender(self, header, results):
target_names = ["Male", "Female"]
labels = [0 if r["gender"] < 0.5 else 1 for r in results]
msg = ClassificationResult(
header=header,
classifier=self.classifier_name,
target_names=target_names,
labels=labels,
label_names=[target_names[l] for l in labels],
label_proba=[r["detection"] for r in results],
)
self.pub_gender.publish(msg)
def image_cb(self, msg):
if not hasattr(self, 'engine'):
return
if self.transport_hint == 'compressed':
np_arr = np.fromstring(msg.data, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
img = img[:, :, ::-1]
else:
img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='rgb8')
points, key_names, visibles, bboxes, labels, scores \
= self._estimate(img)
poses_msg = PeoplePoseArray(header=msg.header)
rects_msg = RectArray(header=msg.header)
for point, key_name, visible, bbox, label, score in zip(
points, key_names, visibles, bboxes, labels, scores):
pose_msg = PeoplePose()
for pt, key_nm, vs, sc in zip(point, key_name, visible, score):
if vs:
key_y, key_x = pt
pose_msg.limb_names.append(key_nm)
pose_msg.scores.append(sc)
pose_msg.poses.append(
Pose(position=Point(x=key_x, y=key_y)))
poses_msg.poses.append(pose_msg)
y_min, x_min, y_max, x_max = bbox
rect = Rect(x=x_min,
y=y_min,
width=x_max - x_min,
height=y_max - y_min)
rects_msg.rects.append(rect)
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[lbl] for lbl in labels],
label_proba=[np.average(score) for score in scores])
self.pub_pose.publish(poses_msg)
self.pub_rects.publish(rects_msg)
self.pub_class.publish(cls_msg)
if self.enable_visualization:
with self.lock:
self.img = img
self.header = msg.header
self.points = points
self.visibles = visibles
def _predict(self, msg):
X = np.array(msg.data).reshape((-1, msg.vector_dim))
normalize(X, copy=False)
y_proba = self.clf.predict_proba(X)
y_pred = np.argmax(y_proba, axis=-1)
target_names = np.array(self.clf.target_names_)
self._pub.publish(
ClassificationResult(
header=msg.header,
labels=y_pred,
label_names=target_names[y_pred],
label_proba=y_proba[:, y_pred],
probabilities=y_proba.reshape(-1),
classifier=self.clf.__str__(),
target_names=target_names,
))
def _recognize(self, imgmsg, mask_msg=None):
bridge = cv_bridge.CvBridge()
bgr = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
if mask_msg is not None:
mask = bridge.imgmsg_to_cv2(mask_msg)
if mask.shape != bgr.shape[:2]:
logerr_throttle(10,
'Size of input image and mask is different')
return
elif mask.size == 0:
logerr_throttle(10, 'Size of input mask is 0')
return
bgr[mask == 0] = self.mean_bgr
bgr = skimage.transform.resize(bgr, (self.insize, self.insize),
preserve_range=True)
input_msg = bridge.cv2_to_imgmsg(bgr.astype(np.uint8), encoding='bgr8')
input_msg.header = imgmsg.header
self.pub_input.publish(input_msg)
blob = (bgr - self.mean_bgr).transpose((2, 0, 1))
x_data = np.array([blob], dtype=np.float32)
if self.gpu != -1:
x_data = cuda.to_gpu(x_data, device=self.gpu)
if LooseVersion(chainer.__version__) < LooseVersion('2.0.0'):
x = Variable(x_data, volatile=True)
self.model.train = False
self.model(x)
else:
with chainer.using_config('train', False), \
chainer.no_backprop_mode():
x = Variable(x_data)
self.model(x)
proba = cuda.to_cpu(self.model.pred.data)[0]
label = np.argmax(proba)
label_name = self.target_names[label]
label_proba = proba[label]
cls_msg = ClassificationResult(
header=imgmsg.header,
labels=[label],
label_names=[label_name],
label_proba=[label_proba],
probabilities=proba,
classifier=self.model_name,
target_names=self.target_names,
)
self.pub.publish(cls_msg)
def callback(self, imgmsg, mask_msg):
bridge = cv_bridge.CvBridge()
img = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
mask = bridge.imgmsg_to_cv2(mask_msg, desired_encoding='mono8')
if mask.ndim == 3:
mask = np.squeeze(mask, axis=2)
mask = mask >= 127 # uint8 -> bool
img = img.astype(np.float64)
img[mask] -= self.mean[mask]
img[~mask] = 0
img = img.transpose(2, 0, 1)
img = img.astype(np.float32)
x_data = np.asarray([img])
if self.gpu >= 0:
x_data = cuda.to_gpu(x_data)
x = chainer.Variable(x_data)
y = self.model(x)
feat = cuda.to_cpu(y.data)
feat = feat.squeeze(axis=(2, 3))
X_query = feat
y_pred_proba = self.knn.predict_proba(X_query)
y_pred = np.argmax(y_pred_proba, axis=1)
classes = self.knn.classes_
target_names = self.target_names[classes]
msg = ClassificationResult()
msg.header = imgmsg.header
msg.labels = y_pred.tolist()
msg.label_names = target_names[y_pred].tolist()
msg.label_proba = y_pred_proba[:, y_pred].flatten().tolist()
msg.probabilities = y_pred_proba.flatten().tolist()
msg.target_names = target_names.tolist()
self.pub.publish(msg)
def image_cb(self, msg):
if not hasattr(self, 'engine'):
return
if self.transport_hint == 'compressed':
np_arr = np.fromstring(msg.data, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
img = img[:, :, ::-1]
else:
img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='rgb8')
bboxes, labels, scores = self._detect(img)
rect_msg = RectArray(header=msg.header)
for bbox in bboxes:
y_min, x_min, y_max, x_max = bbox
rect = Rect(x=x_min,
y=y_min,
width=x_max - x_min,
height=y_max - y_min)
rect_msg.rects.append(rect)
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[lbl] for lbl in labels],
label_proba=scores)
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.enable_visualization:
with self.lock:
self.img = img
self.header = msg.header
self.bboxes = bboxes
self.labels = labels
self.scores = scores
def callback(self, imgmsg, mask_msg):
bridge = cv_bridge.CvBridge()
img = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='bgr8')
mask = bridge.imgmsg_to_cv2(mask_msg, desired_encoding='mono8')
if mask.ndim == 3:
mask = np.squeeze(mask, axis=2)
mask = mask >= 127 # uint8 -> bool
img = img.astype(np.float64)
img[mask] -= self.mean[mask]
img[~mask] = 0
img = img.transpose(2, 0, 1)
img = img.astype(np.float32)
x_data = np.asarray([img])
if self.gpu >= 0:
x_data = cuda.to_gpu(x_data)
x = chainer.Variable(x_data)
y = self.model(x)
feat = cuda.to_cpu(y.data)
feat = feat.squeeze(axis=(2, 3))
X_query = feat
y_pred_proba = self.knn.predict_proba(X_query)
y_pred = np.argmax(y_pred_proba, axis=1)
classes = self.knn.classes_
target_names = self.target_names[classes]
msg = ClassificationResult()
msg.header = imgmsg.header
msg.labels = y_pred.tolist()
msg.label_names = target_names[y_pred].tolist()
msg.label_proba = y_pred_proba[:, y_pred].flatten().tolist()
msg.probabilities = y_pred_proba.flatten().tolist()
msg.target_names = target_names.tolist()
self.pub.publish(msg)
def callback(self, imgmsg):
bridge = cv_bridge.CvBridge()
img = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='rgb8')
img_chw = img.transpose(2, 0, 1) # C, H, W
bboxes, masks, labels, scores = self.model.predict([img_chw])
bboxes = bboxes[0]
masks = masks[0]
labels = labels[0]
scores = scores[0]
msg_indices = ClusterPointIndices(header=imgmsg.header)
msg_labels = LabelArray(header=imgmsg.header)
# -1: label for background
lbl_cls = -np.ones(img.shape[:2], dtype=np.int32)
lbl_ins = -np.ones(img.shape[:2], dtype=np.int32)
for ins_id, (mask, label) in enumerate(zip(masks, labels)):
indices = np.where(mask.flatten())[0]
indices_msg = PointIndices(header=imgmsg.header, indices=indices)
msg_indices.cluster_indices.append(indices_msg)
class_name = self.fg_class_names[label]
msg_labels.labels.append(Label(id=label, name=class_name))
lbl_cls[mask] = label
lbl_ins[mask] = ins_id # instance_id
self.pub_indices.publish(msg_indices)
self.pub_labels.publish(msg_labels)
msg_lbl_cls = bridge.cv2_to_imgmsg(lbl_cls)
msg_lbl_ins = bridge.cv2_to_imgmsg(lbl_ins)
msg_lbl_cls.header = msg_lbl_ins.header = imgmsg.header
self.pub_lbl_cls.publish(msg_lbl_cls)
self.pub_lbl_ins.publish(msg_lbl_ins)
cls_msg = ClassificationResult(
header=imgmsg.header,
classifier=self.classifier_name,
target_names=self.fg_class_names,
labels=labels,
label_names=[self.fg_class_names[l] for l in labels],
label_proba=scores,
)
rects_msg = RectArray(header=imgmsg.header)
for bbox in bboxes:
rect = Rect(x=bbox[1],
y=bbox[0],
width=bbox[3] - bbox[1],
height=bbox[2] - bbox[0])
rects_msg.rects.append(rect)
self.pub_rects.publish(rects_msg)
self.pub_class.publish(cls_msg)
if self.pub_viz.get_num_connections() > 0:
n_fg_class = len(self.fg_class_names)
captions = [
'{:d}: {:s}'.format(l, self.fg_class_names[l]) for l in labels
]
viz = chainer_mask_rcnn.utils.draw_instance_bboxes(
img,
bboxes,
labels + 1,
n_class=n_fg_class + 1,
masks=masks,
captions=captions)
msg_viz = bridge.cv2_to_imgmsg(viz, encoding='rgb8')
msg_viz.header = imgmsg.header
self.pub_viz.publish(msg_viz)
def image_cb(self, msg):
img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='bgr8')
H, W = img.shape[:2]
objs = self.engine.DetectWithImage(
PIL.Image.fromarray(img), threshold=self.score_thresh,
keep_aspect_ratio=True, relative_coord=True,
top_k=self.top_k)
bboxes = []
scores = []
labels = []
rect_msg = RectArray(header=msg.header)
for obj in objs:
x_min, y_min, x_max, y_max = obj.bounding_box.flatten().tolist()
x_min = int(np.round(x_min * W))
y_min = int(np.round(y_min * H))
x_max = int(np.round(x_max * W))
y_max = int(np.round(y_max * H))
bboxes.append([y_min, x_min, y_max, x_max])
scores.append(obj.score)
labels.append(self.label_ids.index(int(obj.label_id)))
rect = Rect(
x=x_min, y=y_min,
width=x_max-x_min, height=y_max-y_min)
rect_msg.rects.append(rect)
bboxes = np.array(bboxes)
scores = np.array(scores)
labels = np.array(labels)
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[l] for l in labels],
label_proba=scores)
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.visualize:
fig = plt.figure(
tight_layout={'pad': 0})
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.axis('off')
fig.add_axes(ax)
vis_bbox(
img[:, :, ::-1].transpose((2, 0, 1)),
bboxes, labels, scores,
label_names=self.label_names, ax=ax)
fig.canvas.draw()
w, h = fig.canvas.get_width_height()
vis_img = np.fromstring(
fig.canvas.tostring_rgb(), dtype=np.uint8)
vis_img.shape = (h, w, 3)
fig.clf()
plt.close()
vis_msg = self.bridge.cv2_to_imgmsg(vis_img, 'rgb8')
# BUG: https://answers.ros.org/question/316362/sensor_msgsimage-generates-float-instead-of-int-with-python3/ # NOQA
vis_msg.step = int(vis_msg.step)
vis_msg.header = msg.header
self.pub_image.publish(vis_msg)
def image_cb(self, msg):
if self.profiling:
rospy.loginfo(
"callback start: incomming msg is %s msec behind" %
((rospy.Time.now() - msg.header.stamp).to_sec() * 1000.0))
tprev = time.time()
try:
# transform image to RGB, float, CHW
img = self.cv_bridge.imgmsg_to_cv2(msg, desired_encoding="rgb8")
img = np.asarray(img, dtype=np.float32)
img = img.transpose(2, 0, 1) # (H, W, C) -> (C, H, W)
except Exception as e:
rospy.logerr("Failed to convert image: %s" % str(e))
return
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("convert",
(tcur - tprev) * 1000))
tprev = tcur
bboxes, labels, scores = self.model.predict([img])
bboxes, labels, scores = bboxes[0], labels[0], scores[0]
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("predict",
(tcur - tprev) * 1000))
tprev = tcur
rect_msg = RectArray(header=msg.header)
for bbox in bboxes:
rect = Rect(x=bbox[1],
y=bbox[0],
width=bbox[3] - bbox[1],
height=bbox[2] - bbox[0])
rect_msg.rects.append(rect)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make rect msg",
(tcur - tprev) * 1000))
tprev = tcur
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[l] for l in labels],
label_proba=scores,
)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make cls msg",
(tcur - tprev) * 1000))
tprev = tcur
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("publish msg",
(tcur - tprev) * 1000))
tprev = tcur
if self.visualize:
self.publish_bbox_image(img, bboxes, labels, scores)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("callback end",
(tcur - tprev) * 1000))
tprev = tcur
def image_cb(self, msg):
if self.profiling:
rospy.loginfo(
"callback start: incomming msg is %s msec behind" %
((rospy.Time.now() - msg.header.stamp).to_sec() * 1000.0))
tprev = time.time()
try:
# transform image to RGB, float, CHW
img = self.cv_bridge.imgmsg_to_cv2(msg, desired_encoding="rgb8")
img = np.asarray(img, dtype=np.float32)
img = img.transpose(2, 0, 1) # (H, W, C) -> (C, H, W)
except Exception as e:
rospy.logerr("Failed to convert image: %s" % str(e))
return
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("convert",
(tcur - tprev) * 1000))
tprev = tcur
if self.gpu >= 0:
chainer.cuda.get_device_from_id(self.gpu).use()
bboxes, labels, scores = self.model.predict([img])
bboxes, labels, scores = bboxes[0], labels[0], scores[0]
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("predict",
(tcur - tprev) * 1000))
tprev = tcur
labels_msg = LabelArray(header=msg.header)
for l in labels:
l_name = self.label_names[l]
labels_msg.labels.append(Label(id=l, name=l_name))
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make labels msg",
(tcur - tprev) * 1000))
cluster_indices_msg = ClusterPointIndices(header=msg.header)
H = img.shape[1]
W = img.shape[2]
for bbox in bboxes:
ymin = max(0, int(np.floor(bbox[0])))
xmin = max(0, int(np.floor(bbox[1])))
ymax = min(H, int(np.ceil(bbox[2])))
xmax = min(W, int(np.ceil(bbox[3])))
indices = [
range(W * y + xmin, W * y + xmax) for y in range(ymin, ymax)
]
indices = np.array(indices, dtype=np.int32).flatten()
indices_msg = PointIndices(header=msg.header, indices=indices)
cluster_indices_msg.cluster_indices.append(indices_msg)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make cluster_indices msg",
(tcur - tprev) * 1000))
tprev = tcur
rect_msg = RectArray(header=msg.header)
for bbox in bboxes:
rect = Rect(x=bbox[1],
y=bbox[0],
width=bbox[3] - bbox[1],
height=bbox[2] - bbox[0])
rect_msg.rects.append(rect)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make rect msg",
(tcur - tprev) * 1000))
tprev = tcur
cls_msg = ClassificationResult(
header=msg.header,
classifier=self.classifier_name,
target_names=self.label_names,
labels=labels,
label_names=[self.label_names[l] for l in labels],
label_proba=scores,
)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("make cls msg",
(tcur - tprev) * 1000))
tprev = tcur
self.pub_labels.publish(labels_msg)
self.pub_indices.publish(cluster_indices_msg)
self.pub_rects.publish(rect_msg)
self.pub_class.publish(cls_msg)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("publish msg",
(tcur - tprev) * 1000))
tprev = tcur
if self.visualize:
self.publish_bbox_image(img, bboxes, labels, scores, msg.header)
if self.profiling:
tcur = time.time()
rospy.loginfo("%s: elapsed %f msec" % ("callback end",
(tcur - tprev) * 1000))
tprev = tcur
