def test_process_image(self):
train_gen = MongoDBGenerator(
self.collection_details,
self.train_data,
batch_size=12,
processors=[CenterTrackerProcess(self.params)]
)
batch_x, batch_y = train_gen[0]
batch_size = len(batch_x["img"])
for i in range(batch_size):
# input_data is a list in this case and consist of:
# [0]: img current frame
# [1]: img prev frame
# [3]: single channel heatmap
curr_img = batch_x["img"][i]
prev_img = batch_x["prev_img"][i]
mask_img = to_3channel(batch_y[i], OD_CLASS_MAPPING, 0.1, True)
prev_mask_img = to_3channel(batch_x["prev_heatmap"][i], OrderedDict([("obj", (255, 255, 255))]), 0.1, True)
cv2.imshow("img", curr_img.astype(np.uint8))
cv2.imshow("prev_img", prev_img.astype(np.uint8))
cv2.imshow("prev_heatmap", prev_mask_img)
cv2.imshow("mask", mask_img)
cv2.waitKey(0)
def show_semseg(self, inp, y_true, y_pred):
inp_img = cv2.cvtColor(inp[0].numpy().astype(np.uint8), cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_img = pygame.surfarray.make_surface(inp_img)
self.display.blit(surface_img, (0, 0))
semseg_true = cv2.cvtColor(to_3channel(y_true[0].numpy(), List(SEMSEG_CLASS_MAPPING.items())), cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_y_true = pygame.surfarray.make_surface(semseg_true)
self.display.blit(surface_y_true, (0, self.params.INPUT_HEIGHT))
semseg_pred = cv2.cvtColor(to_3channel(y_pred[0].numpy(), List(SEMSEG_CLASS_MAPPING.items())), cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_y_pred = pygame.surfarray.make_surface(semseg_pred)
self.display.blit(surface_y_pred, (0, int(self.params.INPUT_HEIGHT + self.params.MASK_HEIGHT)))
self.step_counter += 1
if self.step_counter % 2000 == 0:
pygame.image.save(self.display, f"{self._storage_path}/train_result_{self.step_counter}.png")
pygame.display.flip()
def test_process_image(self):
train_gen = MongoDBGenerator(
[self.collection_details], [self.train_data],
batch_size=30,
processors=[
ProcessImages(self.params,
start_augmentation=[0, 0],
show_debug_img=True)
])
for batch_x, batch_y in train_gen:
print("New batch")
for i in range(len(batch_x[0])):
assert len(batch_x[0]) > 0
img1 = batch_x[0][i]
heatmap = np.array(
batch_y[i]
[:, :, :1]) # needed because otherwise numba makes mimimi
heatmap = to_3channel(heatmap, List([("object", (0, 0, 255))]),
0.01, True, False)
weights = np.stack([batch_y[i][:, :, -1]] * 3, axis=-1)
f, (ax1, ax2, ax3) = plt.subplots(1, 3)
ax1.imshow(
cv2.cvtColor(batch_x[0][i].astype(np.uint8),
cv2.COLOR_BGR2RGB))
ax2.imshow(cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB))
ax3.imshow(weights)
plt.show()
def test_process_image(self):
train_gen = MongoDBGenerator(
[self.collection_details],
[self.td],
batch_size=10,
processors=[ProcessImages(self.params, [0, 0])],
shuffle_data=True
)
for batch_x, batch_y in train_gen:
print("New batch")
for i in range(len(batch_x[0])):
assert len(batch_x[0]) > 0
img1 = batch_x[0][i]
# [0: centernet_channel]: centernet
# [centernet_channels+1]: centernet weights
# [centernet_channels+1: semseg_classes]: semseg_masks
# [semseg_classes+1]: semseg weights
# [-1]: depth
semseg_cls_items = List(SEMSEG_CLASS_MAPPING.items())
cn_idx_end = self.params.cn_params.mask_channels()
semseg_idx_end = (cn_idx_end+1) + len(semseg_cls_items)
cn_heatmap = np.array(batch_y[i][:, :, :cn_idx_end])
cn_weights = np.stack([batch_y[i][:, :, cn_idx_end]]*3, axis=-1)
cn_heatmap = to_3channel(cn_heatmap, List([("object", (0, 0, 255))]), 0.01, True, False)
semseg_mask = np.array(batch_y[i][:, :, cn_idx_end+1:semseg_idx_end])
semseg_mask = to_3channel(semseg_mask, List(SEMSEG_CLASS_MAPPING.items()), 0.01, True, False)
semseg_weights = np.stack([batch_y[i][:, :, semseg_idx_end]]*3, axis=-1)
depth_map = np.array(batch_y[i][:, :, -1])
f, ((ax11, ax12), (ax21, ax22), (ax31, ax32)) = plt.subplots(3, 2)
ax11.imshow(cv2.cvtColor(batch_x[0][i].astype(np.uint8), cv2.COLOR_BGR2RGB))
ax12.imshow(cv2.cvtColor(cmap_depth(depth_map, vmin=0.1, vmax=255.0), cv2.COLOR_BGR2RGB))
ax21.imshow(cv2.cvtColor(cn_heatmap, cv2.COLOR_BGR2RGB))
ax22.imshow(cn_weights)
ax31.imshow(cv2.cvtColor(semseg_mask, cv2.COLOR_BGR2RGB))
ax32.imshow(semseg_weights)
plt.show()
def test_process_image(self):
train_gen = MongoDBGenerator([self.collection_details],
[self.train_data],
batch_size=10,
processors=[ProcessImages(self.params)])
batch_x, batch_y = train_gen[0]
for i, input_data in enumerate(batch_x):
assert len(input_data) > 0
cls_items = List(SEMSEG_CLASS_MAPPING.items())
nb_classes = len(cls_items)
semseg_mask = np.array(
batch_y[i]
[:, :, :-1]) # needed because otherwise numba makes mimimi
mask_img = to_3channel(semseg_mask, cls_items, threshold=0.999)
pos_mask = batch_y[i][:, :, -1]
f, (ax1, ax2, ax3) = plt.subplots(1, 3)
ax1.imshow(
cv2.cvtColor(input_data.astype(np.uint8), cv2.COLOR_BGR2RGB))
ax2.imshow(cv2.cvtColor(mask_img, cv2.COLOR_BGR2RGB))
ax3.imshow(pos_mask, cmap="gray", vmin=0.0, vmax=1.0)
plt.show()
decoded_mask = np.frombuffer(doc["mask"], np.uint8)
gt_mask = cv2.imdecode(decoded_mask, cv2.IMREAD_COLOR)
img, roi = resize_img(img, args.img_width, args.img_height,
args.offset_bottom)
if is_tf_lite:
img_input = img
interpreter.set_tensor(input_details[0]['index'], [img_input])
#input_shape = input_details[0]['shape']
start_time = time.time()
interpreter.invoke()
elapsed_time = time.time() - start_time
# The function `get_tensor()` returns a copy of the tensor data. Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])
semseg_img = to_3channel(output_data[0].astype(np.float64),
List(SEMSEG_CLASS_MAPPING.items()))
else:
img_arr = np.array([img])
start_time = time.time()
raw_result = model.predict(img_arr)
elapsed_time = time.time() - start_time
semseg_img = raw_result[0]
semseg_img = to_3channel(semseg_img,
List(SEMSEG_CLASS_MAPPING.items()),
threshold=0.5)
# in case we are sure, there is no care parts (e.g. nuscenes) mask these detections out
# semseg_img[np.where((semseg_img==[255, 0, 204]).all(axis=2))] = [0, 0, 0]
# # somehow the model likes to put non drivable holes in front of us, mask these out
# semseg_cut = semseg_img[182:, 240:400, :]
def show_od(self, inp, y_true, y_pred):
heatmap_true = np.array(
y_true[0][:, :, :1]) # needed because otherwise numba makes mimimi
heatmap_true = to_3channel(heatmap_true,
List([("object", (0, 0, 255))]), 0.01, True,
False)
weights = np.stack([y_true[0][:, :, -1]] * 3, axis=-1)
heatmap_pred = np.array(y_pred[0][:, :, :1])
heatmap_pred = to_3channel(heatmap_pred,
List([("object", (0, 0, 255))]), 0.01, True,
False)
# display img
show_inp_img = inp[0][0].numpy()
show_inp_img = show_inp_img.astype(np.uint8)
# get objects from y_pred
if self._process_output:
roi = Roi()
prediction = np.array(y_pred[0])
objects = post_processing.process_2d_output(
prediction, roi, self.params, 0.2)
for obj in objects:
color = list(OD_CLASS_MAPPING.values())[obj["cls_idx"]]
# top_center = (int(obj["bottom_center"][0]), int(obj["bottom_center"][1] - obj["center_height"]))
# bottom_left = (int(obj["bottom_left"][0]), int(obj["bottom_left"][1]))
# bottom_center = (int(obj["bottom_center"][0]), int(obj["bottom_center"][1]))
# bottom_right = (int(obj["bottom_right"][0]), int(obj["bottom_right"][1]))
# cv2.line(show_inp_img, bottom_left, bottom_center, (0, 255, 0) , 1)
# cv2.line(show_inp_img, bottom_center, bottom_right, (0, 255, 0) , 1)
# cv2.line(show_inp_img, bottom_center, top_center, (0, 255, 0) , 1)
top_left = (int(obj["fullbox"][0]), int(obj["fullbox"][1]))
bottom_right = (int(obj["fullbox"][0] + obj["fullbox"][2]),
int(obj["fullbox"][1] + obj["fullbox"][3]))
cv2.rectangle(show_inp_img, top_left, bottom_right, color, 1)
cv2.circle(show_inp_img,
(int(obj["center"][0]), int(obj["center"][1])), 2,
color, 1)
inp_img = cv2.cvtColor(show_inp_img, cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_img = pygame.surfarray.make_surface(inp_img)
self._display.blit(surface_img, (0, 0))
# display heatmap y_pred
heatmap_pred = cv2.cvtColor(heatmap_pred,
cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_y_pred = pygame.surfarray.make_surface(heatmap_pred)
self._display.blit(surface_y_pred, (0, self.params.INPUT_HEIGHT))
# display heatmap y_true
heatmap_true = cv2.cvtColor(heatmap_true,
cv2.COLOR_BGR2RGB).swapaxes(0, 1)
surface_y_true = pygame.surfarray.make_surface(heatmap_true)
self._display.blit(
surface_y_true,
(0, int(self.params.INPUT_HEIGHT + self.params.MASK_HEIGHT)))
self._step_counter += 1
if self._step_counter % 2000 == 0:
pygame.image.save(
self._display,
f"{self._storage_path}/train_result_{self._step_counter}.png")
pygame.display.flip()