def plot_predictions(image_paths, images, det_boxes, det_labels, det_scores,
true_boxes, true_labels, class_names):
assert len(det_boxes) == len(det_labels) == len(det_scores) == len(
true_boxes) == len(true_labels)
for i in range(len(image_paths)):
use_original = False
output_path = BASE_PATH + "/outputs"
if not os.path.exists(output_path):
os.mkdir(output_path)
save_path = "{}/{}".format(output_path, image_paths[i].split('/')[-1])
# Output
p_bboxes = det_boxes[i].cpu().data.numpy()
p_labels = det_labels[i].cpu().data.numpy()
t_bboxes = true_boxes[i].cpu().data.numpy()
t_labels = true_labels[i].cpu().data.numpy()
if use_original:
input_img = img2img(image_paths[i])
# Rescale boxes
p_bboxes = rescale_boxes(p_bboxes,
current_shape=input_img.shape[:2],
original_shape=input_img.shape[:2],
relxyxy=True)
t_bboxes = rescale_boxes(t_bboxes,
current_shape=input_img.shape[:2],
original_shape=input_img.shape[:2],
relxyxy=True)
plot_bboxes(use_original,
p_bboxes,
class_ids=p_labels,
class_names=class_names,
show_results=False,
coords_rel=False,
t_bboxes=t_bboxes,
title="Detection + ground truth ({})".format(
image_paths[i]),
save_path=save_path)
else:
input_img = img2img(images[i])
plot_bboxes(input_img,
p_bboxes,
class_ids=p_labels,
class_names=class_names,
show_results=False,
coords_rel=True,
t_bboxes=t_bboxes,
title="Detection + ground truth ({})".format(
image_paths[i]),
save_path=save_path)
def make_publish_msg(self, img_detections, bev):
for detections in img_detections:
if detections is None:
continue
# Rescale boxes to original image
detections = utils.rescale_boxes(detections, self.img_size,
bev.shape[:2])
for x, y, w, l, im, re, conf, cls_conf, cls_pred in detections:
yaw = np.arctan2(im, re)
h = self.classes_h[int(cls_pred)]
# Draw rotated box
bev_utils.drawRotatedBox(bev, x, y, w, l, yaw,
cnf.colors[int(cls_pred)])
detection_msg = DetectedObject()
detection_msg.pose.position.x = x
detection_msg.pose.position.y = y
detection_msg.pose.position.z = h / 2.
detection_msg.pose.orientation.x = .0
detection_msg.pose.orientation.y = .0
detection_msg.pose.orientation.z = yaw
detection_msg.pose.orientation.w = 1.
detection_msg.dimensions.x = w
detection_msg.dimensions.y = l
detection_msg.dimensions.z = h
detection_msg.label = self.classes[int(cls_pred)]
detection_msg.score = conf
detection_msg.id = self.dets_id
self.detection_results.objects.append(detection_msg)
self.dets_id += 1
def test(img_path: str = 'data/custom/images/', anno_path: str = 'data/custom/annos/') -> None:
# transform something
initialize(img_path=img_path, anno_path=anno_path, split_ratio=1.0)
# some common config
iou_thres = 0.5
conf_thres = 0.01
nms_thres = 0.5
img_size = 416
batch_size = 16
device = tc.device('cuda' if tc.cuda.is_available() else 'cpu')
# other paths
weights_path = 'my/yolov3_ckpt_1.pth'
model_def = 'config/custom.cfg'
test_path = 'data/custom/train.txt'
class_path = 'data/custom/classes.names'
# load model
class_names = load_classes(class_path)
model = Darknet(model_def).to(device)
model.load_state_dict(tc.load(weights_path))
imgs, img_detections = detect(
model=model,
path=img_path,
conf_thres=conf_thres,
nms_thres=nms_thres,
img_size=img_size,
batch_size=batch_size,
n_cpu=8,
device=device,
)
os.makedirs('predicted_file', exist_ok=True)
class1 = open('predicted_file/det_test_core.txt', 'w')
class2 = open('predicted_file/det_test_coreless.txt', 'w')
for path, boxes in zip(imgs, img_detections):
w, h = Image.open(path).size
boxes = rescale_boxes(boxes, img_size, (h, w))
for box in boxes:
line = [
# os.path.split(path)[1].split('_')[1].split('.')[0], # no prefix
os.path.split(path)[1].split('.')[0], # with prefix 'core_' or 'coreless_'
f'{box[4].tolist():.3f}', # conf
f'{box[0].tolist():.1f}',
f'{box[1].tolist():.1f}',
f'{box[2].tolist():.1f}',
f'{box[3].tolist():.1f}',
]
if box[-1] == 0.0:
class1.write(' '.join(line) + '\n')
elif box[-1] == 1.0:
class2.write(' '.join(line) + '\n')
class1.close()
class2.close()
print('Output file saved.\n')
def save_imgs(self, opt, candidates, image, target_sz, f):
# Bounding-box colors
color = (0.6, 0.3, 0.3, 1)
print("\nSaving images:")
# Create plot
plt.figure()
fig, ax = plt.subplots(1)
ax.imshow(image)
# Draw bounding boxes and labels of detections
if candidates is not None:
# Rescale boxes to original image
candidates = rescale_boxes(candidates, opt.img_size,
image.shape[:2])
candidate_sz = candidates[:, 2:4] - candidates[:, :2]
target_sz1 = torch.Tensor([target_sz[1],
target_sz[0]]).expand_as(candidate_sz)
# double filtering
valid_mask_1 = ~(torch.any(
candidate_sz > opt.one_scale_thres * target_sz1, dim=1)
| torch.any(candidate_sz <
1 / opt.one_scale_thres * target_sz1,
dim=1))
valid_mask_2 = ~(torch.all(
candidate_sz > opt.two_scale_thres * target_sz1, dim=1)
| torch.all(candidate_sz <
1 / opt.two_scale_thres * target_sz1,
dim=1))
valid_mask = valid_mask_1 & valid_mask_2
candidates = candidates[valid_mask]
for x1, y1, x2, y2, conf in candidates:
box_w = x2 - x1
box_h = y2 - y1
# Create a Rectangle patch
bbox = patches.Rectangle((x1, y1),
box_w,
box_h,
linewidth=2,
edgecolor=color,
facecolor="none")
# Add the bbox to the plot
ax.add_patch(bbox)
# Save generated image with detections
plt.axis("off")
plt.gca().xaxis.set_major_locator(NullLocator())
plt.gca().yaxis.set_major_locator(NullLocator())
plt.savefig(f"my_output1/{f}.png", bbox_inches="tight", pad_inches=0.0)
plt.close()
def test(self, image, orig_cv):
img = transforms.ToTensor()(image)
img, _ = pad_to_square(img, 0)
img = F.interpolate(img.unsqueeze(0),
size=self.img_size,
mode="nearest").squeeze(0)
img.unsqueeze_(0)
prev_time = time.time()
img = Variable(img.type(self.Tensor))
with torch.no_grad():
detections = self.model_DarkNet(img)
detections = non_max_suppression(detections, config['conf_thres'],
config['nms_thres'])
for detection in detections:
img = orig_cv
if detection is not None:
detection = rescale_boxes(detection, self.img_size,
img.shape[:2])
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detection:
img = cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0),
3)
if conf > 0.99:
cropped_image = image.crop(box=(x1.item(), y1.item(),
x2.item(), y2.item()))
oldX, oldY = cropped_image.size
cropped_resized = cropped_image.resize(
(128, 128), resample=Image.BILINEAR)
test_tensor = self.transform_image(cropped_resized)
test_tensor.unsqueeze_(0)
test_image = test_tensor.to(self.device)
output = self.model_RegNet(test_image)
#tensor_out = output['heatmaps'].cpu().data.numpy()
vec_2d = output['vector_2d'].cpu().data.numpy()
vec_2d = vec_2d * 128
pred_kpts_2d = vec_2d[0].reshape(-1, 2)
x = np.array(
pred_kpts_2d[:, 0]) * (oldX / 128) + x1.item()
y = np.array(
pred_kpts_2d[:, 1]) * (oldY / 128) + y1.item()
plt.imshow(image, aspect='auto')
plt.gca().add_patch(
Rectangle((x1.item(), y1.item()),
x2.item() - x1.item(),
y2.item() - y1.item(),
linewidth=2,
edgecolor='r',
facecolor='none'))
plot_2d(x, y, x, y)
plt.show()
# Log progress
#current_time = time.time()
#inference_time = datetime.timedelta(seconds=current_time - prev_time)
#prev_time = current_time
#print(inference_time)
print("FPS: ", 1.0 / (time.time() - prev_time))
def detect_object(self, img, conf_thres, nms_thres):
img = transforms.ToTensor()(img)
img, _ = pad_to_square(img, 0)
# Resize
img = resize(img, self.img_size)
img = img.to(self.device)
img = img.unsqueeze(0)
detection = self.model_yolov3(img)
# From [x,y,w,h] to [x_l,y_l,x_r,y_r]
detection = non_max_suppression(detection, conf_thres, nms_thres)
if detection[0] is not None:
detection = rescale_boxes(detection[0], self.img_size, (600, 600))
return detection
def infer_video(model,
video_path,
save_path,
iou_thres,
conf_thres,
nms_thres,
img_size,
batch_size=1):
model.eval()
if os.path.isfile(video_path):
cap = cv2.VideoCapture(video_path)
else:
raise Exception('no such video')
tp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
stream = cv2.VideoWriter(os.path.join(save_path, 'out.mp4'),
cv2.VideoWriter_fourcc(*'MJPG'), 20.0,
(int(cap.get(3)), int(cap.get(4))))
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
while cap.isOpened():
ret, frame = cap.read()
if ret:
print(frame.shape)
img = transform(frame).to(device)
print(img.shape)
img, _ = pad_to_square(img, 0)
print(img.shape)
img = F.interpolate(img.unsqueeze(0), img_size,
mode='nearest').squeeze()
print(img.shape)
img = img.unsqueeze(0)
with torch.no_grad():
output = model(img)
output = non_max_suppression(output,
conf_thres=conf_thres,
nms_thres=nms_thres)
output = rescale_boxes(output[0], img_size, frame.shape[:2])
# print(output)
# print(len(output[0][0]))
img = vis_frame(frame, output)
stream.write(img)
else:
break
cap.release()
stream.release()
def detect_and_draw(model, bev_maps, Tensor, is_front=True):
# If back side bev, flip around vertical axis
if not is_front:
bev_maps = torch.flip(bev_maps, [2, 3])
imgs = Variable(bev_maps.type(Tensor))
# Get Detections
img_detections = []
with torch.no_grad():
detections = model(imgs)
detections = utils.non_max_suppression_rotated_bbox(
detections, opt.conf_thres, opt.nms_thres)
img_detections.extend(detections)
# Only supports single batch
display_bev = np.zeros((cnf.BEV_WIDTH, cnf.BEV_WIDTH, 3))
bev_map = bev_maps[0].numpy()
display_bev[:, :, 2] = bev_map[0, :, :] # r_map
display_bev[:, :, 1] = bev_map[1, :, :] # g_map
display_bev[:, :, 0] = bev_map[2, :, :] # b_map
display_bev *= 255
display_bev = display_bev.astype(np.uint8)
for detections in img_detections:
if detections is None:
continue
# Rescale boxes to original image
detections = utils.rescale_boxes(detections, opt.img_size,
display_bev.shape[:2])
for x, y, w, l, im, re, conf, cls_conf, cls_pred in detections:
yaw = np.arctan2(im, re)
# Draw rotated box
bev_utils.drawRotatedBox(display_bev, x, y, w, l, yaw,
cnf.colors[int(cls_pred)])
return display_bev, img_detections
def Yolo_detect(model,
camInputFrame,
img_size=416,
conf_thres=0.8,
nms_thres=0.4):
img = transforms.ToTensor()(Image.fromarray(camInputFrame))
# Pad to square resolution
img, _ = pad_to_square(img, 0)
# Resize
img = resize(img, img_size)
img = img.unsqueeze(0) #(1,3,416.419)
input_imgs = img.cuda()
with torch.no_grad():
detections = model(input_imgs)
detections = non_max_suppression(detections, conf_thres, nms_thres)
if detections is not None:
detections = detections[0]
if detections is not None:
detections = rescale_boxes(detections, img_size,
camInputFrame.shape[:2])
return detections
colors = [cmap(i) for i in np.linspace(0, 1, 20)]
print("\nSaving images:")
"""Iterate through images and save plot of detections"""
for img_i, (path, detections) in enumerate(zip(imgs, img_detections)):
print("(%d) Image: '%s'" % (img_i, path))
"""Create plot"""
img = np.array(Image.open(path))
plt.figure()
fig, ax = plt.subplots(1)
ax.imshow(img)
"""Draw bounding boxes and labels of detections"""
if detections is not None:
"""Rescale boxes to original image"""
detections = rescale_boxes(detections, opt.img_size, img.shape[:2])
unique_labels = detections[:, -1].cpu().unique()
n_cls_preds = len(unique_labels)
bbox_colors = random.sample(colors, n_cls_preds)
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
print("\t+ Label: %s, Conf: %.5f" %
(classes[int(cls_pred)], cls_conf.item()))
box_w = x2 - x1
box_h = y2 - y1
color = bbox_colors[int(
np.where(unique_labels == int(cls_pred))[0])]
"""Create a Rectangle patch"""
bbox = patches.Rectangle((x1, y1),
box_w,
bev_maps = torch.squeeze(bev_maps).numpy()
RGB_Map = np.zeros((cnf.BEV_WIDTH, cnf.BEV_WIDTH, 3))
RGB_Map[:, :, 2] = bev_maps[0, :, :] # r_map
RGB_Map[:, :, 1] = bev_maps[1, :, :] # g_map
RGB_Map[:, :, 0] = bev_maps[2, :, :] # b_map
RGB_Map *= 255
RGB_Map = RGB_Map.astype(np.uint8)
for detections in img_detections:
if detections is None:
continue
# Rescale boxes to original image
detections = utils.rescale_boxes(detections, opt.img_size, RGB_Map.shape[:2])
for x, y, w, l, im, re, conf, cls_conf, cls_pred in detections:
yaw = np.arctan2(im, re)
# Draw rotated box
bev_utils.drawRotatedBox(RGB_Map, x, y, w, l, yaw, cnf.colors[int(cls_pred)])
img2d = cv2.imread(img_paths[0])
calib = kitti_utils.Calibration(img_paths[0].replace(".png", ".txt").replace("image_2", "calib"))
objects_pred = predictions_to_kitti_format(img_detections, calib, img2d.shape, opt.img_size)
img2d = mview.show_image_with_boxes(img2d, objects_pred, calib, False)
cv2.imshow("bev img", RGB_Map)
cv2.imshow("img2d", img2d)
if cv2.waitKey(0) & 0xFF == 27:
with torch.no_grad():
detections = model(img)
detections = non_max_suppression(detections, opt.conf_thres, opt.nms_thres)
# Bounding-box colors
cmap = plt.get_cmap("tab20b")
colors = [cmap(i) for i in np.linspace(0, 1, 20)]
# Create plot
# img = np.array(frame)
# plt.figure()
fig, ax = plt.subplots(1)
ax.imshow(np.array(frame))
# Draw bounding boxes and labels of detections
if detections[0] is not None:
# Rescale boxes to original image
detections = rescale_boxes(detections[0], opt.img_size, frame.size[::-1])
unique_labels = detections[:, -1].cpu().unique()
n_cls_preds = len(unique_labels)
# bbox_colors = random.sample(colors, n_cls_preds)
print('-----------------')
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
print("\t+ Label: %s, Conf: %.5f" % (classes[int(cls_pred)], cls_conf.item()))
box_w = x2 - x1
box_h = y2 - y1
# color = bbox_colors[int(np.where(unique_labels == int(cls_pred))[0])]
color = colors[int(cls_pred)%20]
# Create a Rectangle patch
bbox = patches.Rectangle((x1, y1), box_w, box_h, linewidth=2, edgecolor=color, facecolor="none")
np.array(detections[0][:, :]))
print(track_bbs_ids)
# Bounding-box colors
cmap = plt.get_cmap("tab20b")
colors = [cmap(i) for i in np.linspace(0, 1, 20)]
# Create plot
# img = np.array(frame)
# plt.figure()
fig, ax = plt.subplots(1)
ax.imshow(np.array(frame))
# Draw bounding boxes and labels of detections
if detections[0] is not None:
# Rescale boxes to original image
#detections = rescale_boxes(detections[0], opt.img_size, frame.size[::-1])
detections = rescale_boxes(track_bbs_ids, opt.img_size,
frame.size[::-1])
#unique_labels = detections[:, -1].cpu().unique()
#unique_labels = detections[:, -1].unique()
#n_cls_preds = len(unique_labels)
#bbox_colors = random.sample(colors, n_cls_preds)
print('-----------------')
for x1, y1, x2, y2, tid, cls in detections:
print("\t+ Conf: %.5f" % (int(tid.item())))
box_w = x2 - x1
box_h = y2 - y1
#color = bbox_colors[int(np.where(unique_labels == int(cls_pred))[0])]
color = colors[int(tid) % 20]
# Create a Rectangle patch
def main():
# load model
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
model = Darknet(cfg.MODEL, img_size=cfg.SIZE).to(device)
model.load_darknet_weights(cfg.WEIGHTS)
model.eval()
# coco classes
classes = load_classes(cfg.CLASSES)
# animals and person
app_classes = [0, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
# tensor type
Tensor = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor
# create video capture
cap = cv2.VideoCapture('udp://127.0.0.1:5000', cv2.CAP_FFMPEG)
if not cap.isOpened():
print('VideoCapture not opened')
exit(-1)
# preprocess pipeline
t = transforms.Compose([
transforms.Resize((cfg.SIZE, cfg.SIZE)),
transforms.ToTensor()
])
# tracker
tracker = Sort()
# bbox colors
colors=[
(255,0,0),
(0,255,0),
(0,0,255),
(255,0,255),
(128,0,0),
(0,128,0),
(0,0,128),
(128,0,128),
(128,128,0),
(0,128,128)
]
# process stream
while True:
# read frame
ret, frame = cap.read()
# frame = cv2.flip(cv2.flip(frame, 0), 1)
orig = frame
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = Image.fromarray(frame)
# process image
img = t(img).unsqueeze(0).type(Tensor)
with torch.no_grad():
detections = model(img)
detections = non_max_suppression(detections, cfg.CONF, cfg.NMS)
detections = detections[0]
if detections is not None:
# track objects
tracked_objects = tracker.update(detections.cpu())
det = rescale_boxes(tracked_objects, cfg.SIZE, frame.shape[:2])
for x1, y1, x2, y2, obj_id, cls_pred in det:
# ignore not necessary classes
if int(cls_pred) not in app_classes:
continue
# draw bbox
color = colors[int(obj_id) % len(colors)]
cls = classes[int(cls_pred)]
x1, x2, y1, y2 = int(x1), int(x2), int(y1), int(y2)
cv2.rectangle(orig, (x1, y1), (x2, y2), color, 2)
cv2.putText(
orig,
cls + '-' + str(int(obj_id)),
(x1, y1 - 10),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(255, 255, 255),
3
)
cv2.imshow('YoloV3', orig)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
bev_maps = torch.squeeze(bev_maps).numpy()
RGB_Map = np.zeros((cnf.BEV_WIDTH, cnf.BEV_WIDTH, 3))
RGB_Map[:, :, 2] = bev_maps[0, :, :] # r_map
RGB_Map[:, :, 1] = bev_maps[1, :, :] # g_map
RGB_Map[:, :, 0] = bev_maps[2, :, :] # b_map
RGB_Map *= 255
RGB_Map = RGB_Map.astype(np.uint8)
for detections in img_detections:
if detections is None:
continue
# Rescale boxes to original image
detections = utils.rescale_boxes(detections, cnf.BEV_WIDTH,
RGB_Map.shape[:2])
for x, y, w, l, im, re, conf, cls_conf, cls_pred in detections:
yaw = np.arctan2(im, re)
# Draw rotated box
bev_utils.drawRotatedBox(RGB_Map, x, y, w, l, yaw,
cnf.colors[int(cls_pred)])
img2d = cv2.imread(img_paths[0])
calib = kitti_utils.Calibration(img_paths[0].replace(
".png", ".txt").replace("image_2", "calib"))
objects_pred = predictions_to_kitti_format(img_detections, calib,
img2d.shape, cnf.BEV_WIDTH)
img2d = mview.show_image_with_boxes(img2d, objects_pred, calib, False)
cv2.imwrite("test_%s.jpeg" % index, img2d)
def plot_detections(img_paths, img_detections, class_names, output_path):
# bbox colors
cmap = plt.get_cmap("tab20b")
colors = [cmap(i) for i in np.linspace(0, 1, len(class_names))]
print("\nSaving images:")
os.makedirs(output_path, exist_ok=True)
# Iterate through images and save plot of detections
for img_i, (path, detections) in enumerate(zip(img_paths, img_detections)):
print("(%d) Image: '%s'" % (img_i, path))
# Create plot
img = np.array(Image.open(path))
plt.figure()
fig, ax = plt.subplots(1)
ax.imshow(img)
# Draw bounding boxes and labels of detections
if detections is not None:
# Rescale boxes to original image
detections = rescale_boxes(detections, opt.img_size, img.shape[:2])
unique_labels = detections[:, -1].cpu().unique()
n_cls_preds = len(unique_labels)
bbox_colors = random.sample(colors, n_cls_preds)
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
print("\t+ Label: %s, Conf: %.5f" %
(class_names[int(cls_pred)], cls_conf.item()))
box_w = x2 - x1
box_h = y2 - y1
color = bbox_colors[int(
np.where(unique_labels == int(cls_pred))[0])]
# Create a Rectangle patch
bbox = patches.Rectangle((x1, y1),
box_w,
box_h,
linewidth=2,
edgecolor=color,
facecolor="none")
# Add the bbox to the plot
ax.add_patch(bbox)
# Add label
plt.text(
x1,
y1,
s=class_names[int(cls_pred)],
color="white",
verticalalignment="top",
bbox={
"color": color,
"pad": 0
},
)
# Save generated image with detections
plt.axis("off")
plt.gca().xaxis.set_major_locator(NullLocator())
plt.gca().yaxis.set_major_locator(NullLocator())
filename = path.split("/")[-1].split(".")[0]
fig.savefig("{}/{}.png".format(output_path, filename),
bbox_inches="tight",
pad_inches=0.0)
plt.close()
