def video(args):
global net_img_g, net_out_g, net_thread_start
h, w = LPD.size
if args.trt: # tensorRT Inference can't in thread
import numpy as np
from yolo_modules.tensorrt_module import do_inference_wrapper
from yolo_modules.tensorrt_module import get_engine_wrapper
engine_wrapper = get_engine_wrapper(args.version)
net_out_shape = (1, LPD.LP_slice_point[-1], h / 2**LPD.num_downsample,
w / 2**LPD.num_downsample)
else:
import mxnet
# mx_resize = mxnet.image.ForceResizeAug((w, h), interp=2) # not always available
net = yolo_gluon.init_executor(LPD.export_file, (h, w), LPD.ctx[0])
yolo_gluon.test_inference_rate(net, (1, 3, h, w),
cycles=100,
ctx=LPD.ctx[0])
rospy.init_node("LP_Detection_Video_Node", anonymous=True)
threading.Thread(target=_video_thread).start()
net_thread_start = True
while not rospy.is_shutdown():
if 'img_g' not in globals() or img_g is None:
print('Wait For Image')
time.sleep(1.0)
continue
if not net_thread_start:
time.sleep(0.01)
continue
try:
net_start_time = time.time() # tic
net_img = img_g.copy() # (480, 640, 3), np.float32
net_img = cv2.resize(net_img, (w, h)) # (320, 512, 3)
if args.trt:
# if cuMemcpyHtoDAsync failed: invalid argument, check input image size
trt_outputs = do_inference_wrapper(engine_wrapper, net_img)
net_out = np.array(trt_outputs).reshape(
net_out_shape) # list to np.array
else:
nd_img = yolo_gluon.cv_img_2_ndarray(
net_img, LPD.ctx[0]) #, mxnet_resize=mx_resize)
net_out = net.forward(is_train=False, data=nd_img)[0]
net_out[-1].wait_to_read()
yolo_gluon.switch_print(time.time() - net_start_time,
video_verbose)
net_img_g = net_img
net_out_g = net_out
net_thread_start = False
except Exception as e:
print(global_variable.red)
rospy.signal_shutdown('main_thread Error')
print(e)
print(global_variable.reset_color)
sys.exit(0)
def run(self):
print(global_variable.green)
print('Start Single Thread Video Node')
print(global_variable.reset_color)
h, w = self.yolo.size
shape = (1, 3, h, w)
yolo_gluon.test_inference_rate(self.yolo.net, shape, cycles=200)
#mx_resize = mxnet.image.ForceResizeAug((w, h))
while not rospy.is_shutdown():
if not hasattr(self, 'image') or self.image is None:
print('Wait For Image')
time.sleep(1.0)
continue
self.net_img_time = self.img_cb_time
# -------------------- image -------------------- #
net_dep = copy.copy(self.depth_image)
net_img = copy.copy(self.image)
net_img = cv2.resize(net_img, (w, h))
nd_img = yolo_gluon.cv_img_2_ndarray(net_img, self.ctx[0])
# nd_img = yolo_gluon.nd_white_balance(nd_img, bgr=[1.0, 1.0, 1.0])
# if self.yolo.use_fp16:
# nd_img = nd_img.astype('float16')
net_out = self.yolo.net.forward(is_train=False, data=nd_img)
self.process(net_img, net_out, net_dep)
def _image_callback(img):
img = bridge.imgmsg_to_cv2(img, "bgr8")
img = cv2.resize(img, tuple(size[::-1]))
nd_img = yolo_gluon.cv_img_2_ndarray(img, ctx[0])
score, text = predict(nd_img)
cv2_show_OCR_result(img, score, text)
pub.publish(text)
def __call__(self):
print(global_variable.green)
print('Start Double Thread Video Node')
ctx = self.ctx[0]
h, w = self.yolo.size
threading.Thread(target=self._video_thread).start()
self.net_thread_start = True
while not rospy.is_shutdown():
if self.image is None:
print('Net thread Wait For Image')
time.sleep(1.0)
continue
if not self.net_thread_start:
time.sleep(0.01)
continue
# -------------------- additional image info-------------------- #
net_img_time, net_img_seq = self.img_cb_time, self.img_cb_seq
now = rospy.get_rostime()
yolo_gluon.switch_print(
'cam to net: %f' % (now - net_img_time).to_sec(), verbose)
net_dep = copy.copy(self.depth_image)
net_img = copy.copy(self.image)
net_img = cv2.resize(net_img, (w, h))
# -------------------- image -------------------- #
if self.trt:
pass
else:
nd_img = yolo_gluon.cv_img_2_ndarray(net_img, ctx)
# if self.yolo.use_fp16:
# nd_img = nd_img.astype('float16')
# nd_img = yolo_gluon.nd_white_balance(nd_img, bgr=[1.0, 1.0, 1.0])
net_out = self.yolo.net.forward(is_train=False, data=nd_img)
net_out[0].wait_to_read()
# -------------------- additional image info-------------------- #
self.net_img_time = net_img_time
self.net_img_seq = net_img_seq
now = rospy.get_rostime()
yolo_gluon.switch_print(
'net done time: %f' % (now - net_img_time).to_sec(), verbose)
# -------------------- image -------------------- #
self.net_dep = net_dep
self.net_img = net_img
self.net_out = net_out
self.net_thread_start = False
def inference(self, net_img):
if self.trt:
trt_outputs = do_inference_wrapper(self.yolo.net, net_img)
net_out = nd.array(trt_outputs).as_in_context(self.ctx[0])
net_out = [net_out.reshape((1, 160, 5, 30))]
else:
nd_img = yolo_gluon.cv_img_2_ndarray(net_img, self.ctx[0])
# if self.yolo.use_fp16:
# nd_img = nd_img.astype('float16')
# nd_img = yolo_gluon.nd_white_balance(nd_img, bgr=[1.0, 1.0, 1.0])
net_out = self.yolo.net.forward(is_train=False, data=nd_img)
net_out[0].wait_to_read()
return net_out
def _net_thread(self):
h, w = self.size
mx_resize = mxnet.image.ForceResizeAug((w, h), interp=2)
while not rospy.is_shutdown():
if not hasattr(self, 'img') or self.img is None:
print('Wait For Image')
time.sleep(1.0)
continue
net_img = self.img.copy()
nd_img = yolo_gluon.cv_img_2_ndarray(net_img,
self.ctx[0],
mxnet_resize=mx_resize)
#nd_img = yolo_gluon.nd_white_balance(nd_img, bgr=[1.0, 1.0, 1.0])
net_out = self.net.forward(is_train=False, data=nd_img)[0]
# self.net_out = [nd.array((1, 10, 10, 16))]
net_out[-1].wait_to_read()
self.net_img = net_img
self.net_out = net_out
def valid_Nima(self):
'''
compare with "Unsupervised Generation of a Viewpoint"
Annotated Car Dataset from Videos
-path: /media/nolan/SSD1/YOLO_backup/freiburg_static_cars_52_v1.1
|-annotations (d)
|-txt_path (d)
|-all car folders (d)
|-result_path (v)
|-all car folders with box (v)
|-img_path (v)
|-annotations (v)
|-save_txt_path (v)
|-plot (p)
|-all car azi compare (p)
d: download from "Unsupervised Generation of a Viewpoint"
v: generate from valid_Nima
o: generate from valid_Nima_plot
'''
import cv2
image_w = 960.
image_h = 540.
mx_resize = mxnet.image.ForceResizeAug(tuple(self.size[::-1]))
radar_prob = yolo_cv.RadarProb(self.num_class, self.classes)
x = np.arange(53)
x = np.delete(x, [0, 6, 20, 23, 31, 36])
for i in x:
result_path = os.path.join(freiburg_path,
"result_%s" % self.version)
save_img_path = os.path.join(result_path, "car_%d" % i)
if not os.path.exists(save_img_path):
os.makedirs(save_img_path)
save_txt_path = os.path.join(result_path, 'annotations')
if not os.path.exists(save_txt_path):
os.makedirs(save_txt_path)
txt_path = os.path.join(freiburg_path, "annotations",
"%d_annot.txt" % i)
with open(txt_path, 'r') as f:
all_lines = f.readlines()
for line_i, line in enumerate(all_lines):
img_name = line.split('\t')[0].split('.')[0] + '.png'
img_path = os.path.join(freiburg_path, img_name)
save_img = os.path.join(save_img_path, img_name.split('/')[-1])
img = cv2.imread(img_path)
# -------------------- Prediction -------------------- #
nd_img = yolo_gluon.cv_img_2_ndarray(img,
self.ctx[0],
mxnet_resize=mx_resize)
net_out = self.net.forward(is_train=False, data=nd_img)
pred_car = self.predict(net_out[:3])
pred_car[0, 5] = -1 # (1, 80)
Cout = pred_car[0]
left_ = (Cout[2] - 0.5 * Cout[4]) * image_w
up_ = (Cout[1] - 0.5 * Cout[3]) * image_h
right_ = (Cout[2] + 0.5 * Cout[4]) * image_w
down_ = (Cout[1] + 0.5 * Cout[3]) * image_h
vec_ang, vec_rad, prob = radar_prob.cls2ang(
Cout[0], Cout[-self.num_class:])
# -------------------- Ground Truth -------------------- #
box_para = np.fromstring(line.split('\t')[1],
dtype='float32',
sep=' ')
left, up, right, down = box_para
h = (down - up) / image_h
w = (right - left) / image_w
y = (down + up) / 2 / image_h
x = (right + left) / 2 / image_w
boxA = [0, y, x, h, w, 0]
azi_label = int(line.split('\t')[2]) - 90
azi_label = azi_label - 360 if azi_label > 180 else azi_label
inter = (min(right, right_) -
max(left, left_)) * (min(down, down_) - max(up, up_))
a1 = (right - left) * (down - up)
a2 = (right_ - left_) * (down_ - up_)
iou = (inter) / (a1 + a2 - inter)
save_txt = os.path.join(save_txt_path, '%d_annot' % i)
with open(save_txt, 'a') as f:
f.write('%s %f %f %f %f\n' %
(img_name, iou, azi_label, vec_ang * 180 / math.pi,
vec_rad))
yolo_cv.cv2_add_bbox(img, Cout, 3, use_r=False)
yolo_cv.cv2_add_bbox(img, boxA, 4, use_r=False)
cv2.imshow('img', img)
cv2.waitKey(1)
cv2.imwrite(save_img, img)
