def ImgDetect(img_path,
net,
meta,
darknet_data,
save_path='./',
noshow_img=True,
save_img=False):
import YoloObj
img = cv2.imread(img_path)
results = DFUNC.detect(net,
meta,
bytes(img_path, encoding='utf-8'),
thresh=float(args.thresh))
objs = []
for result in results:
obj = YoloObj.DetectedObj(result)
objs.append(obj)
for obj in objs:
print(obj.obj_string, obj.cx, obj.cy)
print('Number of objects: ', len(objs), '\n')
YoloObj.DrawBBox(objs,
img,
show=not noshow_img,
save=save_img,
save_path=save_path)
return objs
def YoloTrackDetect(img_base64,
temp_img,
net_track, meta_track,
net_detect, meta_detect):
# decode image data from base64 format
img_decode = base64.b64decode(img_base64)
nparr = np.fromstring(img_decode, np.uint8)
img_np = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
print('input size: ', img_np.shape)
# for tracking system
DFUNC.set_gpu(track_gpu)
track_objs = ImgDetect(net_track, meta_track, img_np)
if len(track_objs) > 0:
# exclude the ugly objects
track_objs = [obj for obj in track_objs if obj.name != '1']
# exclude the edge objects
central_ratio = 0.6
img_central_area = ( (int(img_np.shape[1]*(1-central_ratio)/2),
int(img_np.shape[0]*(1-central_ratio)/2)),
(int(img_np.shape[1]*(1-(1-central_ratio)/2)),
int(img_np.shape[0]*(1-(1-central_ratio)/2))) )
track_objs = [ obj for obj in track_objs
if obj.cx >= img_central_area[0][0]
and obj.cy >= img_central_area[0][1] ]
track_objs = [ obj for obj in track_objs
if obj.cx <= img_central_area[1][0]
and obj.cy <= img_central_area[1][1] ]
cam_orient = YoloObj.CamOrient(track_objs,
img_np.shape,
cam_fov,
temp_real_size,
temp_img.shape,
meta_track,
temp_objs_coord_file)
print('cam orientation: ', cam_orient.position_real,
cam_orient.yaw_deg)
# for detecting system
DFUNC.set_gpu(detect_gpu)
detect_objs = ImgDetect(net_detect, meta_detect, img_np)
if len(detect_objs) == 0:
detect_objs = None
# img = YoloObj.DrawBBox(track_objs, img_np)
# YoloObj.ShowImg(img)
# print('detect_objs, track_objs: ', detect_objs, track_objs)
return cam_orient, track_objs, detect_objs, img_np
def ImgDetect(net, meta, img_np, thresh=0.25):
# net: DFUNC.load_net(bytes(darknet_cfg, 'utf-8'),
# bytes(darknet_weights, 'utf-8'), 0)
# meta: DFUNC.load_meta(bytes(darknet_data, 'utf-8'))
# img_np: image in numpy array
results = DFUNC.detect(net, meta, img_np, thresh)
objs = []
for result in results:
obj = YoloObj.DetectedObj(result)
objs.append(obj)
# sort the objects by confidence
objs = sorted(objs, key=lambda x: x.conf, reverse=True)
return objs
def ReduceFinalReport2(final_objs, overlap=0.75):
reduce_objs = final_objs.copy()
idx2 = 0
for idx1, obj1 in enumerate(final_objs[:-1]):
idx2 += 1
for obj2 in final_objs[idx2:]:
if obj2.a >= obj1.a:
innerObj = obj1
outerObj = obj2
else:
innerObj = obj2
outerObj = obj1
POI = YoloObj.ObjsPOI(innerObj, outerObj)
print('idx2: ', idx1, idx2, (obj2.cx, obj2.cy))
print('POI: ', POI)
print()
if POI >= overlap and obj2 in reduce_objs:
reduce_objs.remove(obj2)
return reduce_objs
def VideoDetect(video_path, label_dict,
save_video=False, auto_label=False, skip_nolabel=False,
resize=1.0, exclude_objs='background', autolabel_dir='images'):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
cap = cv2.VideoCapture(video_path)
fps = cap.get(cv2.CAP_PROP_FPS)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) * float(resize))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) * float(resize))
print('fps:', fps)
if save_video:
out = cv2.VideoWriter('output.avi', fourcc, 30.0, (width, height))
ii=0
while (cap.isOpened()):
ii+=1
ret, frame = cap.read()
print('frame : ', ii, ret, type(frame))
if not ret:
break
# frame = cv2.resize( frame, (int(frame.shape[1]*float(resize)),
# int(frame.shape[0]*float(resize))) )
frame = cv2.resize(frame, (width, height))
cv2.imwrite('tmp.jpg', frame)
objs = ImgDetect('tmp.jpg', net, meta, darknet_data)
new_objs = [obj for obj in objs if obj.name not in exclude_objs]
for obj in new_objs:
print('obj: ', obj.name, obj.conf)
if auto_label:
if not os.path.isdir(autolabel_dir): os.mkdir(autolabel_dir)
# if not os.path.isdir('labels'): os.mkdir('labels')
YoloObj.AutoLabeling(frame, new_objs, label_dict,
autolabel_dir + '/frame{:05d}.jpg'.format(ii),
autolabel_dir + '/frame{:05d}.txt'.format(ii),
skip_nolabel=args.skip_nolabel
)
img = YoloObj.DrawBBox(new_objs, frame, show=False, save=False)
if save_video:
out.write(img)
# cv2.imshow(dirname + '/' + filename, img)
cv2.imshow(args.video_path, img)
k = cv2.waitKey(1) & 0xFF
if k == 27 or k== ord('q'):
break
print('Prediction is finished.')
cap.release()
cv2.destroyAllWindows()
def run():
global conn_3rdp
# connect to pad with socket
recv_conn = send_conn = None
while recv_conn is None and send_conn is None:
recv_conn, recv_addr, send_conn, send_addr = \
SS.accept(s_recv, s_send)
padip = recv_addr[0]
print('pad accept done: ', recv_addr, send_addr) # for_test
# create report directory
time_stamp = time.strftime("%Y-%m-%d_%H%M%S", time.localtime())
report_dir = os.path.join('reports', time_stamp + '_reports')
os.makedirs(report_dir)
result_pic_dir = os.path.join(report_dir, 'results')
os.makedirs(result_pic_dir)
final_report = []
total_time = 0
ii = 0
throu_ckpt = -1
startup = False
allOut_dicts = []
all_defect_locs = []
all_defect_sizes = []
ckpt_imgs = []
report_doc_imgs = []
# check charging status of laptop
battery = psutil.sensors_battery()
charging = battery.power_plugged
plugin = '0' if charging == False else '1'
while True:
print()
## start pad process ##
# get the data length from pad
process1 = mp.Process(name='get_data',
target=SS.get_data,
args=(recv_conn, 10, q))
process1.daemon = True
process1.start()
print('waiting for head data')
head = q.get()
data_len = int(SS.bytes2str(head).strip('\x00'))
# for_test
init_time = time.time()
count = data_len
data_str = ''
while count > 0:
print('count: ', count) # for_test
# get data from pad
process1 = mp.Process(target=SS.get_data,
args=(recv_conn, count, q))
process1.daemon = True
process1.start()
print('waiting for the data')
data = q.get()
data_decode = SS.bytes2str(data)
data_str += data_decode
count -= len(data)
if count == 0:
# get info. from pad
recv_id, img_base64, ckpt_id = \
utils.ParseRecvJsonMsg(data_str)
print('recv_id: ', recv_id)
print('ii: ', ii)
# start AI
if recv_id == '1':
AI_ready_json = utils.ParseSendJsonMsg(recv_id, plugin)
print('AI_ready_json', AI_ready_json)
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(AI_ready_json))
send_conn.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
send_conn.send(AI_ready_json.encode())
startup = True
# start scanning
elif recv_id in ['2', '2_diag', '3']:
AI_init_time = time.time() # for_test
cam_orient, track_objs, orig_detect_objs, img_np = \
utils.YoloTrackDetect(img_base64, temp_img,
net_track, meta_track,
net_detect, meta_detect)
# exclude self_diag_abnormal objects
if orig_detect_objs is not None:
detect_objs = [
obj for obj in orig_detect_objs
if obj.name != 'self_diag_abnormal'
]
else:
detect_objs = None
print('padip: ', padip) # for_test
print('detect objs: ', detect_objs) # for_test
print('plugin: ', plugin) # for_test
detect_json = utils.ParseSendJsonMsg(
recv_id, plugin, cam_orient.position_real,
cam_orient.yaw_deg, detect_objs, allOut_dicts)
print('detect_json: ', detect_json) # for_test
# # for_test
# if detect_objs is not None:
# for obj in detect_objs:
# print(obj.name, obj.conf)
# self diagnosis
if recv_id == '2_diag':
if cam_orient.mm2pixel is not None:
img_np_cx = int(img_np.shape[1] / 2)
img_np_cy = int(img_np.shape[0] / 2)
if detect_objs is not None:
diag_objs = [
obj for obj in orig_detect_objs
if obj.name == 'self_diag_abnormal'
]
else:
diag_objs = []
diag_objs_pos = []
for obj in diag_objs:
obj_rel_pos = ((obj.cx - img_np_cx) /
cam_orient.mm2pixel[0],
(obj.cy - img_np_cy) /
cam_orient.mm2pixel[1])
obj_real_pos = (cam_orient.position_real[0] +
obj_rel_pos[0],
cam_orient.position_real[1] +
obj_rel_pos[1])
diag_objs_pos.append(obj_real_pos)
diag_objs_exist = [
obj for obj in diag_objs_pos
if 0 + 20 <= obj[0] <= temp_real_size[1] - 20
and 0 + 20 <= obj[1] <= temp_real_size[0] - 20
]
# diag_id == '2_diag' if len(diag_objs_exist) != 0 else '2_diag_none'
if len(
diag_objs_exist
) != 0 and cam_orient.position_real is not None:
diag_id = recv_id
else:
diag_id = '2_diag_none'
else:
diag_id = recv_id
diag_objs_exist = []
diag_objs = []
print('diag test: ', diag_objs_exist) # for_test
print('diag recv_id: ', diag_id) # for_test
detect_json = utils.ParseSendJsonMsg(
diag_id, plugin, cam_orient.position_real,
cam_orient.yaw_deg, diag_objs, allOut_dicts)
# through check point and save checkpoint
if recv_id == '3':
# save images of each check point
ckpt_imgs.append(img_np)
report_objs = utils.SaveReport(time_stamp, ckpt_id,
cam_orient, track_objs,
detect_objs, img_np)
final_report.extend(report_objs)
throu_ckpt += 1
# not through any check point yet
if throu_ckpt < 0:
# dict_str = str([])
out_dicts = []
# through check point
else:
if detect_objs is None:
out_dicts, ckpt_defects, defect_locs, defect_sizes = \
utils.Gen3rdpCkptDictStr(config,
throu_ckpt,
ckpt_id,
cam_orient,
[])
else:
out_dicts, ckpt_defects, defect_locs, defect_sizes = \
utils.Gen3rdpCkptDictStr(config,
throu_ckpt,
ckpt_id,
cam_orient,
detect_objs)
allOut_dicts.extend(out_dicts)
all_defect_locs.extend(defect_locs)
all_defect_sizes.extend(defect_sizes)
# for_test
with open('ckpt_report_' + str(throu_ckpt) + '.txt',
'w') as f:
f.write(str(out_dicts) + ' ' + str(len(out_dicts)))
# input('stop in recv_id = 3: ') # for_test
print('throu_ckpt: ', throu_ckpt) # for_test
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(detect_json))
send_conn.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
send_conn.send(detect_json.encode())
print('AI time: ',
time.time() - AI_init_time, 's') # for_test
elif recv_id == '4':
allOut_dicts.sort(key=lambda x: int(x['id']))
# send final report images to 3rd party program
if conn_3rdp is not None:
report_3rdp_json = \
utils.Gen3rdpReportJson(cam_orient, report_dir,
ckpt_imgs, ckpt_defects,
allOut_dicts,
all_defect_locs,
all_defect_sizes,
padip,
plugin)
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(report_3rdp_json))
conn_3rdp.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
# send data to 3rd party program
conn_3rdp.send(report_3rdp_json.encode())
closeAI_json = utils.ParseSendJsonMsg(recv_id, plugin)
print('recv_id4 test', closeAI_json) # for_test
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(closeAI_json))
send_conn.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
send_conn.send(closeAI_json.encode())
print('recv_id4 test2') # for_test
elif recv_id == '5':
final_img = os.path.join(report_dir, 'final_report.jpg')
report_img = YoloObj.DrawBBox(final_report, temp_img)
YoloObj.SaveImg(report_img, save_path=final_img)
# send report to pad
report_json = utils.ParseSendJsonMsg(
recv_id, plugin, detect_objs=final_report)
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(report_json))
send_conn.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
send_conn.send(report_json.encode())
raise ValueError
interval = time.time() - init_time
total_time += interval
ii += 1
average_time = total_time / ii
print('perframe: ', interval, 's') # for_test
print('average_time: ', average_time, 's') # for_test
# f.write(str(ii) + ' ' + str(perframe) + '\n') # for_test
## end pad process ##
print('startup : ', startup) # for_test
## start 3rd party program process ##
# get status of 3rd party connection
print('3rd-p status: ', inst.get())
if conn_3rdp is None:
print('not get 3rd status yet.')
conn_3rdp, addr_3rdp = inst.get()
elif conn_3rdp is not None and startup == True:
init_3rdp_time = time.time()
img_3rdp = YoloObj.DrawBBox(detect_objs,
img_np,
color=(0, 0, 255))
# img_3rdp = YoloObj.DrawBBox(track_objs, img_3rdp, color=(0, 255, 0)) # for_test
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 20]
img_3rdp_enc = cv2.imencode('.jpg', img_3rdp,
encode_param)[1].tostring()
img_3rdp_b64 = base64.b64encode(img_3rdp_enc).decode(
'utf-8')
print('allOut_dicts2: ', allOut_dicts)
out_3rdp_json = utils.ParseSend3rdPartyJsonMsg(
img_3rdp_b64,
padip,
plugin,
# eval(dict_str),
allOut_dicts,
cam_orient.position_real,
cam_orient.yaw_deg)
# print('out_dicts: ', out_dicts) # for_test
# print('allOut_dicts: ', allOut_dicts) # for_test
try:
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
print('test 3rdp') # for_test
output_len_str = str(len(out_3rdp_json))
conn_3rdp.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
# print('test true out_3rdp_json: ', (out_3rdp_json) ) # for_test
# send data to 3rd party program
conn_3rdp.send(out_3rdp_json.encode())
print('3rdp time: ',
time.time() - init_3rdp_time, 's') # for_test
# except socket.error, ConnectionResetError as sockerr:
except:
# print('sockerr: ', sockerr) # for_test
# input('enter to continue') # for_test
inst.set(None, None)
conn_3rdp, addr_3rdp = inst.get()
elif conn_3rdp is not None and startup == False:
init_3rdp_time = time.time()
out_3rdp_json = utils.ParseSend3rdPartyJsonMsg(
'', padip, plugin)
try:
# send the length info. of data to 3rd-p program
# (10 bytes: ex. '123456\x00\x00\x00\x00')
output_len_str = str(len(out_3rdp_json))
conn_3rdp.send((output_len_str + '\x00' *
(10 - len(output_len_str))).encode())
# send data to 3rd party program
conn_3rdp.send(out_3rdp_json.encode())
print('test false out_3rdp_json: ',
(out_3rdp_json)) # for_test
print('3rdp time: ',
time.time() - init_3rdp_time, 's') # for_test
# except socket.error, ConnectionResetError as sockerr:
except:
# print('sockerr: ', sockerr) # for_test
# input('enter to continue') # for_test
inst.set(None, None)
conn_3rdp, addr_3rdp = inst.get()
def SaveReport(time_stamp, checkpt_id, cam_orient,
track_objs, detect_objs, img_np):
w_ratio = cam_orient.temp_tgt_ratio[0]
h_ratio = cam_orient.temp_tgt_ratio[1]
tgt_cx = cam_orient.tgt_cx
tgt_cy = cam_orient.tgt_cy
tgt_cam_x = cam_orient.xy_position_pixel[0]
tgt_cam_y = cam_orient.xy_position_pixel[1]
tgt_cam_realx = cam_orient.position_real[0]
tgt_cam_realy = cam_orient.position_real[1]
img = img_np
img = YoloObj.DrawBBox(track_objs, img, width=3)
img = YoloObj.DrawBBox(detect_objs, img, color=(0,0,255), width=3)
img = cv2.putText(img,
'real pos ({}, {})'.format(int(tgt_cam_realx),
int(tgt_cam_realy)),
(tgt_cx, tgt_cy-30),
cv2.FONT_HERSHEY_TRIPLEX,
0.5,
(0,0,255),
1,
cv2.LINE_AA)
objs_temp = []
report_objs = []
if detect_objs is None:
detect_objs = []
for obj in detect_objs:
tgtc2obj_vec = (obj.cx-tgt_cx, obj.cy-tgt_cy)
obj_temp_position = ( tgt_cam_x + tgtc2obj_vec[0]*w_ratio,
tgt_cam_y + tgtc2obj_vec[1]*h_ratio )
obj_temp_shape = ( obj.w * w_ratio, obj.h * h_ratio)
objs_temp.append({obj.name: obj_temp_position})
report_obj = ReportObj(obj.name, obj.conf,
obj_temp_position, obj_temp_shape)
report_objs.append(report_obj)
report_dir = os.path.join('reports', time_stamp + '_reports')
if not os.path.isdir(report_dir):
os.makedirs(report_dir)
report_file = os.path.join(report_dir,
'check_point_{:03d}.txt'.format(int(checkpt_id)))
report_img = os.path.join(report_dir,
'check_point_{:03d}.jpg'.format(int(checkpt_id)))
with open(report_file, 'w+') as f:
# f.write('temp_cxy: ' + str(temp_cx) + ' ' + str(temp_cy), '\n')
f.write('track_obj: ' + str(track_objs[0].name) + ' ' + str(track_objs[0].conf) + ' ' + str((track_objs[0].cx, track_objs[0].cy)) + '\n')
f.write('cam_realpoition: ' + str(cam_orient.position_real) + str(cam_orient.yaw_deg) + '\n')
f.write('tgtc2obj_vec: ' + str(tgtc2obj_vec) + '\n')
f.write('ratio: ' + str(w_ratio) + ' ' + str(h_ratio) + '\n')
f.write(str(objs_temp) + '\n')
YoloObj.SaveImg(img, save_path=report_img)
# input('report saving test') # for_test
return report_objs
def VideoDetect(video_path, label_dict,
save_video=False, auto_label=False, skip_nolabel=False,
resize=1.0, exclude_objs='background', autolabel_dir='images'):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
cap = cv2.VideoCapture(video_path)
fps = cap.get(cv2.CAP_PROP_FPS)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) * float(resize))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) * float(resize))
print('fps:', fps)
if save_video:
out = cv2.VideoWriter('output.avi', fourcc, 30.0, (width, height))
ii=0
total_num_objs = 0
while (cap.isOpened()):
ii+=1
ret, frame = cap.read()
print('frame : ', ii, ret, type(frame))
if not ret:
break
# frame = cv2.resize( frame, (int(frame.shape[1]*float(resize)),
# int(frame.shape[0]*float(resize))) )
frame = cv2.resize(frame, (width, height))
cv2.imwrite('tmp.jpg', frame)
objs = ImgDetect('tmp.jpg', net, meta, darknet_data)
new_objs = [obj for obj in objs if obj.name not in exclude_objs]
# for obj in new_objs:
# print('obj: ', obj.name, obj.conf)
baseline = width*0.1
if ii > 1:
num_objs = YoloObj.ObjFlowNum(new_objs, pre_objs, "left", baseline)
print('num_objs: ', num_objs)
total_num_objs += num_objs
pre_objs = new_objs
if auto_label:
if not os.path.isdir(autolabel_dir): os.mkdir(autolabel_dir)
# if not os.path.isdir('labels'): os.mkdir('labels')
YoloObj.AutoLabeling(frame, new_objs, label_dict,
autolabel_dir + '/frame{:05d}.jpg'.format(ii),
autolabel_dir + '/frame{:05d}.txt'.format(ii),
skip_nolabel=args.skip_nolabel
)
img = YoloObj.DrawBBox(new_objs, frame, show=False, save=False)
cv2.line(img, (int(baseline), 0), (int(baseline), height), (0,0,255), 5)
cv2.putText(img, 'Object detected: ' + str(total_num_objs), (int(width*0.7), int(height*0.1)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
if save_video:
out.write(img)
# cv2.imshow(dirname + '/' + filename, img)
cv2.imshow(args.video_path, img)
k = cv2.waitKey(1) & 0xFF
if k == 27 or k== ord('q'):
break
print('Prediction is finished.')
print('total_num_objs: ', total_num_objs)
cap.release()
cv2.destroyAllWindows()