def ImgDetect(img_path, net, meta, darknet_data, save_path='./', noshow_img=True, save_img=False): import YoloObj img = cv2.imread(img_path) results = DFUNC.detect(net, meta, bytes(img_path, encoding='utf-8'), thresh=float(args.thresh)) objs = [] for result in results: obj = YoloObj.DetectedObj(result) objs.append(obj) for obj in objs: print(obj.obj_string, obj.cx, obj.cy) print('Number of objects: ', len(objs), '\n') YoloObj.DrawBBox(objs, img, show=not noshow_img, save=save_img, save_path=save_path) return objs
def YoloTrackDetect(img_base64, temp_img, net_track, meta_track, net_detect, meta_detect): # decode image data from base64 format img_decode = base64.b64decode(img_base64) nparr = np.fromstring(img_decode, np.uint8) img_np = cv2.imdecode(nparr, cv2.IMREAD_COLOR) print('input size: ', img_np.shape) # for tracking system DFUNC.set_gpu(track_gpu) track_objs = ImgDetect(net_track, meta_track, img_np) if len(track_objs) > 0: # exclude the ugly objects track_objs = [obj for obj in track_objs if obj.name != '1'] # exclude the edge objects central_ratio = 0.6 img_central_area = ( (int(img_np.shape[1]*(1-central_ratio)/2), int(img_np.shape[0]*(1-central_ratio)/2)), (int(img_np.shape[1]*(1-(1-central_ratio)/2)), int(img_np.shape[0]*(1-(1-central_ratio)/2))) ) track_objs = [ obj for obj in track_objs if obj.cx >= img_central_area[0][0] and obj.cy >= img_central_area[0][1] ] track_objs = [ obj for obj in track_objs if obj.cx <= img_central_area[1][0] and obj.cy <= img_central_area[1][1] ] cam_orient = YoloObj.CamOrient(track_objs, img_np.shape, cam_fov, temp_real_size, temp_img.shape, meta_track, temp_objs_coord_file) print('cam orientation: ', cam_orient.position_real, cam_orient.yaw_deg) # for detecting system DFUNC.set_gpu(detect_gpu) detect_objs = ImgDetect(net_detect, meta_detect, img_np) if len(detect_objs) == 0: detect_objs = None # img = YoloObj.DrawBBox(track_objs, img_np) # YoloObj.ShowImg(img) # print('detect_objs, track_objs: ', detect_objs, track_objs) return cam_orient, track_objs, detect_objs, img_np
def ImgDetect(net, meta, img_np, thresh=0.25): # net: DFUNC.load_net(bytes(darknet_cfg, 'utf-8'), # bytes(darknet_weights, 'utf-8'), 0) # meta: DFUNC.load_meta(bytes(darknet_data, 'utf-8')) # img_np: image in numpy array results = DFUNC.detect(net, meta, img_np, thresh) objs = [] for result in results: obj = YoloObj.DetectedObj(result) objs.append(obj) # sort the objects by confidence objs = sorted(objs, key=lambda x: x.conf, reverse=True) return objs
def ReduceFinalReport2(final_objs, overlap=0.75): reduce_objs = final_objs.copy() idx2 = 0 for idx1, obj1 in enumerate(final_objs[:-1]): idx2 += 1 for obj2 in final_objs[idx2:]: if obj2.a >= obj1.a: innerObj = obj1 outerObj = obj2 else: innerObj = obj2 outerObj = obj1 POI = YoloObj.ObjsPOI(innerObj, outerObj) print('idx2: ', idx1, idx2, (obj2.cx, obj2.cy)) print('POI: ', POI) print() if POI >= overlap and obj2 in reduce_objs: reduce_objs.remove(obj2) return reduce_objs
def VideoDetect(video_path, label_dict, save_video=False, auto_label=False, skip_nolabel=False, resize=1.0, exclude_objs='background', autolabel_dir='images'): fourcc = cv2.VideoWriter_fourcc(*'XVID') cap = cv2.VideoCapture(video_path) fps = cap.get(cv2.CAP_PROP_FPS) width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) * float(resize)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) * float(resize)) print('fps:', fps) if save_video: out = cv2.VideoWriter('output.avi', fourcc, 30.0, (width, height)) ii=0 while (cap.isOpened()): ii+=1 ret, frame = cap.read() print('frame : ', ii, ret, type(frame)) if not ret: break # frame = cv2.resize( frame, (int(frame.shape[1]*float(resize)), # int(frame.shape[0]*float(resize))) ) frame = cv2.resize(frame, (width, height)) cv2.imwrite('tmp.jpg', frame) objs = ImgDetect('tmp.jpg', net, meta, darknet_data) new_objs = [obj for obj in objs if obj.name not in exclude_objs] for obj in new_objs: print('obj: ', obj.name, obj.conf) if auto_label: if not os.path.isdir(autolabel_dir): os.mkdir(autolabel_dir) # if not os.path.isdir('labels'): os.mkdir('labels') YoloObj.AutoLabeling(frame, new_objs, label_dict, autolabel_dir + '/frame{:05d}.jpg'.format(ii), autolabel_dir + '/frame{:05d}.txt'.format(ii), skip_nolabel=args.skip_nolabel ) img = YoloObj.DrawBBox(new_objs, frame, show=False, save=False) if save_video: out.write(img) # cv2.imshow(dirname + '/' + filename, img) cv2.imshow(args.video_path, img) k = cv2.waitKey(1) & 0xFF if k == 27 or k== ord('q'): break print('Prediction is finished.') cap.release() cv2.destroyAllWindows()
def run(): global conn_3rdp # connect to pad with socket recv_conn = send_conn = None while recv_conn is None and send_conn is None: recv_conn, recv_addr, send_conn, send_addr = \ SS.accept(s_recv, s_send) padip = recv_addr[0] print('pad accept done: ', recv_addr, send_addr) # for_test # create report directory time_stamp = time.strftime("%Y-%m-%d_%H%M%S", time.localtime()) report_dir = os.path.join('reports', time_stamp + '_reports') os.makedirs(report_dir) result_pic_dir = os.path.join(report_dir, 'results') os.makedirs(result_pic_dir) final_report = [] total_time = 0 ii = 0 throu_ckpt = -1 startup = False allOut_dicts = [] all_defect_locs = [] all_defect_sizes = [] ckpt_imgs = [] report_doc_imgs = [] # check charging status of laptop battery = psutil.sensors_battery() charging = battery.power_plugged plugin = '0' if charging == False else '1' while True: print() ## start pad process ## # get the data length from pad process1 = mp.Process(name='get_data', target=SS.get_data, args=(recv_conn, 10, q)) process1.daemon = True process1.start() print('waiting for head data') head = q.get() data_len = int(SS.bytes2str(head).strip('\x00')) # for_test init_time = time.time() count = data_len data_str = '' while count > 0: print('count: ', count) # for_test # get data from pad process1 = mp.Process(target=SS.get_data, args=(recv_conn, count, q)) process1.daemon = True process1.start() print('waiting for the data') data = q.get() data_decode = SS.bytes2str(data) data_str += data_decode count -= len(data) if count == 0: # get info. from pad recv_id, img_base64, ckpt_id = \ utils.ParseRecvJsonMsg(data_str) print('recv_id: ', recv_id) print('ii: ', ii) # start AI if recv_id == '1': AI_ready_json = utils.ParseSendJsonMsg(recv_id, plugin) print('AI_ready_json', AI_ready_json) # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(AI_ready_json)) send_conn.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) send_conn.send(AI_ready_json.encode()) startup = True # start scanning elif recv_id in ['2', '2_diag', '3']: AI_init_time = time.time() # for_test cam_orient, track_objs, orig_detect_objs, img_np = \ utils.YoloTrackDetect(img_base64, temp_img, net_track, meta_track, net_detect, meta_detect) # exclude self_diag_abnormal objects if orig_detect_objs is not None: detect_objs = [ obj for obj in orig_detect_objs if obj.name != 'self_diag_abnormal' ] else: detect_objs = None print('padip: ', padip) # for_test print('detect objs: ', detect_objs) # for_test print('plugin: ', plugin) # for_test detect_json = utils.ParseSendJsonMsg( recv_id, plugin, cam_orient.position_real, cam_orient.yaw_deg, detect_objs, allOut_dicts) print('detect_json: ', detect_json) # for_test # # for_test # if detect_objs is not None: # for obj in detect_objs: # print(obj.name, obj.conf) # self diagnosis if recv_id == '2_diag': if cam_orient.mm2pixel is not None: img_np_cx = int(img_np.shape[1] / 2) img_np_cy = int(img_np.shape[0] / 2) if detect_objs is not None: diag_objs = [ obj for obj in orig_detect_objs if obj.name == 'self_diag_abnormal' ] else: diag_objs = [] diag_objs_pos = [] for obj in diag_objs: obj_rel_pos = ((obj.cx - img_np_cx) / cam_orient.mm2pixel[0], (obj.cy - img_np_cy) / cam_orient.mm2pixel[1]) obj_real_pos = (cam_orient.position_real[0] + obj_rel_pos[0], cam_orient.position_real[1] + obj_rel_pos[1]) diag_objs_pos.append(obj_real_pos) diag_objs_exist = [ obj for obj in diag_objs_pos if 0 + 20 <= obj[0] <= temp_real_size[1] - 20 and 0 + 20 <= obj[1] <= temp_real_size[0] - 20 ] # diag_id == '2_diag' if len(diag_objs_exist) != 0 else '2_diag_none' if len( diag_objs_exist ) != 0 and cam_orient.position_real is not None: diag_id = recv_id else: diag_id = '2_diag_none' else: diag_id = recv_id diag_objs_exist = [] diag_objs = [] print('diag test: ', diag_objs_exist) # for_test print('diag recv_id: ', diag_id) # for_test detect_json = utils.ParseSendJsonMsg( diag_id, plugin, cam_orient.position_real, cam_orient.yaw_deg, diag_objs, allOut_dicts) # through check point and save checkpoint if recv_id == '3': # save images of each check point ckpt_imgs.append(img_np) report_objs = utils.SaveReport(time_stamp, ckpt_id, cam_orient, track_objs, detect_objs, img_np) final_report.extend(report_objs) throu_ckpt += 1 # not through any check point yet if throu_ckpt < 0: # dict_str = str([]) out_dicts = [] # through check point else: if detect_objs is None: out_dicts, ckpt_defects, defect_locs, defect_sizes = \ utils.Gen3rdpCkptDictStr(config, throu_ckpt, ckpt_id, cam_orient, []) else: out_dicts, ckpt_defects, defect_locs, defect_sizes = \ utils.Gen3rdpCkptDictStr(config, throu_ckpt, ckpt_id, cam_orient, detect_objs) allOut_dicts.extend(out_dicts) all_defect_locs.extend(defect_locs) all_defect_sizes.extend(defect_sizes) # for_test with open('ckpt_report_' + str(throu_ckpt) + '.txt', 'w') as f: f.write(str(out_dicts) + ' ' + str(len(out_dicts))) # input('stop in recv_id = 3: ') # for_test print('throu_ckpt: ', throu_ckpt) # for_test # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(detect_json)) send_conn.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) send_conn.send(detect_json.encode()) print('AI time: ', time.time() - AI_init_time, 's') # for_test elif recv_id == '4': allOut_dicts.sort(key=lambda x: int(x['id'])) # send final report images to 3rd party program if conn_3rdp is not None: report_3rdp_json = \ utils.Gen3rdpReportJson(cam_orient, report_dir, ckpt_imgs, ckpt_defects, allOut_dicts, all_defect_locs, all_defect_sizes, padip, plugin) # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(report_3rdp_json)) conn_3rdp.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) # send data to 3rd party program conn_3rdp.send(report_3rdp_json.encode()) closeAI_json = utils.ParseSendJsonMsg(recv_id, plugin) print('recv_id4 test', closeAI_json) # for_test # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(closeAI_json)) send_conn.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) send_conn.send(closeAI_json.encode()) print('recv_id4 test2') # for_test elif recv_id == '5': final_img = os.path.join(report_dir, 'final_report.jpg') report_img = YoloObj.DrawBBox(final_report, temp_img) YoloObj.SaveImg(report_img, save_path=final_img) # send report to pad report_json = utils.ParseSendJsonMsg( recv_id, plugin, detect_objs=final_report) # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(report_json)) send_conn.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) send_conn.send(report_json.encode()) raise ValueError interval = time.time() - init_time total_time += interval ii += 1 average_time = total_time / ii print('perframe: ', interval, 's') # for_test print('average_time: ', average_time, 's') # for_test # f.write(str(ii) + ' ' + str(perframe) + '\n') # for_test ## end pad process ## print('startup : ', startup) # for_test ## start 3rd party program process ## # get status of 3rd party connection print('3rd-p status: ', inst.get()) if conn_3rdp is None: print('not get 3rd status yet.') conn_3rdp, addr_3rdp = inst.get() elif conn_3rdp is not None and startup == True: init_3rdp_time = time.time() img_3rdp = YoloObj.DrawBBox(detect_objs, img_np, color=(0, 0, 255)) # img_3rdp = YoloObj.DrawBBox(track_objs, img_3rdp, color=(0, 255, 0)) # for_test encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 20] img_3rdp_enc = cv2.imencode('.jpg', img_3rdp, encode_param)[1].tostring() img_3rdp_b64 = base64.b64encode(img_3rdp_enc).decode( 'utf-8') print('allOut_dicts2: ', allOut_dicts) out_3rdp_json = utils.ParseSend3rdPartyJsonMsg( img_3rdp_b64, padip, plugin, # eval(dict_str), allOut_dicts, cam_orient.position_real, cam_orient.yaw_deg) # print('out_dicts: ', out_dicts) # for_test # print('allOut_dicts: ', allOut_dicts) # for_test try: # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') print('test 3rdp') # for_test output_len_str = str(len(out_3rdp_json)) conn_3rdp.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) # print('test true out_3rdp_json: ', (out_3rdp_json) ) # for_test # send data to 3rd party program conn_3rdp.send(out_3rdp_json.encode()) print('3rdp time: ', time.time() - init_3rdp_time, 's') # for_test # except socket.error, ConnectionResetError as sockerr: except: # print('sockerr: ', sockerr) # for_test # input('enter to continue') # for_test inst.set(None, None) conn_3rdp, addr_3rdp = inst.get() elif conn_3rdp is not None and startup == False: init_3rdp_time = time.time() out_3rdp_json = utils.ParseSend3rdPartyJsonMsg( '', padip, plugin) try: # send the length info. of data to 3rd-p program # (10 bytes: ex. '123456\x00\x00\x00\x00') output_len_str = str(len(out_3rdp_json)) conn_3rdp.send((output_len_str + '\x00' * (10 - len(output_len_str))).encode()) # send data to 3rd party program conn_3rdp.send(out_3rdp_json.encode()) print('test false out_3rdp_json: ', (out_3rdp_json)) # for_test print('3rdp time: ', time.time() - init_3rdp_time, 's') # for_test # except socket.error, ConnectionResetError as sockerr: except: # print('sockerr: ', sockerr) # for_test # input('enter to continue') # for_test inst.set(None, None) conn_3rdp, addr_3rdp = inst.get()
def SaveReport(time_stamp, checkpt_id, cam_orient, track_objs, detect_objs, img_np): w_ratio = cam_orient.temp_tgt_ratio[0] h_ratio = cam_orient.temp_tgt_ratio[1] tgt_cx = cam_orient.tgt_cx tgt_cy = cam_orient.tgt_cy tgt_cam_x = cam_orient.xy_position_pixel[0] tgt_cam_y = cam_orient.xy_position_pixel[1] tgt_cam_realx = cam_orient.position_real[0] tgt_cam_realy = cam_orient.position_real[1] img = img_np img = YoloObj.DrawBBox(track_objs, img, width=3) img = YoloObj.DrawBBox(detect_objs, img, color=(0,0,255), width=3) img = cv2.putText(img, 'real pos ({}, {})'.format(int(tgt_cam_realx), int(tgt_cam_realy)), (tgt_cx, tgt_cy-30), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (0,0,255), 1, cv2.LINE_AA) objs_temp = [] report_objs = [] if detect_objs is None: detect_objs = [] for obj in detect_objs: tgtc2obj_vec = (obj.cx-tgt_cx, obj.cy-tgt_cy) obj_temp_position = ( tgt_cam_x + tgtc2obj_vec[0]*w_ratio, tgt_cam_y + tgtc2obj_vec[1]*h_ratio ) obj_temp_shape = ( obj.w * w_ratio, obj.h * h_ratio) objs_temp.append({obj.name: obj_temp_position}) report_obj = ReportObj(obj.name, obj.conf, obj_temp_position, obj_temp_shape) report_objs.append(report_obj) report_dir = os.path.join('reports', time_stamp + '_reports') if not os.path.isdir(report_dir): os.makedirs(report_dir) report_file = os.path.join(report_dir, 'check_point_{:03d}.txt'.format(int(checkpt_id))) report_img = os.path.join(report_dir, 'check_point_{:03d}.jpg'.format(int(checkpt_id))) with open(report_file, 'w+') as f: # f.write('temp_cxy: ' + str(temp_cx) + ' ' + str(temp_cy), '\n') f.write('track_obj: ' + str(track_objs[0].name) + ' ' + str(track_objs[0].conf) + ' ' + str((track_objs[0].cx, track_objs[0].cy)) + '\n') f.write('cam_realpoition: ' + str(cam_orient.position_real) + str(cam_orient.yaw_deg) + '\n') f.write('tgtc2obj_vec: ' + str(tgtc2obj_vec) + '\n') f.write('ratio: ' + str(w_ratio) + ' ' + str(h_ratio) + '\n') f.write(str(objs_temp) + '\n') YoloObj.SaveImg(img, save_path=report_img) # input('report saving test') # for_test return report_objs
def VideoDetect(video_path, label_dict, save_video=False, auto_label=False, skip_nolabel=False, resize=1.0, exclude_objs='background', autolabel_dir='images'): fourcc = cv2.VideoWriter_fourcc(*'XVID') cap = cv2.VideoCapture(video_path) fps = cap.get(cv2.CAP_PROP_FPS) width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) * float(resize)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) * float(resize)) print('fps:', fps) if save_video: out = cv2.VideoWriter('output.avi', fourcc, 30.0, (width, height)) ii=0 total_num_objs = 0 while (cap.isOpened()): ii+=1 ret, frame = cap.read() print('frame : ', ii, ret, type(frame)) if not ret: break # frame = cv2.resize( frame, (int(frame.shape[1]*float(resize)), # int(frame.shape[0]*float(resize))) ) frame = cv2.resize(frame, (width, height)) cv2.imwrite('tmp.jpg', frame) objs = ImgDetect('tmp.jpg', net, meta, darknet_data) new_objs = [obj for obj in objs if obj.name not in exclude_objs] # for obj in new_objs: # print('obj: ', obj.name, obj.conf) baseline = width*0.1 if ii > 1: num_objs = YoloObj.ObjFlowNum(new_objs, pre_objs, "left", baseline) print('num_objs: ', num_objs) total_num_objs += num_objs pre_objs = new_objs if auto_label: if not os.path.isdir(autolabel_dir): os.mkdir(autolabel_dir) # if not os.path.isdir('labels'): os.mkdir('labels') YoloObj.AutoLabeling(frame, new_objs, label_dict, autolabel_dir + '/frame{:05d}.jpg'.format(ii), autolabel_dir + '/frame{:05d}.txt'.format(ii), skip_nolabel=args.skip_nolabel ) img = YoloObj.DrawBBox(new_objs, frame, show=False, save=False) cv2.line(img, (int(baseline), 0), (int(baseline), height), (0,0,255), 5) cv2.putText(img, 'Object detected: ' + str(total_num_objs), (int(width*0.7), int(height*0.1)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2) if save_video: out.write(img) # cv2.imshow(dirname + '/' + filename, img) cv2.imshow(args.video_path, img) k = cv2.waitKey(1) & 0xFF if k == 27 or k== ord('q'): break print('Prediction is finished.') print('total_num_objs: ', total_num_objs) cap.release() cv2.destroyAllWindows()