def callback(data): list_obj = listobj() list_obj.single_obj_info = [] brocili_list = broclist() brocili_list.broc_uv_list = [] temp_list_broc = [] global temp_single_obj gettime = data.header #print("22222222222",data.header) #print(data.objects_vector[0].id) # print(data.objects_vector[0].classname) # print(data.objects_vector[0].score) #print("---"*20) single_obj = sglobj() #print("single_obj",single_obj) single_obj.element_data_temp = [] single_obj.classname = [] single_obj.score = [] get_msg = data.objects_vector[0] bridge = CvBridge() cv_image = bridge.imgmsg_to_cv2(data.rgb_img, "bgr8") cv_image_hd = bridge.imgmsg_to_cv2(data.rgb_img_hd, "bgr8") #cv_depth_image = bridge.imgmsg_to_cv2(data.depth_img,"passthrough") rgb_img_to_pos = cv_image.copy() #depth_img_to_pos = cv_depth_image.copy() plate_score = [0, 0] pan_score = [0] vegetablebowl_score = [0] broccoli_score = [0, 0, 0] souppothandle_score = [0] panhandle_score = [0] beef_score = [0] nethandle_score = [0] seasoningbottle_score = [0, 0] seasoningbowl_score = [0] for i in range(len(get_msg.id)): if (get_msg.classname[i] == "pan"): pan_score.append(get_msg.score[i]) if (get_msg.classname[i] == "beef"): beef_score.append(get_msg.score[i]) if (get_msg.classname[i] == "plate"): plate_score.append(get_msg.score[i]) if (get_msg.classname[i] == "vegetablebowl"): vegetablebowl_score.append(get_msg.score[i]) if (get_msg.classname[i] == "broccoli"): broccoli_score.append(get_msg.score[i]) if (get_msg.classname[i] == "souppothandle"): souppothandle_score.append(get_msg.score[i]) if (get_msg.classname[i] == "panhandle"): panhandle_score.append(get_msg.score[i]) if (get_msg.classname[i] == "seasoningbowl"): beef_score.append(get_msg.score[i]) if (get_msg.classname[i] == "nethandle"): nethandle_score.append(get_msg.score[i]) if (get_msg.classname[i] == "seasoningbottle"): seasoningbottle_score.append(get_msg.score[i]) plate_score = sorted(plate_score, reverse=True) pan_score = sorted(pan_score, reverse=True) vegetablebowl_score = sorted(vegetablebowl_score, reverse=True) broccoli_score = sorted(broccoli_score, reverse=True) souppothandle_score = sorted(souppothandle_score, reverse=True) panhandle_score = sorted(panhandle_score, reverse=True) beef_score = sorted(beef_score, reverse=True) nethandle_score = sorted(nethandle_score, reverse=True) seasoningbottle_score = sorted(seasoningbottle_score, reverse=True) seasoningbowl_score = sorted(seasoningbowl_score, reverse=True) #cv2.imshow("niubi",cv_image_hd) det_bar_pos = detectionbar.model_detection(cv_image_hd, cv_image) bro_list = broccolidetection.broccoli_detection(cv_image) global temp_list_broc if len(bro_list) > 0: brocili_list.broc_uv_list = [] brocili_list.broc_uv_list = bro_list temp_list_broc = bro_list if len(bro_list) <= 0: brocili_list.broc_uv_list = temp_list_broc #print("wo cao :",temp_list_broc) #print("xi lan hua: ",brocili_list.broc_uv_list) ''' print("plate score : ",plate_score) print("pan score : ",pan_score) print("vegetablebowl score : ",vegetablebowl_score) print("broccoli score : ",broccoli_score) print("souppothandle score : ",souppothandle_score) print("panhandle score : ",panhandle_score) print("beef score : ",beef_score) print("nethandle score : ",nethandle_score) print("seasoningbottle score : ",seasoningbottle_score) print("seasoningbowl score : ",seasoningbowl_score) ''' single_obj.element_data_temp = [] single_obj.classname = [] single_obj.score = [] element_data = element_info() element_data.id_info = [] for i in range(len(get_msg.id)): element_data = element_info() element_data.id_info = [] mask_area = get_msg.roi[i].height * get_msg.roi[i].width if mask_area > 0 and get_msg.score[i] > 0.19: mask_image = bridge.imgmsg_to_cv2(get_msg.masks[i], "passthrough") if get_msg.classname[i] == "plate": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > plate1[0] and center_x < plate1[ 2] and center_y > plate1[1] and center_y < plate1[3]: temp_list = [] temp_list = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) log.info( "Got plate1 center:get_time: {},center_pos: {}".format( gettime, temp_list[7:9])) if temp_list != None: #element_data.id_info = [] element_data.id_info = temp_list single_obj.element_data_temp.append(element_data) #print("aaa:",single_obj.element_data_temp) single_obj.score.append(get_msg.score[i]) single_obj.classname.append("plate1") if get_msg.classname[i] == "plate": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > plate2[0] and center_x < plate2[ 2] and center_y > plate2[1] and center_y < plate2[3]: temp_list1 = [] temp_list1 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) log.info( "Got plate2 center:get_time: {},center_pos_u_v: {}". format(gettime, temp_list1[7:9])) if temp_list1 != None: #element_data.id_info = [] element_data.id_info = temp_list1 single_obj.element_data_temp.append(element_data) #single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) single_obj.classname.append("plate2") if get_msg.classname[i] == "plate": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > plate3[0] and center_x < plate3[ 2] and center_y > plate3[1] and center_y < plate3[3]: temp_list2 = [] temp_list2 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) log.info( "Got plate3 center:get_time: {},center_pos_u_v: {}". format(gettime, temp_list2[7:9])) if temp_list2 != None: element_data.id_info = temp_list2 #print("wo-2-cia",temp_list) single_obj.element_data_temp.append(element_data) #single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) single_obj.classname.append("plate3") if get_msg.classname[i] == "beef" and get_msg.score[ i] == beef_score[0]: center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > beef_roi[0] and center_x < beef_roi[ 2] and center_y > beef_roi[1] and center_y < beef_roi[ 3]: temp_list3 = [] temp_list3 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) log.info( "Got beef center:get_time: {},center_pos_u_v: {},beef_roi: {}" .format(gettime, temp_list3[7:9], temp_list3[17:19])) if temp_list3 != None: element_data.id_info = temp_list3 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "pan": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > pan_roi[0] and center_x < pan_roi[ 2] and center_y > pan_roi[1] and center_y < pan_roi[3]: temp_list4 = [] temp_list4 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) log.info("Got pan center: get_time: {},center_pos_u_v: {}". format(gettime, temp_list4[7:9])) if temp_list4 != None: element_data.id_info = temp_list4 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) ''' if get_msg.classname[i] == "broccoli" and get_msg.score[i] == broccoli_score[0]: temp_list5 =[] temp_list = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list5!=None: element_data.id_info = temp_list5 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "broccoli" and get_msg.score[i] == broccoli_score[1]: temp_list6 = [] temp_list6 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list6!=None: element_data.id_info = temp_list6 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "broccoli" and get_msg.score[i] == broccoli_score[2]: temp_list7 = [] temp_list7 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list7!=None: element_data.id_info = temp_list7 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "souppothandle" and get_msg.score[i] == souppothandle_score[0]: temp_list8 = [] temp_list8 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list8!=None: element_data.id_info = temp_list8 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) ''' ''' if get_msg.classname[i] == "nethandle" and get_msg.score[i] == nethandle_score[0]: temp_list9 = [] temp_list9 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list9!=None: element_data.id_info = temp_list9 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "vegetablebowl" and get_msg.score[i] == vegetablebowl_score[0]: temp_list10 = [] temp_list10 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list10!=None: element_data.id_info = temp_list10 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) ''' if get_msg.classname[i] == "seasoningbowl" and get_msg.score[ i] == seasoningbowl_score[0]: temp_list11 = [] temp_list11 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list11 != None: element_data.id_info = temp_list11 single_obj.element_data_temp.append(element_data) #single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "seasoningbottle": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > bottle1[0] and center_x < bottle1[ 2] and center_y > bottle1[1] and center_y < bottle1[3]: temp_list12 = [] temp_list12 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list12 != None: single_obj.classname.append("seasoningbottle1") element_data.id_info = temp_list12 single_obj.element_data_temp.append(element_data) #single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) if get_msg.classname[i] == "seasoningbottle": center_x = int(get_msg.roi[i].x_offset + get_msg.roi[i].height / 2) # rect area center center_y = int(get_msg.roi[i].y_offset + get_msg.roi[i].width / 2) # rect area center if center_x > bottle2[0] and center_x < bottle2[ 2] and center_y > bottle2[1] and center_y < bottle2[3]: temp_list13 = [] temp_list13 = prcess_contours( cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list13 != None: single_obj.classname.append("seasoningbottle2") element_data.id_info = temp_list13 single_obj.element_data_temp.append(element_data) #single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) ''' if get_msg.classname[i] == "panhandle" and get_msg.score[i] == panhandle_score[0]: temp_list14 = [] temp_list14 = prcess_contours(cv_image, get_msg.classname[i], mask_image, get_msg.id[i], get_msg.roi[i].x_offset, get_msg.roi[i].y_offset, get_msg.roi[i].height, get_msg.roi[i].width, get_msg.score[i]) if temp_list14!=None: element_data.id_info = temp_list14 single_obj.element_data_temp.append(element_data) single_obj.classname.append(get_msg.classname[i]) single_obj.score.append(get_msg.score[i]) ''' #print("88888",single_obj) if len(single_obj.score) > 0: #print("1111:",single_obj) temp_single_obj = single_obj #print("****",temp_single_obj) list_obj.single_obj_info.append(single_obj) else: list_obj.single_obj_info.append(temp_single_obj) #print("1111:",list_obj.single_obj_info) #list_obj.header.stamp = data.header.stamp #list_obj.header.frame_id = data.header.frame_id print("---" * 20) #print("--ww--",list_obj.header) list_obj.rgb_img_to_pos = bridge.cv2_to_imgmsg(rgb_img_to_pos, encoding="bgr8") #list_obj.depth_img_to_pos = bridge.cv2_to_imgmsg(depth_img_to_pos,"passthrough") temp_det_bar_pos = Float32MultiArray(data=det_bar_pos) #print("1111:",list_obj) global list_obj_pub global temp_det_bar_pos_pub global list_broc_pub list_broc_pub.publish(brocili_list) list_obj_pub.publish(list_obj) temp_det_bar_pos_pub.publish(temp_det_bar_pos) log.info("Got broc center: get_time: {},center_pos_u_v: {}".format( gettime, brocili_list)) log.info("Got bar pos: get_time: {},pos_info_u_v: {}".format( gettime, temp_det_bar_pos)) cv2.rectangle(cv_image, (plate2[0], plate2[1]), (plate2[0] + (plate2[2] - plate2[0]), plate2[1] + (plate2[3] - plate2[1])), (0, 0, 255), 1) cv2.rectangle(cv_image, (plate1[0], plate1[1]), (plate1[0] + (plate1[2] - plate1[0]), plate1[1] + (plate1[3] - plate1[1])), (0, 0, 255), 1) cv2.rectangle(cv_image, (plate3[0], plate3[1]), (plate3[0] + (plate3[2] - plate3[0]), plate3[1] + (plate3[3] - plate3[1])), (0, 0, 255), 1) cv2.rectangle(cv_image, (pan_roi[0], pan_roi[1]), (pan_roi[0] + (pan_roi[2] - pan_roi[0]), pan_roi[1] + (pan_roi[3] - pan_roi[1])), (0, 0, 255), 1) cv2.rectangle(cv_image, (beef_roi[0], beef_roi[1]), (beef_roi[0] + (beef_roi[2] - beef_roi[0]), beef_roi[1] + (beef_roi[3] - beef_roi[1])), (0, 0, 255), 1) #cv2.rectangle(cv_image,(bottle1[0], bottle1[1]),(bottle1[0]+(bottle1[2]-bottle1[0])/2, bottle1[1]+(bottle1[3]-bottle1[1])),(0,0,255),2) #cv2.rectangle(cv_image,(bottle2[0], bottle2[1]),(bottle2[0]+(bottle2[2]-bottle2[0])/2, bottle2[1]+(bottle2[3]-bottle2[1])),(0,0,255),2) cv2.imshow("img2", cv_image) cv2.waitKey(1)
int(press3_y), int(press4_x), int(press4_y), int(width), int(height) ] #print("111111111:",list_info) return list_info img_out1 = np.zeros((540, 960), dtype=np.uint8) img_out2 = np.zeros((540, 960), dtype=np.uint8) img_out3 = np.zeros((540, 960), dtype=np.uint8) det_bar_pos = np.zeros(12, dtype=float) temp_single_obj = sglobj() #postion_list = [left_plate1,right_plate1,up_plate1,down_plate1] plate1 = [119, 424, 230, 530] #(90,415) (190,525) plate2 = [690, 270, 795, 370] #(675,260) (775,340) plate3 = [695, 395, 825, 525] #(680,375) (795,475) pan_roi = [340, 260, 480, 408] #(680,375) (795,475) beef_roi = [170, 275, 480, 400] bottle1 = [60, 70, 150, 150] bottle2 = [170, 70, 240, 150] def callback(data): list_obj = listobj()