def draw_boxes_and_objects(self,boxes, img, cls_names, detection_size, is_letter_box_image): draw = ImageDraw.Draw(img) if not self.objects: self.first_time = True # For every detected box from all clases for cls, bboxs in boxes.items(): color = tuple(np.random.randint(0, 256, 3)) for box, score in bboxs: # Box processing, changing from Yolo format box = convert_to_original_size(box, np.array(detection_size), np.array(img.size), is_letter_box_image) # Create and update tracking objects from the boxes self.tracking_objects(box) # Draw boxes and names draw.rectangle(box, outline=colors_array[cls]) draw.text(box[:2], '{} {:.2f}%'.format( cls_names[cls], score * 100), fill=colors_array[cls]) # For all detected objects if self.objects: for key in self.objects.keys(): # r is the probability of existence, and it determines the radius of the circle r = self.update(key) print(key) draw.ellipse((self.objects[key]['X'] - r, self.objects[key]['Y'] - r, self.objects[key]['X'] + r, self.objects[key]['Y'] + r), fill=(255, 0, 0, 255)) #if self.objects[key]['Prob'] < 0.2: # del self.objects[key] # self.discarded.append(key) for key in [key for key in self.objects if self.objects[key]['Prob'] < 0.3]: del self.objects[key] self.discarded.append(key)
def get_classification(self, cv_image): """Determines the color of the traffic light in the image Args: image (cv::Mat): image containing the traffic light Returns: int: ID of traffic light color (specified in styx_msgs/TrafficLight) """ #TODO implement light color prediction image = Image.fromarray(cv_image) img_resized = letter_box_image(image, options['image_size'], options['image_size'], 128) img_resized = img_resized.astype(np.float32) boxes, inputs = get_boxes_and_inputs_pb(self.frozenGraph) # with tf.Session(graph=self.frozenGraph, config=self.config) as sess: t0 = time.time() detected_boxes = self.sess.run(boxes, feed_dict={inputs: [img_resized]}) filtered_boxes = non_max_suppression( detected_boxes, confidence_threshold=options['thresh'], iou_threshold=options['iou']) print("Predictions found in {:.2f}s".format(time.time() - t0)) inp = filtered_boxes.get(9) inp_new = dict() inp_new[9] = inp if (inp_new[9] != None): if (len(inp_new[9]) > 0): for cls, bboxs in inp_new.items(): for box, score in bboxs: box = convert_to_original_size( box, (options['image_size'], options['image_size']), np.array(image.size), True) # print(inp_new) a = analyze_color(inp_new, cv_image) # print(a) light_color = state_predict(a) print("the light color is {}".format(light_color)) if light_color: if light_color == 'YELLOW': return TrafficLight.YELLOW elif light_color == 'RED': return TrafficLight.RED elif light_color == 'GREEN': return TrafficLight.GREEN return TrafficLight.UNKNOWN
def evaluate(filtered_boxes, gt_anno, orig_img, thresh=0.5): tp = 0 fp = 0 highest_conf = 0.0 highest_conf_label = -1 gt_label = [key for key in gt_anno.keys()][0] bbox_list = [] # bbox_dict: [bbox_infomation, cls_id] for cls, bboxs in filtered_boxes.items(): for bbox in bboxs: bbox_list.append([bbox, cls]) iou_l = [] total_positives = len(bbox_list) for [bbox, cls] in bbox_list: _bbox = copy.deepcopy(bbox[0]) orig_scale_bbox = convert_to_original_size(_bbox, np.array((cfg.IMAGE_SIZE, cfg.IMAGE_SIZE)), np.array(orig_img.size), True) # 元のscaleに戻す hoge = [] for gt_bbox in gt_anno.values(): # gtのbbox毎の処理 iou_ = _iou( orig_scale_bbox, gt_bbox ) # :param box1: array of 4 values (top left and bottom right coords): [x0, y0, x1, x2] hoge.append(iou_) iou = max(hoge) # 一番大きい値をとる iou_l.append(iou) if cls == gt_label: # classが一致しているか if iou > thresh: tp += 1 # labelが一致かつiou>threshの時TPを+1 else: fp += 1 else: fp += 1 if bbox[1] > highest_conf: highest_conf = bbox[1] highest_conf_label = cls precision = tp / total_positives average_iou = sum(iou_l) / (len(iou_l) + 1e-05) # 画像一枚のIoU fn = len(gt_anno.values()) - ( tp + fp) if len(gt_anno.values()) - (tp + fp) > 0 else 0 return [tp, fp, fn], average_iou, precision, highest_conf_label
def draw_boxes_and_objects(self,boxes, img, cls_names, detection_size, is_letter_box_image): draw = ImageDraw.Draw(img) if not self.objects: self.first_time = True # For every detected box from all clases for cls, bboxs in boxes.items(): color = tuple(np.random.randint(0, 256, 3)) for box, score in bboxs: # Box processing, changing from Yolo format box = convert_to_original_size(box, np.array(detection_size), np.array(img.size), is_letter_box_image) # Draw boxes and names if cls ==0: draw.rectangle(box, outline=colors_array[cls],fill=(0,0,0))
def validate_fn(dataset, net, validate_file_name=None, logger=None): if validate_file_name is None: validate_file_name = "./tmp/validate_result.json" result_data: List[TuSimpleLabel] = deepcopy(dataset.label_data) # Fixme: hard code target_h = list(range(160, 720, 10)) x_size = 512.0 y_size = 256.0 net.evaluate_mode() pbar = tqdm(total=len(result_data)) for testset_index, sample in enumerate(dataset): ratio_w = x_size / sample["original_size"][1] ratio_h = y_size / sample["original_size"][0] x, y, img = net.test_on_image(sample["image"], threshold_confidence=0.81) x, y = convert_to_original_size(x[0], y[0], ratio_w, ratio_h) x, y = find_target(x, y, target_h, ratio_w, ratio_h) result_data = write_result_json(result_data, x, y, testset_index) pbar.set_description(f'test image id {testset_index}') pbar.update() pbar.close() with open(validate_file_name, 'w') as make_file: for i in result_data: json.dump(i, make_file, separators=(',', ': ')) make_file.write("\n") metrics = LaneEval.bench_one_submit(validate_file_name, "./data/test_label.json") if logger is not None: logger.log_metric("validate accuracy", metrics[0]["value"]) logger.log_metric("validate FP", metrics[1]["value"]) logger.log_metric("validate FN", metrics[2]["value"]) return metrics
def run(self, conf_thresh=0.3, iou_thresh=0.3): ''' :param namefile: :param inputsize: Imgsize to input to the model:(w,h) :param conf_thresh: :param iou_thresh: :return: ''' self.mprint('start--------') self.mprint('net model path: {}'.format(self.netModelPath)) ckpt_path = os.path.join(self.netModelPath, 'bird191125.pb') input_size = (input_image_cols, input_image_rows) detection_graph = tf.Graph() sess = tf.Session(graph=detection_graph) with detection_graph.as_default(): # initial self.mprint('initial graph') od_graph_def = tf.GraphDef() with tf.gfile.GFile(ckpt_path, 'rb') as fid: serialized_graph = fid.read() od_graph_def.ParseFromString(serialized_graph) tf.import_graph_def(od_graph_def, name='') self.mprint('load ckpt done') # Get handles to input and output tensors boxes = tf.get_default_graph().get_tensor_by_name("output_boxes:0") inputs = tf.get_default_graph().get_tensor_by_name("inputs:0") with tf.Session() as sess: # call loop-------------------------- self.mprint('graph initial done, enter call loop') while (self.isRun): images_list = self.inQueue.get() # block if images_list == END_FLAG: break # prepare numpy.array images batch num_imgs = len(images_list) image_batch_np = np.zeros( (num_imgs, input_image_rows, input_image_cols, 3)) t0 = time.time() for k in range(num_imgs): image_batch_np[k] = utils.resize_cv2( images_list[k], input_size) t1 = time.time() self.mprint('resize images time cost {}s'.format(t1 - t0)) # Actual detection. time_start = time.time() # output_dict = run_inference_for_image_batch(image_batch_np, sess, boxes, inputs) detected_boxes = sess.run( boxes, feed_dict={inputs: image_batch_np}) filtered_boxes = utils.non_max_suppression( detected_boxes, confidence_threshold=conf_thresh, iou_threshold=iou_thresh) time_end = time.time() self.mprint('inference image batch time cost: {}s'.format( time_end - time_start)) # boxes_batch[k] is [[xmin,ymin,xmax,ymax,class_id,probility], [...],...] boxes_batch = [] for i, batch in enumerate(filtered_boxes): boxes_list = [] for cls, bboxs in batch.items(): for box, score in bboxs: box = utils.convert_to_original_size( box, np.array(input_size), np.array(images_list[i].shape[:2][::-1])) box.extend([cls, score]) boxes_list.append(box) boxes_batch.append(boxes_list) # print('len boxes_batches:',len(boxes_batch)) self.mprint('result: {}'.format(boxes_batch)) self.outQueue.put(boxes_batch) print("self.isrun", self.isRun) # call loop end-------------------------- self.mprint('over------')
def main(argv=None): gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.GPU_MEMORY_FRACTION) config = tf.ConfigProto( gpu_options=gpu_options, log_device_placement=False, ) classes = load_coco_names(cfg.CLASS_NAME) if cfg.FROZEN_MODEL: pass # # t0 = time.time() # frozenGraph = load_graph(cfg.FROZEN_MODEL) # print("Loaded graph in {:.2f}s".format(time.time()-t0)) # # boxes, inputs = get_boxes_and_inputs_pb(frozenGraph) # # with tf.Session(graph=frozenGraph, config=config) as sess: # t0 = time.time() # detected_boxes = sess.run( # boxes, feed_dict={inputs: [img_resized]}) else: if cfg.TINY: model = yolo_v3_tiny.yolo_v3_tiny else: model = yolo_v3.yolo_v3 boxes, inputs = get_boxes_and_inputs(model, len(classes), cfg.IMAGE_SIZE, cfg.DATA_FORMAT) # boxes : coordinates of top left and bottom right points. saver = tf.train.Saver(var_list=tf.global_variables(scope='detector')) # # for specific object recognition # vgg16_image_size = vgg_16.default_image_size s_class_names = cfg.S_CLASS_PATH s_classes = [l.split(" ") for l in open(s_class_names, "r")] if len(s_classes[0]): # classフォーマットが "id classname"の場合 s_labels = {int(l[0]): l[1].replace("\n", "") for l in s_classes} else: # classフォーマットが "classname"のみの場合 s_labels = { i: l.replace("\n", "") for i, l in enumerate(s_classes) } num_classes_s = len(s_labels.keys()) num_classes_extractor = cfg.S_EXTRACTOR_NUM_OF_CLASSES s_model = cfg.S_CKPT_FILE extractor_name = cfg.S_EXTRACTOR_NAME # specific_pred, [cropped_images_placeholder, original_images_placeholder, keep_prob, is_training] = specific_object_recognition(vgg16_image_size, num_classes_s, num_classes_extractor, extractor_name) specific_pred, [cropped_images_placeholder, keep_prob, is_training] = specific_object_recognition( vgg16_image_size, num_classes_s) variables_to_restore = slim.get_variables_to_restore( include=["vgg_16"]) restorer = tf.train.Saver(variables_to_restore) with tf.Session(config=config) as sess: t0 = time.time() saver.restore(sess, cfg.CKPT_FILE) print('YOLO v3 Model restored in {:.2f}s'.format(time.time() - t0), "from:", cfg.CKPT_FILE) t0 = time.time() restorer.restore(sess, s_model) print( 'Specific object recognition Model restored in {:.2f}s'.format( time.time() - t0), "from:", s_model) # prepare test set with open(cfg.TEST_FILE_PATH, 'r') as f: f_ = [line.rstrip().split() for line in f] data = [ [l, get_annotation(l[0], txtname=cfg.GT_INFO_FILE_NAME)] for l in f_ ] # data: [[(path_str, label), [frame, center_x, center_y, size_x, size_y]],...] data = [l for l in data if l[1] is not None] # annotationを取得できなかった画像は飛ばす def is_cropped_file_Exist(orig_filepath): d, file = os.path.split(orig_filepath) cropped_d = d + "_cropped" cropped_file = os.path.join(cropped_d, file) return os.path.exists(cropped_file) data = [l for l in data if is_cropped_file_Exist(l[0][0])] # 対となるcrop画像がない画像は飛ばす # log f = open(cfg.OUTPUT_LOG_PATH, 'w') writer = csv.writer(f, lineterminator='\n') writer.writerow([ 'image path', 'movie_name', 'IoU', 'Average Precision', 'Recall', 'is RoI detected?', 'is label correct?', 'gt label', 'pred label', 'detect time', 'recog time' ]) iou_list = [] # 画像毎のiouのリスト ap_list = [] # 画像毎のaverage precisionのリスト # iterative run for count, gt in enumerate( data ): # gt: [(path_str, label), [frame, center_x, center_y, size_x, size_y] # for evaluation gt_box = [float(i) for i in gt[1][1:]] gt_box = [ gt_box[0] - (gt_box[2] / 2), gt_box[1] - (gt_box[3] / 2), gt_box[0] + (gt_box[2] / 2), gt_box[1] + (gt_box[3] / 2) ] gt_label = int(gt[0][1]) ious = [] precisions = [] print(count, ":", gt[0][0]) img = Image.open(gt[0][0]) img_resized = letter_box_image(img, cfg.IMAGE_SIZE, cfg.IMAGE_SIZE, 128) img_resized = img_resized.astype(np.float32) t0 = time.time() detected_boxes = sess.run(boxes, feed_dict={inputs: [img_resized]}) filtered_boxes = non_max_suppression( detected_boxes, confidence_threshold=cfg.CONF_THRESHOLD, iou_threshold=cfg.IOU_THRESHOLD) detect_time = time.time() - t0 print("detected boxes in :{:.2f}s ".format(detect_time), filtered_boxes) # specific object recognition! np_img = np.array(img) / 255 target_label = 0 # seesaaの場合 (データセットのクラス番号毎にここを変える.) if len(filtered_boxes.keys()) != 0: # 何かしら検出された時 is_detected = True for cls, bboxs in filtered_boxes.items(): if cls == target_label: # ターゲットラベルなら print("target class detected!") bounding_boxes = [] bboxs_ = copy.deepcopy( bboxs ) # convert_to_original_size()がbboxを破壊してしまうため for box, score in bboxs: orig_size_box = convert_to_original_size( box, np.array((cfg.IMAGE_SIZE, cfg.IMAGE_SIZE)), np.array(img.size), True) # print(orig_size_box) cropped_image = np_img[ int(orig_size_box[1]):int(orig_size_box[3] ), int(orig_size_box[0]):int(orig_size_box[2] )] bounding_boxes.append(cropped_image) # input_original = cv2.resize(padding(np_img), (vgg16_image_size, vgg16_image_size)) # input_original = np.tile(input_original, (len(bounding_boxes), 1, 1, 1)) # croppedと同じ枚数分画像を重ねる cropped_images = [] for bbox in bounding_boxes: cropped_images.append( cv2.resize( padding(bbox), (vgg16_image_size, vgg16_image_size))) input_cropped = np.asarray(cropped_images) t0 = time.time() pred = sess.run(specific_pred, feed_dict={ cropped_images_placeholder: input_cropped, keep_prob: 1.0, is_training: False }) recog_time = time.time() - t0 print("Predictions found in {:.2f}s".format( recog_time)) pred_label = [s_labels[i] for i in pred.tolist() ] # idからクラス名を得る classes = [ s_labels[i] for i in range(num_classes_s) ] filtered_boxes = {} for i, n in enumerate(pred.tolist()): if n in filtered_boxes.keys(): filtered_boxes[n].extend([bboxs_[i]]) else: filtered_boxes[n] = [bboxs_[i]] # calc IoU, mAP # gt: [(path_str, label), [frame, center_x, center_y, size_x, size_y] # print(filtered_boxes) iou = 0.0 for key in filtered_boxes.keys(): for pred_box in filtered_boxes[key]: p_box = copy.deepcopy(pred_box[0]) orig_scale_p_box = convert_to_original_size( p_box, np.array( (cfg.IMAGE_SIZE, cfg.IMAGE_SIZE)), np.array(img.size), True) conf = pred_box[1] # print(gt_label, key) if key == gt_label: # 予測したクラスがGTと同じの時 # print(orig_scale_p_box, gt_box) iou = _iou( orig_scale_p_box, gt_box ) # :param box1: array of 4 values (top left and bottom right coords): [x0, y0, x1, x2] precision = calc_precision( orig_scale_p_box, gt_box) is_label_correct = True else: iou = 0.0 precision = 0.0 is_label_correct = False # print("IoU:", iou) ious.append(iou) print("Precision:", precision) precisions.append(precision) else: # ターゲットラベルじゃない時 pass else: #何も検出されなかった時 is_detected = False is_label_correct = "None" pred_label = ["None"] average_iou = sum(ious) / (len(ious) + 1e-05) # 画像一枚のiou print("average IoU:", average_iou) iou_list.append(average_iou) print("mean average IoU:", sum(iou_list) / (len(iou_list) + 1e-05)) ap = sum(precisions) / (len(precisions) + 1e-05) ap_list.append(ap) print("Average Precision:", ap) print("mean Average Precision:", sum(ap_list) / (len(ap_list) + 1e-05)) draw_boxes(filtered_boxes, img, classes, (cfg.IMAGE_SIZE, cfg.IMAGE_SIZE), True) # draw GT draw = ImageDraw.Draw(img) color = (0, 0, 0) draw.rectangle(gt_box, outline=color) draw.text(gt_box[:2], 'GT_' + s_labels[gt_label], fill=color) img.save( os.path.join( cfg.OUTPUT_IMAGE_DIR, '{0:04d}_'.format(count) + os.path.basename(gt[0][0]))) writer.writerow([ gt[0][0], os.path.basename(os.path.dirname(gt[0][0])), average_iou, ap, 'Recall', is_detected, is_label_correct, s_labels[gt_label], pred_label[0], detect_time, recog_time ]) f.close() print("proc finished.")
original_width = image.shape[1] resized_image = utils.resize_image(image, 416) resized_image = np.expand_dims(resized_image, 0) detections = sess.run(predictions, feed_dict={inputs: resized_image}) result = utils.torch_non_max_suppression( detections=torch.from_numpy(detections), confidence_threshold=0.25, num_classes=22, nms_conf=0.4) if result is not None: for box in result: p1 = (int(box[1]), int(box[2])) p1 = utils.convert_to_original_size( p1, original_height, original_width, 416, 416) p2 = (int(box[3]), int(box[4])) p2 = utils.convert_to_original_size( p2, original_height, original_width, 416, 416) class_name = int(box[-1]) class_name = classes[class_name] confidence = box[5] * 100. print('%s: %.2f%%' % (class_name, confidence)) image = utils.draw_bounding_box_with_text( image, p1, p2, class_name, box_color=(0, 0, 255), text_color=(255, 255, 255)) cv2.imshow(os.path.basename(item), image)
x1, y1, x2, y2 = utils.yolo_ratios_to_real_voc(true_box[1], true_box[2], true_box[3], true_box[4], original_height, original_width) image = utils.draw_bounding_box_with_text(image=image, top_left=(int(x1), int(y1)), right_bottom=(int(x2), int(y2)), text=true_class_name, box_color=(0, 255, 255), text_color=(0, 0, 0)) if result is not None: for box in result: # Draw bounding box for visualization p1 = (int(box[1]), int(box[2])) p1 = utils.convert_to_original_size(p1, original_height, original_width, model.HEIGHT, model.WIDTH) p2 = (int(box[3]), int(box[4])) p2 = utils.convert_to_original_size(p2, original_height, original_width, model.HEIGHT, model.WIDTH) class_index = int(box[-1]) class_name = classes[class_index] image = utils.draw_bounding_box_with_text(image=image, top_left=p1, right_bottom=p2, text=class_name, box_color=(0, 0, 255), text_color=(255, 255, 255)) cv2.imshow(os.path.basename(image_path), image) cv2.waitKey(0) cv2.destroyAllWindows()