def draw_boxes_and_objects(self,boxes, img, cls_names, detection_size, is_letter_box_image):
draw = ImageDraw.Draw(img)
if not self.objects:
self.first_time = True
# For every detected box from all clases
for cls, bboxs in boxes.items():
color = tuple(np.random.randint(0, 256, 3))
for box, score in bboxs:
# Box processing, changing from Yolo format
box = convert_to_original_size(box, np.array(detection_size),
np.array(img.size),
is_letter_box_image)
# Create and update tracking objects from the boxes
self.tracking_objects(box)
# Draw boxes and names
draw.rectangle(box, outline=colors_array[cls])
draw.text(box[:2], '{} {:.2f}%'.format(
cls_names[cls], score * 100), fill=colors_array[cls])
# For all detected objects
if self.objects:
for key in self.objects.keys():
# r is the probability of existence, and it determines the radius of the circle
r = self.update(key)
print(key)
draw.ellipse((self.objects[key]['X'] - r, self.objects[key]['Y'] - r, self.objects[key]['X'] + r, self.objects[key]['Y'] + r),
fill=(255, 0, 0, 255))
#if self.objects[key]['Prob'] < 0.2:
# del self.objects[key]
# self.discarded.append(key)
for key in [key for key in self.objects if self.objects[key]['Prob'] < 0.3]:
del self.objects[key]
self.discarded.append(key)
def get_classification(self, cv_image):
"""Determines the color of the traffic light in the image
Args:
image (cv::Mat): image containing the traffic light
Returns:
int: ID of traffic light color (specified in styx_msgs/TrafficLight)
"""
#TODO implement light color prediction
image = Image.fromarray(cv_image)
img_resized = letter_box_image(image, options['image_size'],
options['image_size'], 128)
img_resized = img_resized.astype(np.float32)
boxes, inputs = get_boxes_and_inputs_pb(self.frozenGraph)
# with tf.Session(graph=self.frozenGraph, config=self.config) as sess:
t0 = time.time()
detected_boxes = self.sess.run(boxes,
feed_dict={inputs: [img_resized]})
filtered_boxes = non_max_suppression(
detected_boxes,
confidence_threshold=options['thresh'],
iou_threshold=options['iou'])
print("Predictions found in {:.2f}s".format(time.time() - t0))
inp = filtered_boxes.get(9)
inp_new = dict()
inp_new[9] = inp
if (inp_new[9] != None):
if (len(inp_new[9]) > 0):
for cls, bboxs in inp_new.items():
for box, score in bboxs:
box = convert_to_original_size(
box,
(options['image_size'], options['image_size']),
np.array(image.size), True)
# print(inp_new)
a = analyze_color(inp_new, cv_image)
# print(a)
light_color = state_predict(a)
print("the light color is {}".format(light_color))
if light_color:
if light_color == 'YELLOW':
return TrafficLight.YELLOW
elif light_color == 'RED':
return TrafficLight.RED
elif light_color == 'GREEN':
return TrafficLight.GREEN
return TrafficLight.UNKNOWN
def evaluate(filtered_boxes, gt_anno, orig_img, thresh=0.5):
tp = 0
fp = 0
highest_conf = 0.0
highest_conf_label = -1
gt_label = [key for key in gt_anno.keys()][0]
bbox_list = [] # bbox_dict: [bbox_infomation, cls_id]
for cls, bboxs in filtered_boxes.items():
for bbox in bboxs:
bbox_list.append([bbox, cls])
iou_l = []
total_positives = len(bbox_list)
for [bbox, cls] in bbox_list:
_bbox = copy.deepcopy(bbox[0])
orig_scale_bbox = convert_to_original_size(_bbox,
np.array((cfg.IMAGE_SIZE,
cfg.IMAGE_SIZE)),
np.array(orig_img.size),
True) # 元のscaleに戻す
hoge = []
for gt_bbox in gt_anno.values(): # gtのbbox毎の処理
iou_ = _iou(
orig_scale_bbox, gt_bbox
) # :param box1: array of 4 values (top left and bottom right coords): [x0, y0, x1, x2]
hoge.append(iou_)
iou = max(hoge) # 一番大きい値をとる
iou_l.append(iou)
if cls == gt_label: # classが一致しているか
if iou > thresh:
tp += 1 # labelが一致かつiou>threshの時TPを+1
else:
fp += 1
else:
fp += 1
if bbox[1] > highest_conf:
highest_conf = bbox[1]
highest_conf_label = cls
precision = tp / total_positives
average_iou = sum(iou_l) / (len(iou_l) + 1e-05) # 画像一枚のIoU
fn = len(gt_anno.values()) - (
tp + fp) if len(gt_anno.values()) - (tp + fp) > 0 else 0
return [tp, fp, fn], average_iou, precision, highest_conf_label
def draw_boxes_and_objects(self,boxes, img, cls_names, detection_size, is_letter_box_image):
draw = ImageDraw.Draw(img)
if not self.objects:
self.first_time = True
# For every detected box from all clases
for cls, bboxs in boxes.items():
color = tuple(np.random.randint(0, 256, 3))
for box, score in bboxs:
# Box processing, changing from Yolo format
box = convert_to_original_size(box, np.array(detection_size),
np.array(img.size),
is_letter_box_image)
# Draw boxes and names
if cls ==0:
draw.rectangle(box, outline=colors_array[cls],fill=(0,0,0))
def validate_fn(dataset, net, validate_file_name=None, logger=None):
if validate_file_name is None:
validate_file_name = "./tmp/validate_result.json"
result_data: List[TuSimpleLabel] = deepcopy(dataset.label_data)
# Fixme: hard code
target_h = list(range(160, 720, 10))
x_size = 512.0
y_size = 256.0
net.evaluate_mode()
pbar = tqdm(total=len(result_data))
for testset_index, sample in enumerate(dataset):
ratio_w = x_size / sample["original_size"][1]
ratio_h = y_size / sample["original_size"][0]
x, y, img = net.test_on_image(sample["image"],
threshold_confidence=0.81)
x, y = convert_to_original_size(x[0], y[0], ratio_w, ratio_h)
x, y = find_target(x, y, target_h, ratio_w, ratio_h)
result_data = write_result_json(result_data, x, y, testset_index)
pbar.set_description(f'test image id {testset_index}')
pbar.update()
pbar.close()
with open(validate_file_name, 'w') as make_file:
for i in result_data:
json.dump(i, make_file, separators=(',', ': '))
make_file.write("\n")
metrics = LaneEval.bench_one_submit(validate_file_name,
"./data/test_label.json")
if logger is not None:
logger.log_metric("validate accuracy", metrics[0]["value"])
logger.log_metric("validate FP", metrics[1]["value"])
logger.log_metric("validate FN", metrics[2]["value"])
return metrics
def run(self, conf_thresh=0.3, iou_thresh=0.3):
'''
:param namefile:
:param inputsize: Imgsize to input to the model:(w,h)
:param conf_thresh:
:param iou_thresh:
:return:
'''
self.mprint('start--------')
self.mprint('net model path: {}'.format(self.netModelPath))
ckpt_path = os.path.join(self.netModelPath, 'bird191125.pb')
input_size = (input_image_cols, input_image_rows)
detection_graph = tf.Graph()
sess = tf.Session(graph=detection_graph)
with detection_graph.as_default():
# initial
self.mprint('initial graph')
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(ckpt_path, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
self.mprint('load ckpt done')
# Get handles to input and output tensors
boxes = tf.get_default_graph().get_tensor_by_name("output_boxes:0")
inputs = tf.get_default_graph().get_tensor_by_name("inputs:0")
with tf.Session() as sess:
# call loop--------------------------
self.mprint('graph initial done, enter call loop')
while (self.isRun):
images_list = self.inQueue.get() # block
if images_list == END_FLAG:
break
# prepare numpy.array images batch
num_imgs = len(images_list)
image_batch_np = np.zeros(
(num_imgs, input_image_rows, input_image_cols, 3))
t0 = time.time()
for k in range(num_imgs):
image_batch_np[k] = utils.resize_cv2(
images_list[k], input_size)
t1 = time.time()
self.mprint('resize images time cost {}s'.format(t1 - t0))
# Actual detection.
time_start = time.time()
# output_dict = run_inference_for_image_batch(image_batch_np, sess, boxes, inputs)
detected_boxes = sess.run(
boxes, feed_dict={inputs: image_batch_np})
filtered_boxes = utils.non_max_suppression(
detected_boxes,
confidence_threshold=conf_thresh,
iou_threshold=iou_thresh)
time_end = time.time()
self.mprint('inference image batch time cost: {}s'.format(
time_end - time_start))
# boxes_batch[k] is [[xmin,ymin,xmax,ymax,class_id,probility], [...],...]
boxes_batch = []
for i, batch in enumerate(filtered_boxes):
boxes_list = []
for cls, bboxs in batch.items():
for box, score in bboxs:
box = utils.convert_to_original_size(
box, np.array(input_size),
np.array(images_list[i].shape[:2][::-1]))
box.extend([cls, score])
boxes_list.append(box)
boxes_batch.append(boxes_list)
# print('len boxes_batches:',len(boxes_batch))
self.mprint('result: {}'.format(boxes_batch))
self.outQueue.put(boxes_batch)
print("self.isrun", self.isRun)
# call loop end--------------------------
self.mprint('over------')
def main(argv=None):
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.GPU_MEMORY_FRACTION)
config = tf.ConfigProto(
gpu_options=gpu_options,
log_device_placement=False,
)
classes = load_coco_names(cfg.CLASS_NAME)
if cfg.FROZEN_MODEL:
pass
#
# t0 = time.time()
# frozenGraph = load_graph(cfg.FROZEN_MODEL)
# print("Loaded graph in {:.2f}s".format(time.time()-t0))
#
# boxes, inputs = get_boxes_and_inputs_pb(frozenGraph)
#
# with tf.Session(graph=frozenGraph, config=config) as sess:
# t0 = time.time()
# detected_boxes = sess.run(
# boxes, feed_dict={inputs: [img_resized]})
else:
if cfg.TINY:
model = yolo_v3_tiny.yolo_v3_tiny
else:
model = yolo_v3.yolo_v3
boxes, inputs = get_boxes_and_inputs(model, len(classes),
cfg.IMAGE_SIZE, cfg.DATA_FORMAT)
# boxes : coordinates of top left and bottom right points.
saver = tf.train.Saver(var_list=tf.global_variables(scope='detector'))
#
# for specific object recognition
#
vgg16_image_size = vgg_16.default_image_size
s_class_names = cfg.S_CLASS_PATH
s_classes = [l.split(" ") for l in open(s_class_names, "r")]
if len(s_classes[0]): # classフォーマットが "id classname"の場合
s_labels = {int(l[0]): l[1].replace("\n", "") for l in s_classes}
else: # classフォーマットが "classname"のみの場合
s_labels = {
i: l.replace("\n", "")
for i, l in enumerate(s_classes)
}
num_classes_s = len(s_labels.keys())
num_classes_extractor = cfg.S_EXTRACTOR_NUM_OF_CLASSES
s_model = cfg.S_CKPT_FILE
extractor_name = cfg.S_EXTRACTOR_NAME
# specific_pred, [cropped_images_placeholder, original_images_placeholder, keep_prob, is_training] = specific_object_recognition(vgg16_image_size, num_classes_s, num_classes_extractor, extractor_name)
specific_pred, [cropped_images_placeholder, keep_prob,
is_training] = specific_object_recognition(
vgg16_image_size, num_classes_s)
variables_to_restore = slim.get_variables_to_restore(
include=["vgg_16"])
restorer = tf.train.Saver(variables_to_restore)
with tf.Session(config=config) as sess:
t0 = time.time()
saver.restore(sess, cfg.CKPT_FILE)
print('YOLO v3 Model restored in {:.2f}s'.format(time.time() - t0),
"from:", cfg.CKPT_FILE)
t0 = time.time()
restorer.restore(sess, s_model)
print(
'Specific object recognition Model restored in {:.2f}s'.format(
time.time() - t0), "from:", s_model)
# prepare test set
with open(cfg.TEST_FILE_PATH, 'r') as f:
f_ = [line.rstrip().split() for line in f]
data = [
[l, get_annotation(l[0], txtname=cfg.GT_INFO_FILE_NAME)]
for l in f_
] # data: [[(path_str, label), [frame, center_x, center_y, size_x, size_y]],...]
data = [l for l in data
if l[1] is not None] # annotationを取得できなかった画像は飛ばす
def is_cropped_file_Exist(orig_filepath):
d, file = os.path.split(orig_filepath)
cropped_d = d + "_cropped"
cropped_file = os.path.join(cropped_d, file)
return os.path.exists(cropped_file)
data = [l for l in data
if is_cropped_file_Exist(l[0][0])] # 対となるcrop画像がない画像は飛ばす
# log
f = open(cfg.OUTPUT_LOG_PATH, 'w')
writer = csv.writer(f, lineterminator='\n')
writer.writerow([
'image path', 'movie_name', 'IoU', 'Average Precision',
'Recall', 'is RoI detected?', 'is label correct?', 'gt label',
'pred label', 'detect time', 'recog time'
])
iou_list = [] # 画像毎のiouのリスト
ap_list = [] # 画像毎のaverage precisionのリスト
# iterative run
for count, gt in enumerate(
data
): # gt: [(path_str, label), [frame, center_x, center_y, size_x, size_y]
# for evaluation
gt_box = [float(i) for i in gt[1][1:]]
gt_box = [
gt_box[0] - (gt_box[2] / 2), gt_box[1] - (gt_box[3] / 2),
gt_box[0] + (gt_box[2] / 2), gt_box[1] + (gt_box[3] / 2)
]
gt_label = int(gt[0][1])
ious = []
precisions = []
print(count, ":", gt[0][0])
img = Image.open(gt[0][0])
img_resized = letter_box_image(img, cfg.IMAGE_SIZE,
cfg.IMAGE_SIZE, 128)
img_resized = img_resized.astype(np.float32)
t0 = time.time()
detected_boxes = sess.run(boxes,
feed_dict={inputs: [img_resized]})
filtered_boxes = non_max_suppression(
detected_boxes,
confidence_threshold=cfg.CONF_THRESHOLD,
iou_threshold=cfg.IOU_THRESHOLD)
detect_time = time.time() - t0
print("detected boxes in :{:.2f}s ".format(detect_time),
filtered_boxes)
# specific object recognition!
np_img = np.array(img) / 255
target_label = 0 # seesaaの場合 (データセットのクラス番号毎にここを変える．)
if len(filtered_boxes.keys()) != 0: # 何かしら検出された時
is_detected = True
for cls, bboxs in filtered_boxes.items():
if cls == target_label: # ターゲットラベルなら
print("target class detected!")
bounding_boxes = []
bboxs_ = copy.deepcopy(
bboxs
) # convert_to_original_size()がbboxを破壊してしまうため
for box, score in bboxs:
orig_size_box = convert_to_original_size(
box,
np.array((cfg.IMAGE_SIZE, cfg.IMAGE_SIZE)),
np.array(img.size), True)
# print(orig_size_box)
cropped_image = np_img[
int(orig_size_box[1]):int(orig_size_box[3]
),
int(orig_size_box[0]):int(orig_size_box[2]
)]
bounding_boxes.append(cropped_image)
# input_original = cv2.resize(padding(np_img), (vgg16_image_size, vgg16_image_size))
# input_original = np.tile(input_original, (len(bounding_boxes), 1, 1, 1)) # croppedと同じ枚数分画像を重ねる
cropped_images = []
for bbox in bounding_boxes:
cropped_images.append(
cv2.resize(
padding(bbox),
(vgg16_image_size, vgg16_image_size)))
input_cropped = np.asarray(cropped_images)
t0 = time.time()
pred = sess.run(specific_pred,
feed_dict={
cropped_images_placeholder:
input_cropped,
keep_prob: 1.0,
is_training: False
})
recog_time = time.time() - t0
print("Predictions found in {:.2f}s".format(
recog_time))
pred_label = [s_labels[i] for i in pred.tolist()
] # idからクラス名を得る
classes = [
s_labels[i] for i in range(num_classes_s)
]
filtered_boxes = {}
for i, n in enumerate(pred.tolist()):
if n in filtered_boxes.keys():
filtered_boxes[n].extend([bboxs_[i]])
else:
filtered_boxes[n] = [bboxs_[i]]
# calc IoU, mAP
# gt: [(path_str, label), [frame, center_x, center_y, size_x, size_y]
# print(filtered_boxes)
iou = 0.0
for key in filtered_boxes.keys():
for pred_box in filtered_boxes[key]:
p_box = copy.deepcopy(pred_box[0])
orig_scale_p_box = convert_to_original_size(
p_box,
np.array(
(cfg.IMAGE_SIZE, cfg.IMAGE_SIZE)),
np.array(img.size), True)
conf = pred_box[1]
# print(gt_label, key)
if key == gt_label: # 予測したクラスがGTと同じの時
# print(orig_scale_p_box, gt_box)
iou = _iou(
orig_scale_p_box, gt_box
) # :param box1: array of 4 values (top left and bottom right coords): [x0, y0, x1, x2]
precision = calc_precision(
orig_scale_p_box, gt_box)
is_label_correct = True
else:
iou = 0.0
precision = 0.0
is_label_correct = False
# print("IoU:", iou)
ious.append(iou)
print("Precision:", precision)
precisions.append(precision)
else: # ターゲットラベルじゃない時
pass
else: #何も検出されなかった時
is_detected = False
is_label_correct = "None"
pred_label = ["None"]
average_iou = sum(ious) / (len(ious) + 1e-05) # 画像一枚のiou
print("average IoU:", average_iou)
iou_list.append(average_iou)
print("mean average IoU:",
sum(iou_list) / (len(iou_list) + 1e-05))
ap = sum(precisions) / (len(precisions) + 1e-05)
ap_list.append(ap)
print("Average Precision:", ap)
print("mean Average Precision:",
sum(ap_list) / (len(ap_list) + 1e-05))
draw_boxes(filtered_boxes, img, classes,
(cfg.IMAGE_SIZE, cfg.IMAGE_SIZE), True)
# draw GT
draw = ImageDraw.Draw(img)
color = (0, 0, 0)
draw.rectangle(gt_box, outline=color)
draw.text(gt_box[:2], 'GT_' + s_labels[gt_label], fill=color)
img.save(
os.path.join(
cfg.OUTPUT_IMAGE_DIR,
'{0:04d}_'.format(count) + os.path.basename(gt[0][0])))
writer.writerow([
gt[0][0],
os.path.basename(os.path.dirname(gt[0][0])), average_iou,
ap, 'Recall', is_detected, is_label_correct,
s_labels[gt_label], pred_label[0], detect_time, recog_time
])
f.close()
print("proc finished.")
original_width = image.shape[1]
resized_image = utils.resize_image(image, 416)
resized_image = np.expand_dims(resized_image, 0)
detections = sess.run(predictions,
feed_dict={inputs: resized_image})
result = utils.torch_non_max_suppression(
detections=torch.from_numpy(detections),
confidence_threshold=0.25,
num_classes=22,
nms_conf=0.4)
if result is not None:
for box in result:
p1 = (int(box[1]), int(box[2]))
p1 = utils.convert_to_original_size(
p1, original_height, original_width, 416, 416)
p2 = (int(box[3]), int(box[4]))
p2 = utils.convert_to_original_size(
p2, original_height, original_width, 416, 416)
class_name = int(box[-1])
class_name = classes[class_name]
confidence = box[5] * 100.
print('%s: %.2f%%' % (class_name, confidence))
image = utils.draw_bounding_box_with_text(
image,
p1,
p2,
class_name,
box_color=(0, 0, 255),
text_color=(255, 255, 255))
cv2.imshow(os.path.basename(item), image)
x1, y1, x2, y2 = utils.yolo_ratios_to_real_voc(true_box[1], true_box[2], true_box[3], true_box[4],
original_height, original_width)
image = utils.draw_bounding_box_with_text(image=image,
top_left=(int(x1), int(y1)),
right_bottom=(int(x2), int(y2)),
text=true_class_name,
box_color=(0, 255, 255),
text_color=(0, 0, 0))
if result is not None:
for box in result:
# Draw bounding box for visualization
p1 = (int(box[1]), int(box[2]))
p1 = utils.convert_to_original_size(p1, original_height, original_width, model.HEIGHT, model.WIDTH)
p2 = (int(box[3]), int(box[4]))
p2 = utils.convert_to_original_size(p2, original_height, original_width, model.HEIGHT, model.WIDTH)
class_index = int(box[-1])
class_name = classes[class_index]
image = utils.draw_bounding_box_with_text(image=image,
top_left=p1,
right_bottom=p2,
text=class_name,
box_color=(0, 0, 255),
text_color=(255, 255, 255))
cv2.imshow(os.path.basename(image_path), image)
cv2.waitKey(0)
cv2.destroyAllWindows()