def generate(self):
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
self.yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(self.class_names)
num_anchors = len(self.anchors)
# TODO: Assumes dim ordering is channel last
model_output_channels = self.yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes'
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
self.model_image_size = self.yolo_model.layers[0].input_shape[1:3]
self.is_fixed_size = self.model_image_size != (None, None)
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(self.yolo_model.output, self.anchors, len(self.class_names))
self.input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs, self.input_image_shape, score_threshold=self.score, iou_threshold=self.iou)
return boxes, scores, classes
def __init__(self, viden_yolo_weights_path, viden_crnn_weights_path,
classes_path, out_path):
self.act_model = create_crnn_model(train=False)
self.act_model.load_weights(
viden_crnn_weights_path
) # 'viden_trained_models\\viden_crnn_14May2021.hdf5')
class_names = get_classes(classes_path)
#print(class_names)
self.out_path = out_path
#self.anchors = YOLO_ANCHORS
self.yolo_model_body, self.yolo_model = create_model(
YOLO_ANCHORS,
class_names,
load_pretrained=False,
freeze_body=False)
self.yolo_model_body.load_weights(
viden_yolo_weights_path
) #'viden_trained_models\\viden_yolo_14May2021.h5')
self.yolo_outputs = yolo_head(self.yolo_model_body.output,
YOLO_ANCHORS, len(class_names))
self.yolo_input_image_shape = K.placeholder(shape=(2, ))
self.boxes, self.scores, self.classes = yolo_eval(
self.yolo_outputs,
self.yolo_input_image_shape,
max_boxes=1,
score_threshold=.7,
iou_threshold=0.5)
def __init__(self):
# TODO: parametrize init and review variable usage
self.model_path = './model_data/tiny-yolo-voc.h5'
assert self.model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
self.anchors_path = './model_data/tiny-yolo-voc_anchors.txt'
self.classes_path = './model_data/pascal_classes.txt'
self.score_threshold=0.4
self.iou_threshold=0.5
self.sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(self.classes_path) as f:
self.class_names = f.readlines()
self.class_names = [c.strip() for c in self.class_names]
with open(self.anchors_path) as f:
self.anchors = f.readline()
self.anchors = [float(x) for x in self.anchors.split(',')]
self.anchors = np.array(self.anchors).reshape(-1, 2)
self.yolo_model = load_model(self.model_path)
self.is_fixed_size = True
self.model_image_size = self.yolo_model.layers[0].input_shape[1:3]
self.yolo_outputs = yolo_head(self.yolo_model.output, self.anchors, len(self.class_names))
self.input_image_shape = K.placeholder(shape=(2, ))
self.boxes, self.scores, self.classes = yolo_eval(
self.yolo_outputs,
self.input_image_shape,
score_threshold=self.score_threshold,
iou_threshold=self.iou_threshold)
def draw(model_body,
class_names,
anchors,
image_data,
image_set='val',
weights_name='trained_stage_3_best.h5',
out_path="output_images",
save_all=True):
'''
Draw bounding boxes on image data
'''
if image_set == 'train':
image_data = np.array([
np.expand_dims(image, axis=0)
for image in image_data[:int(len(image_data) * .9)]
])
elif image_set == 'val':
image_data = np.array([
np.expand_dims(image, axis=0)
for image in image_data[int(len(image_data) * .9):]
])
elif image_set == 'all':
image_data = np.array(
[np.expand_dims(image, axis=0) for image in image_data])
else:
ValueError("draw argument image_set must be 'train', 'val', or 'all'")
# model.load_weights(weights_name)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=0.07,
iou_threshold=0)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data[i],
input_image_shape: [image_data.shape[2], image_data.shape[3]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes,
class_names, out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path, str(i) + '.png'))
def make_prediction(image_path, input_image_name, yolo_model):
#Obtaining the dimensions of the input image
input_image = Image.open(image_path + input_image_name)
width, height = input_image.size
width = np.array(width, dtype=float)
height = np.array(height, dtype=float)
#Assign the shape of the input image to image_shapr variable
image_shape = (height, width)
#Loading the classes and the anchor boxes that are provided in the model_data folder
class_names = read_classes("model_data/yolo_classes.txt")
anchors = read_anchors("model_data/yolo_anchors.txt")
#Print the summery of the model
yolo_model.summary()
#Convert final layer features to bounding box parameters
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
#Now yolo_eval function selects the best boxes using filtering and non-max suppression techniques.
# If you want to dive in more to see how this works, refer keras_yolo.py file in yad2k/models
boxes, scores, classes = yolo_eval(yolo_outputs, image_shape)
# Initiate a session
sess = K.get_session()
#Preprocess the input image before feeding into the convolutional network
image, image_data = preprocess_image(image_path + input_image_name, model_image_size = (608, 608))
#Run the session
out_scores, out_boxes, out_classes = sess.run(
[scores, boxes, classes],
feed_dict={
yolo_model.input: image_data,
K.learning_phase(): 0
}
)
#Print the results
print('Found {} boxes for {}'.format(len(out_boxes), input_image_name))
#Produce the colors for the bounding boxs
colors = generate_colors(class_names)
#Draw the bounding boxes
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
#Apply the predicted bounding boxes to the image and save it
image.save("predictions/" + input_image_name, quality=90)
if(len(out_classes) == 0):
result = "No box found"
elif (out_classes[0] == 0):
result = "real"
else:
result = "fake"
return input_image_name, out_scores, result
def obj_detect(file_name):
sess = K.get_session()
model_path = 'model_data/tiny-yolo-voc.h5'
anchors_path = 'model_data/tiny-yolo-voc_anchors.txt'
classes_path = 'model_data/pascal_classes.txt'
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model = load_model(model_path)
num_classes = len(class_names)
num_anchors = len(anchors)
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# seems to return true most of the time.
#print(is_fixed_size)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=.3,
iou_threshold=.5)
results = recognize_image(file_name, sess, boxes, scores, classes,
is_fixed_size, model_image_size, yolo_model,
input_image_shape, class_names, colors)
K.clear_session()
return results
def draw(model_body,
class_names,
anchors,
partition,
images_path,
image_set='validation',
weights_name='trained_stage_3_best.h5',
out_path="output_images",
save_all=True):
'''
Draw bounding boxes on image data
'''
# load validation data
hdf5_file_images = h5py.File(images_path, "r")
image_data = hdf5_file_images["images"][partition[image_set], ...]
hdf5_file_images.close()
image_data = np.expand_dims(image_data, axis=1)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=0.5,
iou_threshold=0.5)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data[i],
input_image_shape: [image_data.shape[2], image_data.shape[3]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[i][0].astype(np.uint8) * 255,
out_boxes, out_classes, class_names,
out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path, str(i) + '.png'))
def test_(args):
df_pred = pd.DataFrame(columns=['guid/image', 'N'])
baselist = []
countlist = []
Ignore_class = [0, 9, 11, 14, 15, 16, 17, 21, 22]
class_num = args.class_num
anchors = YOLO_ANCHORS
model_body = create_model(anchors, class_num)
model_body.load_weights(args.weights_path)
yolo_outputs = yolo_head(model_body.output, anchors, class_num)
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=0.4)
sess = K.get_session()
pic_num = 0
for path in os.listdir(args.data_path):
test_path = os.path.join(args.data_path, path)
for image_file in os.listdir(test_path):
try:
image_type = imghdr.what(os.path.join(test_path, image_file))
if not image_type:
continue
except IsADirectoryError:
continue
pic_num += 1
image = PIL.Image.open(os.path.join(test_path, image_file))
resized_image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0)
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
baselist.append(path + '/' + image_file[:4])
count = 0
print(len(out_classes))
for out_class in out_classes:
count += out_class
countlist.append(count)
print(baselist[-1], countlist[-1], pic_num)
df_pred['guid/image'] = baselist
df_pred['N'] = countlist
df_pred.to_csv(args.save_path, index=False)
sess.close()
def detect_video(image_path, output_path, model_image_size, yolo_model,
class_names, yolo_outputs):
from keras import backend as K
from keras.models import load_model
from yad2k.models.keras_yolo import yolo_head, yolo_eval
from PIL import Image
input_image_shape = K.placeholder(shape=(2, ))
sess = K.get_session()
video_in = cv2.VideoCapture(image_path)
#width = video_in.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH) # float
#height = video_in.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT) # float
width, height = int(video_in.get(3)), int(video_in.get(4))
FPS = video_in.get(5)
video_out = cv2.VideoWriter()
video_out.open(output_path, cv2.VideoWriter_fourcc(*'DIVX'), FPS,
(width, height))
width = np.array(width, dtype=float)
height = np.array(height, dtype=float)
image_shape = (height, width)
while video_in.isOpened():
ret, data = video_in.read()
if ret == False:
break
video_array = cv2.cvtColor(data, cv2.COLOR_BGR2RGB)
image = Image.fromarray(video_array, mode='RGB')
resized_image = image.resize(tuple(reversed(model_image_size)),
Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
boxes, scores, classes = yolo_eval(yolo_outputs, image_shape)
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
colors = generate_colors(class_names)
draw_boxes(image, out_scores, out_boxes, out_classes, class_names,
colors)
video_out.write(cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR))
sess.close()
video_in.release()
video_out.release()
print("detect Done")
def model_body_processing(model_body, class_names, anchors):
'''
function to be called once for loading weights and preparing the boxes,scores and classes
according to anchor boxes values,score threshold and iou threshold.
This is evaluated by non_max_suppression function.
'''
global input_image_shape
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=0.5,
iou_threshold=0.5)
return boxes, scores, classes
def predict_objects(self, img):
# t0 = time.time()
#self.object_model.summary()
c = np.fromstring(bytes(img.data), np.uint8)
img = cv2.imdecode(c, cv2.IMREAD_COLOR)
#im_pil
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
im_pil = Image.fromarray(img)
# weight height of the image_sub
width, height = im_pil.size
print(width)
print(height)
width = np.array(width, dtype=float)
height = np.array(height, dtype=float)
image_shape = (height, width)
t0 = time.time()
#class names and anchors
class_names = read_classes("/lanefollowing/ros2_ws/src/lane_following/model/coco_classes.txt")
anchors = read_anchors("/lanefollowing/ros2_ws/src/lane_following/model/yolov2-tiny_anchors.txt")
#yolo head
yolo_outputs = yolo_head(self.object_model.output, anchors, len(class_names))
# yolo eval
boxes, scores, classes = yolo_eval(yolo_outputs, image_shape)
# print('boxes')
# print(boxes)
#yolo preprocess
# pil_img, img_data = preprocess_image_object(im_pil, model_image_size = (416, 416))
pil_img, img_data = preprocess_image_object(im_pil, model_image_size = (608, 608))
#yolo predict
# print('test1')
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],feed_dict={self.object_model.input:img_data,K.learning_phase(): 0})
t1 = time.time()
self.inference_time = t1 - t0
# print("score = " )
# print(out_scores)
# print("\n")
# print('test2')
print('Found {} boxes for '.format(len(out_boxes)))
colors = generate_colors(class_names)
draw_boxes(pil_img, out_scores, out_boxes, out_classes, class_names, colors)
#might need to add resizing here...
image = np.asarray(pil_img)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
cv2.putText(image, "Prediction time: %d ms" % (self.inference_time * 1000), (30, 220), cv2.FONT_HERSHEY_PLAIN, 3, (0, 255, 0), 2)
cv2.putText(image, "Frame speed: %d fps" % (self.fps), (30, 270), cv2.FONT_HERSHEY_PLAIN, 3, (0, 255, 0), 2)
image = cv2.resize(image, (round(image.shape[1] / 2), round(image.shape[0] / 2)), interpolation=cv2.INTER_AREA)
cv2.imshow('YOLO Detector', image)
cv2.waitKey(1)
def draw(model_body, class_names, anchors, image_data, image_set='val',
weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True):
'''
Draw bounding boxes on image data
'''
if image_set == 'train':
image_data = np.array([np.expand_dims(image, axis=0)
for image in image_data[:int(len(image_data)*.9)]])
elif image_set == 'val':
image_data = np.array([np.expand_dims(image, axis=0)
for image in image_data[int(len(image_data)*.9):]])
elif image_set == 'all':
image_data = np.array([np.expand_dims(image, axis=0)
for image in image_data])
else:
ValueError("draw argument image_set must be 'train', 'val', or 'all'")
# model.load_weights(weights_name)
print(image_data.shape)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data[i],
input_image_shape: [image_data.shape[2], image_data.shape[3]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes,
class_names, out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path,str(i)+'.png'))
def prepare_yolo(model_path, class_names):
yolo_model = load_model(model_path)
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs,
input_image_shape,
score_threshold=args['score_threshold'],
iou_threshold=args['iou_threshold'],
max_boxes=50
)
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
sess = K.get_session()
return {
'yolo_model': yolo_model,
'input_image_shape': input_image_shape,
'boxes': boxes,
'scores': scores,
'classes': classes,
'sess': sess,
'model_image_size': model_image_size,
'class_names': class_names,
'colors': colors
}
def pred(self, image_s, display_shape=(640, 480)):
# Make predictions for images in (num_images, height, width, channel) format
assert len(
image_s.shape
) == 4 # image must have 4 dims ready to be sent into the graph
# TODO allow multiple images at once.
assert image_s.shape[0] == 1
features = self.yolo_model.predict(image_s)
proc = yolo_head_np(features, self.anchors, len(self.class_names))
out_boxes, out_scores, out_classes = yolo_eval(
proc,
display_shape,
max_boxes=self.max_boxes,
score_threshold=self.score_threshold,
iou_threshold=self.iou_threshold)
return out_boxes, out_scores, out_classes
def draw(model_body, class_names, anchors,
image_data, weights_name='trained_stage_3_best.h5',
out_path="output_images", save_all=True):
'''
Draw bounding boxes on image data
'''
# model.load_weights(weights_name)
print(image_data.shape)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0.0)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data[i],
input_image_shape: [image_data.shape[2], image_data.shape[3]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes,
class_names, out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path,str(i)+'.png'))
# To display (pauses the program):
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def predict(image_file):
sess = K.get_session()
"""
Runs the graph stored in "sess" to predict boxes for "image_file". Prints and plots the preditions.
Arguments:
sess -- your tensorflow/Keras session containing the YOLO graph
image_file -- name of an image stored in the "images" folder.
Returns:
out_boxes -- tensor of shape (None, 4), coordinates of the predicted boxes
out_scores -- tensor of shape (None, ), scores of the predicted boxes
out_classes -- tensor of shape (None, ), class index of the predicted boxes
Note: "None" actually represents the number of predicted boxes, it varies between 0 and max_boxes.
"""
image, image_data = preprocess_image(image_file,
model_image_size=(608, 608))
scores, boxes, classes = yolo_eval(
yolo_outputs,
np.reshape(image.size[::-1], [2]).astype(np.float32))
result = sess.run([scores, boxes, classes],
feed_dict={
yolo_model.input: image_data,
K.learning_phase(): 0
})
print(result)
out_boxes, out_scores, out_classes = result
# Print predictions info
print('Found %s boxes for %s' % (len(out_boxes), image_file))
# Generate colors for drawing bounding boxes.
colors = generate_colors(class_names)
# Draw bounding boxes on the image file
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
imshow(image)
plt.rcParams["figure.figsize"] = (20, 20)
plt.show()
return out_scores, out_boxes, out_classes
def start_model(self):
'''
This needs to run to be able to use
get_single_image
'''
#Make the model
self.yolo_model = self.get_model()
self.summary()
self.sess = K.get_session()
time.sleep(2)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / self.CLASS, 1., 1.) for x in range(self.CLASS)]
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
self.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(
self.colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
self.yolo_outputs = yolo_head(self.yolo_model.output, self.ANCHORS,
self.CLASS)
self.input_image_shape = K.placeholder(shape=(2, ))
self.boxes, self.scores, self.classes = yolo_eval(
self.yolo_outputs,
self.input_image_shape,
score_threshold=self.SCORE_THRESHOLD,
iou_threshold=self.IOU_THRESHOLD)
self.label_size = 4
self.font = cv2.FONT_HERSHEY_SIMPLEX
def _main(args):
sess = K.get_session()
print("loading class file")
global class_names
with open(CLASSES_PATH) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
print("loading anchors")
with open(ANCHORS_PATH) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
print("loading yolo model")
global yolo_model
yolo_model = load_model(MODEL_PATH)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
# TODO: Assumes dim ordering is channel last
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == len(anchors) * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Exit.'
print('{} model, anchors, and classes loaded.'.format(MODEL_PATH))
# Check if model is fully convolutional, assuming channel last order.
global model_image_size
model_image_size = yolo_model.layers[0].input_shape[1:3]
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / num_classes, 1., 1.)
for x in range(num_classes)]
global colors
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, num_classes)
global input_image_shape
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
#########################################################################
print("capture pi camera...")
# set camera resolution
with picamera.PiCamera() as camera:
camera.resolution = (320, 320)
# use opencv to capture frames
cam = cv2.VideoCapture(0)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, 320)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, 320)
cam.set(cv2.CAP_PROP_FPS, 30)
rval, frame = cam.read()
if rval == False:
print("Error: Camera is not Found. Exit")
sys.exit(1)
cv2.imshow("preview", np.array(frame))
while True:
rval, frame = cam.read()
before = time.time()
detect_img = detect_image(frame, sess, boxes, scores, classes)
after = time.time()
cv2.imshow("preview", np.array(detect_img))
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print("FPS: {}".format(1 / (after-before)))
sess.close()
cam.release()
cv2.destroyAllWindows()
def _main(args):
model_path = os.path.expanduser(args.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args.anchors_path)
classes_path = os.path.expanduser(args.classes_path)
test_path = os.path.expanduser(args.test_path)
output_path = os.path.expanduser(args.output_path)
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
# TODO: Assumes dim ordering is channel last
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
for image_file in os.listdir(test_path):
try:
image_type = imghdr.what(os.path.join(test_path, image_file))
if not image_type:
continue
except IsADirectoryError:
continue
image = Image.open(os.path.join(test_path, image_file))
if is_fixed_size: # TODO: When resizing we can use minibatch input.
resized_image = image.resize(
tuple(reversed(model_image_size)), Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
font = ImageFont.truetype(
font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle(
[left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle(
[tuple(text_origin), tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.save(os.path.join(output_path, image_file), quality=90)
sess.close()
def _main(args):
model_path = os.path.expanduser(args['model_path'])
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args['anchors_path'])
classes_path = os.path.expanduser(args['classes_path'])
test_path = os.path.expanduser(args['test_path'])
output_path = os.path.expanduser(args['output_path'])
score_threshold = args['score_threshold']
iou_threshold = args['iou_threshold']
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session()
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=score_threshold,
iou_threshold=iou_threshold)
for image_file in os.listdir(test_path):
try:
image_type = imghdr.what(os.path.join(test_path, image_file))
if not image_type:
continue
except IsADirectoryError:
continue
image = Image.open(os.path.join(test_path, image_file))
if is_fixed_size:
resized_image = image.resize(tuple(reversed(model_image_size)),
Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.save(os.path.join(output_path, image_file), quality=90)
sess.close()
def __predict_output__():
plt.interactive(False)
cfg = Configuration()
GPU = True
if GPU != True:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
# Input Path
root_dir = os.path.dirname(os.path.abspath(__file__))
image_path = cfg.image_path
json_path = os.path.join(root_dir, cfg.input_filename)
testingset = os.path.join(root_dir, 'testingset')
Preprocessor.__generate_kijiji_set__(root_dir, image_path, json_path,
testingset, 'model')
# ------------------generator to compile training data of kijiji dataset----------------------------------------
image_path = os.path.join(root_dir, 'testingset')
data_path = glob(image_path + "/*")
# Image Segmentation Parameters
model_path = os.path.expanduser(cfg.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(cfg.anchors_path)
classes_path = os.path.expanduser(cfg.classes_path)
test_path = os.path.expanduser(cfg.test_path)
output_path = os.path.expanduser(cfg.segmented_output_path)
json_path = os.path.expanduser(cfg.json_output)
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session()
class_names = Preprocessor.__return_class_names__(classes_path)
anchors = Preprocessor.__return_anchors__(anchors_path)
yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
info = 'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and --classes_path flags.'
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), info
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate Colors for drawing bounding boxes
hsv_tuples, colors = Preprocessor.__generate_colors_for_bounding_boxes__(
class_names)
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=cfg.score_threshold,
iou_threshold=cfg.iou_threshold)
# Load Images from the root folder
input_images_model_1, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__(
data_path,
cfg.compressed_image_height,
cfg.compressed_image_width,
cfg.compressed_channel,
cfg.number_of_categories,
cfg.number_of_images_per_category,
root_dir,
is_fixed_size,
model_image_size,
sess,
yolo_model,
input_image_shape,
boxes,
scores,
classes,
cfg.font_path,
class_names,
colors,
output_path,
json_path,
test_path,
True, # Segmentation Flag
False, # Edge-detection Flag
True, # Extract object Flag
False) # Gray Scale Flag
input_images_model_2, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__(
data_path, cfg.compressed_image_height, cfg.compressed_image_width,
cfg.compressed_channel, cfg.number_of_categories,
cfg.number_of_images_per_category, root_dir, is_fixed_size,
model_image_size, sess, yolo_model, input_image_shape, boxes,
scores, classes, cfg.font_path, class_names, colors, output_path,
json_path, test_path, False, True, False, False)
input_images_model_3, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__(
data_path, cfg.image_height, cfg.image_width, cfg.channel,
cfg.number_of_categories, cfg.number_of_images_per_category,
root_dir, is_fixed_size, model_image_size, sess, yolo_model,
input_image_shape, boxes, scores, classes, cfg.font_path,
class_names, colors, output_path, json_path, test_path, False,
False, False, False)
input_shape = [
cfg.compressed_image_height, cfg.compressed_image_width,
cfg.compressed_channel
]
input_shape_3 = [cfg.image_height, cfg.image_width, cfg.channel]
# load (pre-trained) weights for model_1
print('-' * 30)
print('Loading model weights...\n')
weight_folder = cfg.model_1_save # the path where the model weights are stored
weight_file = 'model_1.h5'
model_1 = Preprocessor.__load_model_weights__(weight_folder,
weight_file, input_shape,
input_shape_3, "Model_1")
# load (pre-trained) weights for model_2
print('-' * 30)
print('Loading model weights...\n')
weight_folder = cfg.model_2_save # the path where the model weights are stored
weight_file = 'model_2.h5'
model_2 = Preprocessor.__load_model_weights__(weight_folder,
weight_file, input_shape,
input_shape_3, "Model_2")
# load (pre-trained) weights for model_2
print('-' * 30)
print('Loading model weights...\n')
weight_folder = cfg.model_3_save # the path where the model weights are stored
weight_file = 'model_3.h5'
model_3 = Preprocessor.__load_model_weights__(weight_folder,
weight_file, input_shape,
input_shape_3, "Model_3")
print(root_dir)
print(os.path.join(root_dir, cfg.output_model_1))
output_path_model_1 = os.path.join(root_dir + cfg.output_model_1)
output_path_model_2 = os.path.join(root_dir + cfg.output_model_2)
output_path_model_3 = os.path.join(root_dir + cfg.output_model_3)
Preprocessor.__create_output_directories__(output_path_model_1)
Preprocessor.__create_output_directories__(output_path_model_2)
Preprocessor.__create_output_directories__(output_path_model_3)
features_from_model_1 = Preprocessor.__get_score_model__(
model_1, input_images_model_1, output_path_model_1)
features_from_model_2 = Preprocessor.__get_score_model__(
model_2, input_images_model_2, output_path_model_2)
features_from_model_3 = Preprocessor.__get_score_model__(
model_3, input_images_model_3, output_path_model_3)
features_from_model_1 = Preprocessor.__flatten_img_data__(
features_from_model_1)
features_from_model_2 = Preprocessor.__flatten_img_data__(
features_from_model_2)
features_from_model_3 = Preprocessor.__flatten_img_data__(
features_from_model_3)
fused_features = np.concatenate([
features_from_model_1, features_from_model_2, features_from_model_3
],
axis=1)
fused_features = [
Preprocessor.__binarize__(features) for features in fused_features
]
counter_for_predictions = 0
sub_average_precision_make, sub_average_precision_color = [], []
sub_average_precision_body, sub_average_precision_model = [], []
cum_average_precision_make, cum_average_precision_color = [], []
cum_average_precision_body, cum_average_precision_model = [], []
precision_at_3_5_10_all = ''.join(cfg.precision_counter).split(',')
while counter_for_predictions <= 2:
test_image_idx = int(len(input_images_model_1) * random())
if test_image_idx < len(data_path_with_image_name):
idx_closest = Preprocessor.__get_closest_images__(
test_image_idx, fused_features, cfg.number_of_predictions)
test_image = Preprocessor.__get_concatenated_images__(
data_path_with_image_name, [test_image_idx],
cfg.compressed_image_width)
results_image = Preprocessor.__get_concatenated_images__(
data_path_with_image_name, idx_closest,
cfg.compressed_image_width)
source_category = str(
data_path_with_image_name[test_image_idx]).split('/')
similar_image = []
similar_idx_closest = []
for counter_for_recommendations in range(0, len(idx_closest)):
category = str(data_path_with_image_name[
idx_closest[counter_for_recommendations]]).split('/')
if str(source_category[-2]).strip() == str(
category[-2].strip()):
similar_image.append(data_path_with_image_name[
idx_closest[counter_for_recommendations]])
similar_idx_closest.append(
idx_closest[counter_for_recommendations])
print("Test Image ID:", test_image_idx)
print("\n")
print("Closest Images ID:", idx_closest)
print("\n")
print("Similar Images ID", similar_idx_closest)
print("\n")
precision_per_make, precision_per_color = [], []
precision_per_body_wise, precision_per_model_wise = [], []
results_image_recommendations = []
for i in range(0, len(precision_at_3_5_10_all)):
results_image_recommendations = Preprocessor.__get_concatenated_images__(
data_path_with_image_name, similar_idx_closest,
cfg.compressed_image_width)
list_of_similar_image_names = Preprocessor.__return_image_names__(
data_path_with_image_name, similar_idx_closest)
name_of_test_image = Preprocessor.__return_image_names__(
data_path_with_image_name, [test_image_idx])
dict_of_attributes_of_similar_images = Preprocessor.__get_attributes_list__(
list_of_similar_image_names,
os.path.join(root_dir, cfg.input_filename))
dict_of_attributes_of_test_image = Preprocessor.__get_attributes_list__(
name_of_test_image,
os.path.join(root_dir, cfg.input_filename))
similar_make_wise = Preprocessor.__get_similar__(
dict_of_attributes_of_test_image,
dict_of_attributes_of_similar_images[:int(
precision_at_3_5_10_all[i])], 'make')
similar_color_wise = Preprocessor.__get_similar__(
dict_of_attributes_of_test_image,
dict_of_attributes_of_similar_images[:int(
precision_at_3_5_10_all[i])], 'color')
similar_body_wise = Preprocessor.__get_similar__(
dict_of_attributes_of_test_image,
dict_of_attributes_of_similar_images[:int(
precision_at_3_5_10_all[i])], 'body')
similar_model_wise = Preprocessor.__get_similar__(
dict_of_attributes_of_test_image,
dict_of_attributes_of_similar_images[:int(
precision_at_3_5_10_all[i])], 'model')
precision_per_make.append(
float(
float(len(similar_make_wise)) /
int(precision_at_3_5_10_all[i])))
precision_per_color.append(
float(
float(len(similar_color_wise)) /
int(precision_at_3_5_10_all[i])))
precision_per_body_wise.append(
float(
float(len(similar_body_wise)) /
int(precision_at_3_5_10_all[i])))
precision_per_model_wise.append(
float(
float(len(similar_model_wise)) /
int(precision_at_3_5_10_all[i])))
sub_average_precision_make.append(precision_per_make)
sub_average_precision_color.append(precision_per_color)
sub_average_precision_body.append(precision_per_body_wise)
sub_average_precision_model.append(precision_per_model_wise)
imsave('test.png', test_image)
imsave('recommendations.png', results_image_recommendations)
imsave('total_results.png', results_image)
counter_for_predictions += 1
time.sleep(1)
else:
print("Index is out of bound")
cum_average_precision_make.append(
map(Preprocessor.__mean__, zip(*sub_average_precision_make)))
cum_average_precision_color.append(
map(Preprocessor.__mean__, zip(*sub_average_precision_color)))
cum_average_precision_body.append(
map(Preprocessor.__mean__, zip(*sub_average_precision_body)))
cum_average_precision_model.append(
map(Preprocessor.__mean__, zip(*sub_average_precision_model)))
print("\n \n \n")
print(
"-----------------------------------------------------------------------------------"
)
print("Average Precision Make-Wise", precision_at_3_5_10_all,
map(Preprocessor.__mean__, zip(*cum_average_precision_make)))
print("Average Precision Color-Wise", precision_at_3_5_10_all,
map(Preprocessor.__mean__, zip(*cum_average_precision_color)))
print("Average Precision Body-Wise", precision_at_3_5_10_all,
map(Preprocessor.__mean__, zip(*cum_average_precision_body)))
print("Average Precision Model-Wise", precision_at_3_5_10_all,
map(Preprocessor.__mean__, zip(*cum_average_precision_model)))
writer = csv.writer(open(os.path.join(root_dir, 'results.csv'), 'w'))
writer.writerow([
"Make-Wise: Precision at 3", "Make-Wise: Precision at 5",
"Make-Wise: Precision at 10"
])
for row in zip(*cum_average_precision_make):
writer.writerow(row)
writer.writerow('\n')
writer.writerow([
"Color-Wise: Precision at 3", "Color-Wise: Precision at 5",
"Color-Wise: Precision at 10"
])
for row in zip(*cum_average_precision_color):
writer.writerow(row)
writer.writerow('\n')
writer.writerow([
"Body-Wise: Precision at 3", "Body-Wise: Precision at 5",
"Body-Wise: Precision at 10"
])
for row in zip(*cum_average_precision_body):
writer.writerow(row)
writer.writerow('\n')
writer.writerow([
"Model-Wise: Precision at 3", "Model-Wise: Precision at 5",
"Model-Wise: Precision at 10"
])
for row in zip(*cum_average_precision_model):
writer.writerow(row)
writer.writerow('\n')
def _main(args):
def predict(sess, image):
# Preprocess your image
image, image_data = preprocess_image(image,
model_image_size=(416, 416))
# Run the session with the correct tensors and choose the correct placeholders in the feed_dict.
# You'll need to use feed_dict={yolo_model.input: ... , K.learning_phase(): 0})
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],
feed_dict={
yolo_model.input:
image_data,
K.learning_phase(): 0
})
# Print predictions info
print('Found {} boxes'.format(len(out_boxes)))
# Generate colors for drawing bounding boxes.
colors = generate_colors(class_names)
# Draw bounding boxes on the image file
out_image = draw_boxes(image, out_scores, out_boxes, out_classes,
class_names, colors)
return out_image, out_scores, out_boxes, out_classes
video_path = os.path.join(INPUT_PATH, args.input)
input = cv2.VideoCapture(video_path)
if (input.isOpened() == False):
print("Error opening video file.")
return
height = input.get(cv2.CAP_PROP_FRAME_HEIGHT)
width = input.get(cv2.CAP_PROP_FRAME_WIDTH)
fps = input.get(cv2.CAP_PROP_FPS)
if not os.path.exists(OUTPUT_PATH):
os.mkdir(OUTPUT_PATH)
output_file_path = os.path.join(OUTPUT_PATH, args.input)
output = cv2.VideoWriter(output_file_path, cv2.VideoWriter_fourcc(*'MP4V'),
fps, (int(width), int(height)))
image_shape = (height, width)
sess = K.get_session()
class_names = read_classes(CLASS_LIST_PATH)
anchors = read_anchors(YOLO_ANCHORS_PATH)
yolo_model, model = create_model(anchors, class_names)
yolo_model.load_weights(args.weights)
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
boxes, scores, classes = yolo_eval(yolo_outputs,
image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
n_frame = 0
while input.isOpened():
ret, original_frame = input.read()
if ret:
n_frame = n_frame + 1
print("processing frame {} ...".format(n_frame))
process_frame = cv2.cvtColor(original_frame, cv2.COLOR_BGR2RGB)
process_frame = Image.fromarray(process_frame)
process_frame, out_scores, out_boxes, out_classes = predict(
sess, process_frame)
process_frame = cv2.cvtColor(process_frame, cv2.COLOR_BGR2RGB)
output.write(process_frame)
else:
break
print("Finished. Output file: ", output_file_path)
input.release()
output.release()
cv2.destroyAllWindows()
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs,
input_image_shape,
score_threshold=score_threshold,
iou_threshold=iou_threshold)
def draw_pred(image):
if type(image) == np.ndarray:
image = Image.fromarray(image)
elif not issubclass(type(image),Image.Image):
raise Exception('image must be of type PIL.Image.Image')
if is_fixed_size: # TODO: When resizing we can use minibatch input.
resized_image = image.resize(
tuple(reversed(model_image_size)), Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# Initiate a session
sess = K.get_session()
while True:
# Capture frame-by-frame
try:
ret, frame = cap.read()
image = frame
print("Type of image is: ",type(frame))
width, height = image.size
image_shape = (height, width)
boxes, scores, classes = yolo_eval(yolo_outputs, image_shape)
width = np.array(width, dtype=float)
height = np.array(height, dtype=float)
image, image_data = preprocess_image(image, model_image_size = (608, 608))
#resized_image = image.resize(tuple(reversed((608,608)), Image.BICUBIC)
#image_data = np.array(resized_image, dtype='float32')
#image_data /= 255.
#image_data = np.expand_dims(image_data, 0)
#Run the session
#out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],feed_dict={yolo_model.input:image_data,K.learning_phase(): 0})
#Print the results
print('Found {} boxes for {}'.format(len(out_boxes), input_image_name))
#Produce the colors for the bounding boxs
colors = generate_colors(class_names)
def _main(args):
voc_path = os.path.expanduser(args.data_path)
classes_path = os.path.expanduser(args.classes_path)
anchors_path = os.path.expanduser(args.anchors_path)
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
if os.path.isfile(anchors_path):
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
else:
anchors = YOLO_ANCHORS
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['train/images'][28]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
boxes = voc['train/boxes'][28]
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size
boxes_wh = boxes_wh / orig_size
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
detectors_mask, matching_true_boxes = preprocess_true_boxes(boxes, anchors,
[416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
print('Boxes:')
print(boxes)
print('Box corners:')
print(boxes_extents)
print('Active detectors:')
print(np.where(detectors_mask == 1)[:-1])
print('Matching boxes for active detectors:')
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(
yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={'anchors': anchors,
'num_classes': len(class_names)})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input,
matching_boxes_input], model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=0)
detectors_mask = np.expand_dims(detectors_mask, axis=0)
matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 1000
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
model.fit([image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=1,
epochs=num_steps)
model.save_weights('overfit_weights.h5')
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs, input_image_shape, score_threshold=.3, iou_threshold=.9)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def _main():
voc_path = os.path.expanduser(
'/Users/ss/Data/VOCdevkit/pascal_voc_07_07.hdf5')
classes_path = os.path.expanduser('model_data/pascal_classes.txt')
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['train/images'][28]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
boxes = voc['train/boxes'][28]
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size
boxes_wh = boxes_wh / orig_size
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
anchors = COCO_ANCHORS
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
detectors_mask, matching_true_boxes = preprocess_true_boxes(
boxes, anchors, [416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
print(boxes)
print(boxes_extents)
print(np.where(detectors_mask == 1)[:-1])
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input, matching_boxes_input],
model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=0)
detectors_mask = np.expand_dims(detectors_mask, axis=0)
matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 1000
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
model.fit([image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=1,
epochs=num_steps)
model.save_weights('overfit_weights.h5')
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=.3,
iou_threshold=.9)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def _main(args):
model_path = os.path.expanduser(args.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args.anchors_path)
classes_path = os.path.expanduser(args.classes_path)
test_path = os.path.expanduser(args.test_path)
output_path = os.path.expanduser(args.output_path)
url = args.ws_addr
video = args.video
if video:
pygame.init()
screen = pygame.display.set_mode((720, 720))
x = (screen.get_width() - resolution[0]) / 2
y = (screen.get_height() - resolution[1]) / 2
if url:
ws = websocket.create_connection(url)
ws.send(json.dumps({'device': 'yolo'}))
else:
ws = None
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
# TODO: Assumes dim ordering is channel last
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags.'
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
with picamera.PiCamera() as camera:
output = picamera.array.PiRGBArray(camera)
exitFlag = True
counter = 0
while (exitFlag):
if video:
for event in pygame.event.get():
if (event.type is pygame.MOUSEBUTTONDOWN
or event.type is pygame.QUIT):
exitFlag = False
camera.resolution = resolution
camera.capture(output, 'rgb')
image_pixels = output.array
if is_fixed_size: # TODO: When resizing we can use minibatch input.
image = Image.fromarray(image_pixels)
resized_image = image.resize(tuple(reversed(model_image_size)),
Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), counter))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
output.truncate(0)
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1],
np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0],
np.floor(right + 0.5).astype('int32'))
print("{} identified. Uploading profile to the NSA.".format(
label))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
# image.save(os.path.join(output_path, str(counter) + '.png'), quality=90)
counter += 1
buff = BytesIO()
image.save(buff, format="JPEG")
ws_image_bytes = buff.getvalue()
del draw
if ws:
send_data = {
'type': 'yolo',
'data':
base64.b64encode(ws_image_bytes).decode('utf-8')
}
ws.send(json.dumps(send_data))
image_bytes = image.tobytes()[0:resolution[0] * resolution[1] * 3]
py_image = pygame.image.fromstring(image_bytes, camera.resolution,
'RGB')
if video:
screen.fill(0)
if py_image:
screen.blit(py_image, (x, y))
pygame.display.update()
sess.close()
def yoloThread():
global frames,times
model_path = scriptFolder+"tiny.h5"
anchors_path = scriptFolder+"anchors.txt"
sess = K.get_session()
print("[PiCam] Loading anchors file...")
anchors = [1.08,1.19,3.42,4.41,6.63,11.38,9.42,5.11,16.62,10.52]
anchors = np.array(anchors).reshape(-1, 2)
print("[PiCam] Loading yolo model ({})...".format(scriptFolder+"tiny.h5"))
yolo_model = load_model(model_path)
num_anchors = len(anchors)
print('[PiCam] YOLO model loaded !'.format(model_path))
model_image_size = yolo_model.layers[0].input_shape[1:3]
yolo_outputs = yolo_head(yolo_model.output, anchors, 20)
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,input_image_shape,score_threshold=0.3,iou_threshold=0.4)
num = 0 #Photo name
old_time = 0.0 #Latest time
print("[PiCam] YOLO Thread started!")
### Loop:
while True:
if len(frames) != 0:
try:
cv2.waitKey(17)
mat = frames[0]
mat = cv2.resize(mat,(model_image_size[0],model_image_size[1]))
in_mat = np.array(mat,dtype='float32')
in_mat /= 255.
in_mat = np.expand_dims(in_mat, 0)
if (times[0] - old_time) > time_chunck:
out_boxes, out_scores, out_classes = sess.run([boxes, scores, classes],feed_dict={yolo_model.input: in_mat,input_image_shape: [mat.shape[1], mat.shape[0]],K.learning_phase(): 0})
if len(out_boxes) > 0:
writ = False
xs,ys = [],[]
for i, c in reversed(list(enumerate(out_classes))):
if c == 14: #14 is the label for persons
writ = True
box = out_boxes[i]
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(mat.shape[1], np.floor(bottom + 0.5).astype('int32'))
right = min(mat.shape[0], np.floor(right + 0.5).astype('int32'))
# cv2.rectangle(mat,(left+i,top+i),(right-i,bottom-i),(10,230,10),1)
xs.append(left+i)
xs.append(right-i)
ys.append(top+i)
ys.append(bottom-i)
if writ:
img_name = scriptFolder+"imgs/{}.png".format(num)
cv2.imwrite(img_name,mat[min(ys):max(ys),min(xs):max(xs)]) #we only save the part of the images where we have found persons
out_s = "[{}] Detected person (taken {}s)!\n".format(time.strftime("%H:%M:%S"),round(time.time()-times[0]))
print(out_s)
flog.write(out_s)
flog.flush()
#system("telegram-cli -W -e \"send_photo {} {} \"".format(telegram_user,img_name))
try:
system("telegram-cli -W -e \"send_photo {} {} \"".format(telegram_user,img_name))
except Exception as exc:
print("[PiCam] Some error occured in YOLO Thread ({}) :".format(time.strftime("%H:%M:%S")),exc)
num += 1
old_time = times[0]
except Exception as ex:
print("[PiCam] Some error occured in YOLO Thread ({}) :".format(time.strftime("%H:%M:%S")),ex)
del times[0]
del frames[0]
cv2.waitKey(50)
def _main(args):
model_path = os.path.expanduser(args.model_path)
assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.'
anchors_path = os.path.expanduser(args.anchors_path)
classes_path = os.path.expanduser(args.classes_path)
debug = os.path.expanduser(args.debug)
debugging = False
if (debug != 'off'):
debugging = True
sess = K.get_session()
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model = load_model(model_path)
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes. ' \
'Specify matching anchors and classes with --anchors_path and ' \
'--classes_path flags. Should be in model_data folder.'
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
if debugging:
print(
"Note that debugging mode will make processing time slower due to camera preview time."
)
cachedposition = (-1000, -1000)
cachedprediction = ['nothing', 'nothing']
with picamera.PiCamera(sensor_mode=6) as camera:
camera.resolution = (800, 640)
#camera.video_stabilization = True
#camera.shutter_speed = 800
print("\nPreloading success! Now running...")
while True:
stream = io.BytesIO()
if debugging:
#camera.start_preview()
#time.sleep(2)
#camera.stop_preview()
start_time = time.time()
camera.capture(stream, 'jpeg')
if debugging:
print("Capture time : %.3f" % (time.time() - start_time))
#stream.seek(0)
if debugging: opening = time.time()
image = Image.open(stream)
if debugging:
print("Open Image time : %.3f" % (time.time() - opening))
if is_fixed_size:
resized_image = image.resize(tuple(reversed(model_image_size)),
Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
iZmage.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
if debugging: time1 = time.time()
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [800, 640],
#input_image_shape: [640, 480],
K.learning_phase(): 0
})
if debugging: print("Sess Run time : %.3f" % (time.time() - time1))
#font = ImageFont.truetype(
# font='font/FiraMono-Medium.otf',
# size=2 * np.floor(2e-2 * image.size[1] + 0.5).astype('int32'))
#thickness = 2 * (image.size[0] + image.size[1]) // 300
message = ''
message_head = "\n[INFO] Predicted scene: User is"
label_result = []
match_pair = []
dic = {
'person': 'next to another person!',
'motorbike': 'doing workout with a Gym Bicycle!',
'bicycle': 'doing workout with a Gym Bicycle!',
'sports ball': 'doing workout with a Yoga Ball!',
'apple': 'doing workout with a Yoga Ball!',
'mouse': 'doing workout with a Yoga Ball!',
'chair': 'taking a rest!',
'sofa': 'taking a rest!'
}
weights = {
'apple': 0.1,
'person': 0.7,
'bicycle': 0.1,
'sports ball': 0.1,
'motorbike': 0.1,
'mouse': 0.1,
'chair': 0.1,
'sofa': 0.1
}
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
label_result.append(predicted_class)
#draw = ImageDraw.Draw(image)
#label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1],
np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0],
np.floor(right + 0.5).astype('int32'))
# w and h of the box
w = bottom - top
h = right - left
x = top
y = right
# print(label, (left, top), (right, bottom))
# add to match_pair for comparison
if predicted_class in [
'person', 'bicycle', 'sports ball', 'apple',
'motorbike', 'mouse', 'chair', 'sofa'
]:
match_pair.append([c, [x, y, w, h]])
# now the matched_pair contains only person and predefined items
#if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]])
#else: text_origin = np.array([left, top + 1])
#if predicted_class in ['person']:
# for i in range(thickness):
# draw.rectangle(
# [left + i, top + i, right - i, bottom - i],
# outline=colors[c])
message_origin = np.array([0, 10])
person_box, ball_box, chair_box, bicycle_box = [], [], [], []
match_pair = sorted(match_pair)
founduser = False
for i in range(1, len(match_pair)):
try:
test_pair = match_pair[i][0]
test_class = class_names[test_pair]
test_dic = dic[test_class]
if test_pair == match_pair[0][0]:
continue
if intersect_area(match_pair[0][1], match_pair[i][1]) > 0:
if test_class in ['bicycle', 'motorbike']:
message = "Predicted scenery: Person is " + test_dic
print(message_head, test_dic, "\n[INFO] User Pos:",
body_pos(match_pair[0][1]))
#draw.text(message_origin, message, fill=(255, 0, 0), font=font)
cachedprediction[0] = cachedprediction[1]
cachedprediction[1] = 'bicycle'
break
if test_class in ['sports ball', 'apple', 'mouse']:
message = "Predicted scenery: Person is " + test_dic
print(message_head, test_dic, "\n[INFO] User Pos:",
body_pos(match_pair[0][1]))
#draw.text(message_origin, message, fill=(255, 0, 0), font=font)
cachedprediction[0] = cachedprediction[1]
cachedprediction[1] = 'yoga'
break
if test_class in ['chair', 'sofa']:
message = "Predicted scenery: Person is " + test_dic
print(message_head, test_dic, "\n[INFO] User Pos:",
body_pos(match_pair[0][1]))
#draw.text(message_origin, message, fill=(255, 0, 0), font=font)
cachedprediction[0] = cachedprediction[1]
cachedprediction[1] = 'chair'
break
else:
founduser = True
if cachedprediction[1] != 'nothing':
printpred(cachedprediction[1])
elif cachedprediction[0] != 'nothing':
printpred(cachedprediction[0])
elif (distance(body_pos(match_pair[0][1]),
cachedposition) < 4):
print(
"\n[INF0] Predicted scene: User is taking a rest!"
)
cachedprediction[0] = cachedprediction[1]
cachedprediction[1] = 'chair'
else:
cachedprediction[0] = cachedprediction[1]
cachedprediction[1] = 'nothing'
print("[INFO] User Pos:", body_pos(match_pair[0][1]))
cachedposition = body_pos(match_pair[0][1])
break
except KeyError:
continue
if not founduser:
cachedposition = (-1000, -1000)
label_result = [] # cache
if debugging:
print("Loop time : %.3f\n" %
(time.time() -
start_time)) # -2 for the 2 second camera preview time
sess.close()
def _main(args):
test_path = os.path.expanduser(args.test_path)
output_path = os.path.expanduser(args.output_path)
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.makedirs(output_path)
if args.gpu:
# https://github.com/keras-team/keras/issues/3685
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
yolo_model, class_names, anchors = load_model(args.model_path, args.classes_path, args.anchors_path)
print('{} model, anchors, and classes loaded.'.format(args.model_path))
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs,
input_image_shape,
max_boxes=20,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
for image_file in sorted(os.listdir(test_path)):
try:
image_type = imghdr.what(os.path.join(test_path, image_file))
if not image_type:
continue
except (IsADirectoryError, PermissionError):
continue
image = Image.open(os.path.join(test_path, image_file))
if image.mode != 'RGB':
image = image.convert(mode='RGB')
if is_fixed_size: # TODO: When resizing we can use minibatch input.
resized_image = image.resize(
tuple(reversed(model_image_size)), Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
font = ImageFont.truetype(
font=os.path.join(os.path.dirname(__file__), 'font', 'FiraMono-Medium.otf'),
size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle(
[left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle(
[tuple(text_origin), tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.save(os.path.join(output_path, image_file), quality=90)
if args.text:
# TODO - factor out!
name, _ = os.path.splitext(image_file)
text_file = name + '.txt'
with open(os.path.join(output_path, text_file), 'w') as detection_file:
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
detection_file.write(f'{predicted_class} {score:.6f} {left} {top} {right} {bottom}\n')
sess.close()
def draw(self,
test_model_path=None,
image_set='validation',
weights_name='trained_stage_3_best.h5',
out_path="output_images",
save_all=True):
"""
Draw bounding boxes on image data
"""
if test_model_path is None:
self.model_body.load_weights(weights_name)
else:
self.model_body = load_model(test_model_path)
if self.overfit_single_image:
partition_eval = self.partition["train"][[0, 0]]
else:
partition_eval = self.partition[image_set]
# load validation data
# only annotate 100 images max
if len(partition_eval) > 100:
partition_eval = np.random.choice(partition_eval, (100, ))
files_to_load = self.file_list[partition_eval]
print(files_to_load.shape)
image_data = [np.load(file)['image'] for file in files_to_load]
image_data = process_data(image_data)
image_data = np.array(
[np.expand_dims(image, axis=0) for image in image_data])
# Create output variables for prediction.
yolo_outputs = yolo_head(self.model_body.output, self.anchors,
len(self.class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=0.5,
iou_threshold=0.5)
# Run prediction images.
sess = K.get_session()
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
self.model_body.input: image_data[i],
input_image_shape:
[image_data.shape[2], image_data.shape[3]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[i][0], out_boxes,
out_classes, self.class_names,
out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path, str(i) + '.tif'))
def draw(model_body,
class_names,
anchors,
image_data,
image_data_original,
truth_boxes,
image_set='val',
weights_name='trained_stage_3_best.h5',
out_path="output_images",
save_all=True):
'''
Draw bounding boxes on image data
'''
image_size_orig = tuple(reversed(
image_data_original.shape[1:3])) # width, height
image_data_size = tuple(reversed(image_data.shape[1:3])) # width, height
print(image_size_orig)
print(image_data_size)
plotter = BoxPlotter(image_size_orig)
if image_set == 'train':
image_data = np.array([
np.expand_dims(image, axis=0)
for image in image_data[:int(len(image_data) * .9)]
])
elif image_set == 'val':
image_data = np.array([
np.expand_dims(image, axis=0)
for image in image_data[int(len(image_data) * .9):]
])
elif image_set == 'all':
image_data = np.array(
[np.expand_dims(image, axis=0) for image in image_data])
else:
ValueError("draw argument image_set must be 'train', 'val', or 'all'")
# model.load_weights(weights_name)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=0.07,
iou_threshold=0.5)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
if not os.path.exists(out_path):
os.makedirs(out_path)
for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data[i],
input_image_shape: [image_size_orig[1],
image_size_orig[0]], # height, width
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
# outboxes = [ymin, xmin, ymax, xmax]
out_boxes2 = out_boxes[:, [1, 0, 3, 2]]
image = image_data_original[i]
image = image / np.max(image) * 255
image = image.astype(np.uint8)
y = plotter.package_data(truth_boxes[i][:, 1:],
truth_boxes[i][:, 0].astype(np.int))
yhat = plotter.package_data(out_boxes2, out_classes.astype(np.int),
out_scores)
plotter.comparison(y, yhat, image)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data_original[i], out_boxes,
out_classes, class_names, out_scores)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path, str(i) + '.png'))
# To display (pauses the program):
# plt.imshow(image_with_boxes, interpolation='nearest')
# plt.show()
input("Enter for next image")
def predict_draw(model_body,
class_names,
anchors,
img_path,
weights_name='trained_stage_3_best.h5',
out_path="output_images",
save_all=True):
'''
Draw bounding boxes on image data
'''
head, tail = ntpath.split(img_path)
image = PIL.Image.open(img_path)
resized_image = image.resize(tuple(image_shape), PIL.Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0)
print(image_data.shape)
# model.load_weights(weights_name)
# print(image_data.shape)
model_body.load_weights(weights_name)
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
max_boxes=3,
score_threshold=.7,
iou_threshold=0.05)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
logging = TensorBoard()
if not os.path.exists(out_path):
os.makedirs(out_path)
#for i in range(len(image_data)):
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image_data.shape[1], image_data.shape[2]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores,
out_path + "\\" + tail)
# Save the image:
if save_all or (len(out_boxes) > 0):
image = PIL.Image.fromarray(image_with_boxes)
image.save(os.path.join(out_path, tail + '.png'))
# To display (pauses the program):
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()