def generate(self): model_path = os.path.expanduser(self.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' self.yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(self.class_names) num_anchors = len(self.anchors) # TODO: Assumes dim ordering is channel last model_output_channels = self.yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes' print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. self.model_image_size = self.yolo_model.layers[0].input_shape[1:3] self.is_fixed_size = self.model_image_size != (None, None) # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(self.yolo_model.output, self.anchors, len(self.class_names)) self.input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, self.input_image_shape, score_threshold=self.score, iou_threshold=self.iou) return boxes, scores, classes
def __init__(self, viden_yolo_weights_path, viden_crnn_weights_path, classes_path, out_path): self.act_model = create_crnn_model(train=False) self.act_model.load_weights( viden_crnn_weights_path ) # 'viden_trained_models\\viden_crnn_14May2021.hdf5') class_names = get_classes(classes_path) #print(class_names) self.out_path = out_path #self.anchors = YOLO_ANCHORS self.yolo_model_body, self.yolo_model = create_model( YOLO_ANCHORS, class_names, load_pretrained=False, freeze_body=False) self.yolo_model_body.load_weights( viden_yolo_weights_path ) #'viden_trained_models\\viden_yolo_14May2021.h5') self.yolo_outputs = yolo_head(self.yolo_model_body.output, YOLO_ANCHORS, len(class_names)) self.yolo_input_image_shape = K.placeholder(shape=(2, )) self.boxes, self.scores, self.classes = yolo_eval( self.yolo_outputs, self.yolo_input_image_shape, max_boxes=1, score_threshold=.7, iou_threshold=0.5)
def __init__(self): # TODO: parametrize init and review variable usage self.model_path = './model_data/tiny-yolo-voc.h5' assert self.model_path.endswith('.h5'), 'Keras model must be a .h5 file.' self.anchors_path = './model_data/tiny-yolo-voc_anchors.txt' self.classes_path = './model_data/pascal_classes.txt' self.score_threshold=0.4 self.iou_threshold=0.5 self.sess = K.get_session() # TODO: Remove dependence on Tensorflow session. with open(self.classes_path) as f: self.class_names = f.readlines() self.class_names = [c.strip() for c in self.class_names] with open(self.anchors_path) as f: self.anchors = f.readline() self.anchors = [float(x) for x in self.anchors.split(',')] self.anchors = np.array(self.anchors).reshape(-1, 2) self.yolo_model = load_model(self.model_path) self.is_fixed_size = True self.model_image_size = self.yolo_model.layers[0].input_shape[1:3] self.yolo_outputs = yolo_head(self.yolo_model.output, self.anchors, len(self.class_names)) self.input_image_shape = K.placeholder(shape=(2, )) self.boxes, self.scores, self.classes = yolo_eval( self.yolo_outputs, self.input_image_shape, score_threshold=self.score_threshold, iou_threshold=self.iou_threshold)
def draw(model_body, class_names, anchors, image_data, image_set='val', weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' if image_set == 'train': image_data = np.array([ np.expand_dims(image, axis=0) for image in image_data[:int(len(image_data) * .9)] ]) elif image_set == 'val': image_data = np.array([ np.expand_dims(image, axis=0) for image in image_data[int(len(image_data) * .9):] ]) elif image_set == 'all': image_data = np.array( [np.expand_dims(image, axis=0) for image in image_data]) else: ValueError("draw argument image_set must be 'train', 'val', or 'all'") # model.load_weights(weights_name) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data[i], input_image_shape: [image_data.shape[2], image_data.shape[3]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes, class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path, str(i) + '.png'))
def make_prediction(image_path, input_image_name, yolo_model): #Obtaining the dimensions of the input image input_image = Image.open(image_path + input_image_name) width, height = input_image.size width = np.array(width, dtype=float) height = np.array(height, dtype=float) #Assign the shape of the input image to image_shapr variable image_shape = (height, width) #Loading the classes and the anchor boxes that are provided in the model_data folder class_names = read_classes("model_data/yolo_classes.txt") anchors = read_anchors("model_data/yolo_anchors.txt") #Print the summery of the model yolo_model.summary() #Convert final layer features to bounding box parameters yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) #Now yolo_eval function selects the best boxes using filtering and non-max suppression techniques. # If you want to dive in more to see how this works, refer keras_yolo.py file in yad2k/models boxes, scores, classes = yolo_eval(yolo_outputs, image_shape) # Initiate a session sess = K.get_session() #Preprocess the input image before feeding into the convolutional network image, image_data = preprocess_image(image_path + input_image_name, model_image_size = (608, 608)) #Run the session out_scores, out_boxes, out_classes = sess.run( [scores, boxes, classes], feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 } ) #Print the results print('Found {} boxes for {}'.format(len(out_boxes), input_image_name)) #Produce the colors for the bounding boxs colors = generate_colors(class_names) #Draw the bounding boxes draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) #Apply the predicted bounding boxes to the image and save it image.save("predictions/" + input_image_name, quality=90) if(len(out_classes) == 0): result = "No box found" elif (out_classes[0] == 0): result = "real" else: result = "fake" return input_image_name, out_scores, result
def obj_detect(file_name): sess = K.get_session() model_path = 'model_data/tiny-yolo-voc.h5' anchors_path = 'model_data/tiny-yolo-voc_anchors.txt' classes_path = 'model_data/pascal_classes.txt' with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) yolo_model = load_model(model_path) num_classes = len(class_names) num_anchors = len(anchors) model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags.' print('{} model, anchors, and classes loaded.'.format(model_path)) model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # seems to return true most of the time. #print(is_fixed_size) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=.3, iou_threshold=.5) results = recognize_image(file_name, sess, boxes, scores, classes, is_fixed_size, model_image_size, yolo_model, input_image_shape, class_names, colors) K.clear_session() return results
def draw(model_body, class_names, anchors, partition, images_path, image_set='validation', weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' # load validation data hdf5_file_images = h5py.File(images_path, "r") image_data = hdf5_file_images["images"][partition[image_set], ...] hdf5_file_images.close() image_data = np.expand_dims(image_data, axis=1) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.5, iou_threshold=0.5) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data[i], input_image_shape: [image_data.shape[2], image_data.shape[3]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[i][0].astype(np.uint8) * 255, out_boxes, out_classes, class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path, str(i) + '.png'))
def test_(args): df_pred = pd.DataFrame(columns=['guid/image', 'N']) baselist = [] countlist = [] Ignore_class = [0, 9, 11, 14, 15, 16, 17, 21, 22] class_num = args.class_num anchors = YOLO_ANCHORS model_body = create_model(anchors, class_num) model_body.load_weights(args.weights_path) yolo_outputs = yolo_head(model_body.output, anchors, class_num) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=args.score_threshold, iou_threshold=0.4) sess = K.get_session() pic_num = 0 for path in os.listdir(args.data_path): test_path = os.path.join(args.data_path, path) for image_file in os.listdir(test_path): try: image_type = imghdr.what(os.path.join(test_path, image_file)) if not image_type: continue except IsADirectoryError: continue pic_num += 1 image = PIL.Image.open(os.path.join(test_path, image_file)) resized_image = image.resize((416, 416), PIL.Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') image_data /= 255. image_data = np.expand_dims(image_data, 0) out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) baselist.append(path + '/' + image_file[:4]) count = 0 print(len(out_classes)) for out_class in out_classes: count += out_class countlist.append(count) print(baselist[-1], countlist[-1], pic_num) df_pred['guid/image'] = baselist df_pred['N'] = countlist df_pred.to_csv(args.save_path, index=False) sess.close()
def detect_video(image_path, output_path, model_image_size, yolo_model, class_names, yolo_outputs): from keras import backend as K from keras.models import load_model from yad2k.models.keras_yolo import yolo_head, yolo_eval from PIL import Image input_image_shape = K.placeholder(shape=(2, )) sess = K.get_session() video_in = cv2.VideoCapture(image_path) #width = video_in.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH) # float #height = video_in.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT) # float width, height = int(video_in.get(3)), int(video_in.get(4)) FPS = video_in.get(5) video_out = cv2.VideoWriter() video_out.open(output_path, cv2.VideoWriter_fourcc(*'DIVX'), FPS, (width, height)) width = np.array(width, dtype=float) height = np.array(height, dtype=float) image_shape = (height, width) while video_in.isOpened(): ret, data = video_in.read() if ret == False: break video_array = cv2.cvtColor(data, cv2.COLOR_BGR2RGB) image = Image.fromarray(video_array, mode='RGB') resized_image = image.resize(tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. boxes, scores, classes = yolo_eval(yolo_outputs, image_shape) out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) colors = generate_colors(class_names) draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) video_out.write(cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)) sess.close() video_in.release() video_out.release() print("detect Done")
def model_body_processing(model_body, class_names, anchors): ''' function to be called once for loading weights and preparing the boxes,scores and classes according to anchor boxes values,score threshold and iou threshold. This is evaluated by non_max_suppression function. ''' global input_image_shape yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.5, iou_threshold=0.5) return boxes, scores, classes
def predict_objects(self, img): # t0 = time.time() #self.object_model.summary() c = np.fromstring(bytes(img.data), np.uint8) img = cv2.imdecode(c, cv2.IMREAD_COLOR) #im_pil img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) im_pil = Image.fromarray(img) # weight height of the image_sub width, height = im_pil.size print(width) print(height) width = np.array(width, dtype=float) height = np.array(height, dtype=float) image_shape = (height, width) t0 = time.time() #class names and anchors class_names = read_classes("/lanefollowing/ros2_ws/src/lane_following/model/coco_classes.txt") anchors = read_anchors("/lanefollowing/ros2_ws/src/lane_following/model/yolov2-tiny_anchors.txt") #yolo head yolo_outputs = yolo_head(self.object_model.output, anchors, len(class_names)) # yolo eval boxes, scores, classes = yolo_eval(yolo_outputs, image_shape) # print('boxes') # print(boxes) #yolo preprocess # pil_img, img_data = preprocess_image_object(im_pil, model_image_size = (416, 416)) pil_img, img_data = preprocess_image_object(im_pil, model_image_size = (608, 608)) #yolo predict # print('test1') out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],feed_dict={self.object_model.input:img_data,K.learning_phase(): 0}) t1 = time.time() self.inference_time = t1 - t0 # print("score = " ) # print(out_scores) # print("\n") # print('test2') print('Found {} boxes for '.format(len(out_boxes))) colors = generate_colors(class_names) draw_boxes(pil_img, out_scores, out_boxes, out_classes, class_names, colors) #might need to add resizing here... image = np.asarray(pil_img) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) cv2.putText(image, "Prediction time: %d ms" % (self.inference_time * 1000), (30, 220), cv2.FONT_HERSHEY_PLAIN, 3, (0, 255, 0), 2) cv2.putText(image, "Frame speed: %d fps" % (self.fps), (30, 270), cv2.FONT_HERSHEY_PLAIN, 3, (0, 255, 0), 2) image = cv2.resize(image, (round(image.shape[1] / 2), round(image.shape[0] / 2)), interpolation=cv2.INTER_AREA) cv2.imshow('YOLO Detector', image) cv2.waitKey(1)
def draw(model_body, class_names, anchors, image_data, image_set='val', weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' if image_set == 'train': image_data = np.array([np.expand_dims(image, axis=0) for image in image_data[:int(len(image_data)*.9)]]) elif image_set == 'val': image_data = np.array([np.expand_dims(image, axis=0) for image in image_data[int(len(image_data)*.9):]]) elif image_set == 'all': image_data = np.array([np.expand_dims(image, axis=0) for image in image_data]) else: ValueError("draw argument image_set must be 'train', 'val', or 'all'") # model.load_weights(weights_name) print(image_data.shape) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data[i], input_image_shape: [image_data.shape[2], image_data.shape[3]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes, class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path,str(i)+'.png'))
def prepare_yolo(model_path, class_names): yolo_model = load_model(model_path) yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags.' print('{} model, anchors, and classes loaded.'.format(model_path)) model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=args['score_threshold'], iou_threshold=args['iou_threshold'], max_boxes=50 ) model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) sess = K.get_session() return { 'yolo_model': yolo_model, 'input_image_shape': input_image_shape, 'boxes': boxes, 'scores': scores, 'classes': classes, 'sess': sess, 'model_image_size': model_image_size, 'class_names': class_names, 'colors': colors }
def pred(self, image_s, display_shape=(640, 480)): # Make predictions for images in (num_images, height, width, channel) format assert len( image_s.shape ) == 4 # image must have 4 dims ready to be sent into the graph # TODO allow multiple images at once. assert image_s.shape[0] == 1 features = self.yolo_model.predict(image_s) proc = yolo_head_np(features, self.anchors, len(self.class_names)) out_boxes, out_scores, out_classes = yolo_eval( proc, display_shape, max_boxes=self.max_boxes, score_threshold=self.score_threshold, iou_threshold=self.iou_threshold) return out_boxes, out_scores, out_classes
def draw(model_body, class_names, anchors, image_data, weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' # model.load_weights(weights_name) print(image_data.shape) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0.0) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data[i], input_image_shape: [image_data.shape[2], image_data.shape[3]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes, class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path,str(i)+'.png')) # To display (pauses the program): plt.imshow(image_with_boxes, interpolation='nearest') plt.show()
def predict(image_file): sess = K.get_session() """ Runs the graph stored in "sess" to predict boxes for "image_file". Prints and plots the preditions. Arguments: sess -- your tensorflow/Keras session containing the YOLO graph image_file -- name of an image stored in the "images" folder. Returns: out_boxes -- tensor of shape (None, 4), coordinates of the predicted boxes out_scores -- tensor of shape (None, ), scores of the predicted boxes out_classes -- tensor of shape (None, ), class index of the predicted boxes Note: "None" actually represents the number of predicted boxes, it varies between 0 and max_boxes. """ image, image_data = preprocess_image(image_file, model_image_size=(608, 608)) scores, boxes, classes = yolo_eval( yolo_outputs, np.reshape(image.size[::-1], [2]).astype(np.float32)) result = sess.run([scores, boxes, classes], feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 }) print(result) out_boxes, out_scores, out_classes = result # Print predictions info print('Found %s boxes for %s' % (len(out_boxes), image_file)) # Generate colors for drawing bounding boxes. colors = generate_colors(class_names) # Draw bounding boxes on the image file draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) imshow(image) plt.rcParams["figure.figsize"] = (20, 20) plt.show() return out_scores, out_boxes, out_classes
def start_model(self): ''' This needs to run to be able to use get_single_image ''' #Make the model self.yolo_model = self.get_model() self.summary() self.sess = K.get_session() time.sleep(2) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / self.CLASS, 1., 1.) for x in range(self.CLASS)] self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) self.colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), self.colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle( self.colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. self.yolo_outputs = yolo_head(self.yolo_model.output, self.ANCHORS, self.CLASS) self.input_image_shape = K.placeholder(shape=(2, )) self.boxes, self.scores, self.classes = yolo_eval( self.yolo_outputs, self.input_image_shape, score_threshold=self.SCORE_THRESHOLD, iou_threshold=self.IOU_THRESHOLD) self.label_size = 4 self.font = cv2.FONT_HERSHEY_SIMPLEX
def _main(args): sess = K.get_session() print("loading class file") global class_names with open(CLASSES_PATH) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] print("loading anchors") with open(ANCHORS_PATH) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) print("loading yolo model") global yolo_model yolo_model = load_model(MODEL_PATH) # Verify model, anchors, and classes are compatible num_classes = len(class_names) # TODO: Assumes dim ordering is channel last model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == len(anchors) * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Exit.' print('{} model, anchors, and classes loaded.'.format(MODEL_PATH)) # Check if model is fully convolutional, assuming channel last order. global model_image_size model_image_size = yolo_model.layers[0].input_shape[1:3] # Generate colors for drawing bounding boxes. hsv_tuples = [(x / num_classes, 1., 1.) for x in range(num_classes)] global colors colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(yolo_model.output, anchors, num_classes) global input_image_shape input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) ######################################################################### print("capture pi camera...") # set camera resolution with picamera.PiCamera() as camera: camera.resolution = (320, 320) # use opencv to capture frames cam = cv2.VideoCapture(0) cam.set(cv2.CAP_PROP_FRAME_WIDTH, 320) cam.set(cv2.CAP_PROP_FRAME_HEIGHT, 320) cam.set(cv2.CAP_PROP_FPS, 30) rval, frame = cam.read() if rval == False: print("Error: Camera is not Found. Exit") sys.exit(1) cv2.imshow("preview", np.array(frame)) while True: rval, frame = cam.read() before = time.time() detect_img = detect_image(frame, sess, boxes, scores, classes) after = time.time() cv2.imshow("preview", np.array(detect_img)) if cv2.waitKey(1) & 0xFF == ord('q'): break print("FPS: {}".format(1 / (after-before))) sess.close() cam.release() cv2.destroyAllWindows()
def _main(args): model_path = os.path.expanduser(args.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(args.anchors_path) classes_path = os.path.expanduser(args.classes_path) test_path = os.path.expanduser(args.test_path) output_path = os.path.expanduser(args.output_path) if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() # TODO: Remove dependence on Tensorflow session. with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) # TODO: Assumes dim ordering is channel last model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags.' print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) for image_file in os.listdir(test_path): try: image_type = imghdr.what(os.path.join(test_path, image_file)) if not image_type: continue except IsADirectoryError: continue image = Image.open(os.path.join(test_path, image_file)) if is_fixed_size: # TODO: When resizing we can use minibatch input. resized_image = image.resize( tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_file)) font = ImageFont.truetype( font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw image.save(os.path.join(output_path, image_file), quality=90) sess.close()
def _main(args): model_path = os.path.expanduser(args['model_path']) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(args['anchors_path']) classes_path = os.path.expanduser(args['classes_path']) test_path = os.path.expanduser(args['test_path']) output_path = os.path.expanduser(args['output_path']) score_threshold = args['score_threshold'] iou_threshold = args['iou_threshold'] if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags.' print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=score_threshold, iou_threshold=iou_threshold) for image_file in os.listdir(test_path): try: image_type = imghdr.what(os.path.join(test_path, image_file)) if not image_type: continue except IsADirectoryError: continue image = Image.open(os.path.join(test_path, image_file)) if is_fixed_size: resized_image = image.resize(tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_file)) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw image.save(os.path.join(output_path, image_file), quality=90) sess.close()
def __predict_output__(): plt.interactive(False) cfg = Configuration() GPU = True if GPU != True: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "" # Input Path root_dir = os.path.dirname(os.path.abspath(__file__)) image_path = cfg.image_path json_path = os.path.join(root_dir, cfg.input_filename) testingset = os.path.join(root_dir, 'testingset') Preprocessor.__generate_kijiji_set__(root_dir, image_path, json_path, testingset, 'model') # ------------------generator to compile training data of kijiji dataset---------------------------------------- image_path = os.path.join(root_dir, 'testingset') data_path = glob(image_path + "/*") # Image Segmentation Parameters model_path = os.path.expanduser(cfg.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(cfg.anchors_path) classes_path = os.path.expanduser(cfg.classes_path) test_path = os.path.expanduser(cfg.test_path) output_path = os.path.expanduser(cfg.segmented_output_path) json_path = os.path.expanduser(cfg.json_output) if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() class_names = Preprocessor.__return_class_names__(classes_path) anchors = Preprocessor.__return_anchors__(anchors_path) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) info = 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and --classes_path flags.' model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), info print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate Colors for drawing bounding boxes hsv_tuples, colors = Preprocessor.__generate_colors_for_bounding_boxes__( class_names) yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=cfg.score_threshold, iou_threshold=cfg.iou_threshold) # Load Images from the root folder input_images_model_1, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, True, # Segmentation Flag False, # Edge-detection Flag True, # Extract object Flag False) # Gray Scale Flag input_images_model_2, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, True, False, False) input_images_model_3, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.image_height, cfg.image_width, cfg.channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, False, False, False) input_shape = [ cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel ] input_shape_3 = [cfg.image_height, cfg.image_width, cfg.channel] # load (pre-trained) weights for model_1 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_1_save # the path where the model weights are stored weight_file = 'model_1.h5' model_1 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_1") # load (pre-trained) weights for model_2 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_2_save # the path where the model weights are stored weight_file = 'model_2.h5' model_2 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_2") # load (pre-trained) weights for model_2 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_3_save # the path where the model weights are stored weight_file = 'model_3.h5' model_3 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_3") print(root_dir) print(os.path.join(root_dir, cfg.output_model_1)) output_path_model_1 = os.path.join(root_dir + cfg.output_model_1) output_path_model_2 = os.path.join(root_dir + cfg.output_model_2) output_path_model_3 = os.path.join(root_dir + cfg.output_model_3) Preprocessor.__create_output_directories__(output_path_model_1) Preprocessor.__create_output_directories__(output_path_model_2) Preprocessor.__create_output_directories__(output_path_model_3) features_from_model_1 = Preprocessor.__get_score_model__( model_1, input_images_model_1, output_path_model_1) features_from_model_2 = Preprocessor.__get_score_model__( model_2, input_images_model_2, output_path_model_2) features_from_model_3 = Preprocessor.__get_score_model__( model_3, input_images_model_3, output_path_model_3) features_from_model_1 = Preprocessor.__flatten_img_data__( features_from_model_1) features_from_model_2 = Preprocessor.__flatten_img_data__( features_from_model_2) features_from_model_3 = Preprocessor.__flatten_img_data__( features_from_model_3) fused_features = np.concatenate([ features_from_model_1, features_from_model_2, features_from_model_3 ], axis=1) fused_features = [ Preprocessor.__binarize__(features) for features in fused_features ] counter_for_predictions = 0 sub_average_precision_make, sub_average_precision_color = [], [] sub_average_precision_body, sub_average_precision_model = [], [] cum_average_precision_make, cum_average_precision_color = [], [] cum_average_precision_body, cum_average_precision_model = [], [] precision_at_3_5_10_all = ''.join(cfg.precision_counter).split(',') while counter_for_predictions <= 2: test_image_idx = int(len(input_images_model_1) * random()) if test_image_idx < len(data_path_with_image_name): idx_closest = Preprocessor.__get_closest_images__( test_image_idx, fused_features, cfg.number_of_predictions) test_image = Preprocessor.__get_concatenated_images__( data_path_with_image_name, [test_image_idx], cfg.compressed_image_width) results_image = Preprocessor.__get_concatenated_images__( data_path_with_image_name, idx_closest, cfg.compressed_image_width) source_category = str( data_path_with_image_name[test_image_idx]).split('/') similar_image = [] similar_idx_closest = [] for counter_for_recommendations in range(0, len(idx_closest)): category = str(data_path_with_image_name[ idx_closest[counter_for_recommendations]]).split('/') if str(source_category[-2]).strip() == str( category[-2].strip()): similar_image.append(data_path_with_image_name[ idx_closest[counter_for_recommendations]]) similar_idx_closest.append( idx_closest[counter_for_recommendations]) print("Test Image ID:", test_image_idx) print("\n") print("Closest Images ID:", idx_closest) print("\n") print("Similar Images ID", similar_idx_closest) print("\n") precision_per_make, precision_per_color = [], [] precision_per_body_wise, precision_per_model_wise = [], [] results_image_recommendations = [] for i in range(0, len(precision_at_3_5_10_all)): results_image_recommendations = Preprocessor.__get_concatenated_images__( data_path_with_image_name, similar_idx_closest, cfg.compressed_image_width) list_of_similar_image_names = Preprocessor.__return_image_names__( data_path_with_image_name, similar_idx_closest) name_of_test_image = Preprocessor.__return_image_names__( data_path_with_image_name, [test_image_idx]) dict_of_attributes_of_similar_images = Preprocessor.__get_attributes_list__( list_of_similar_image_names, os.path.join(root_dir, cfg.input_filename)) dict_of_attributes_of_test_image = Preprocessor.__get_attributes_list__( name_of_test_image, os.path.join(root_dir, cfg.input_filename)) similar_make_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'make') similar_color_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'color') similar_body_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'body') similar_model_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'model') precision_per_make.append( float( float(len(similar_make_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_color.append( float( float(len(similar_color_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_body_wise.append( float( float(len(similar_body_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_model_wise.append( float( float(len(similar_model_wise)) / int(precision_at_3_5_10_all[i]))) sub_average_precision_make.append(precision_per_make) sub_average_precision_color.append(precision_per_color) sub_average_precision_body.append(precision_per_body_wise) sub_average_precision_model.append(precision_per_model_wise) imsave('test.png', test_image) imsave('recommendations.png', results_image_recommendations) imsave('total_results.png', results_image) counter_for_predictions += 1 time.sleep(1) else: print("Index is out of bound") cum_average_precision_make.append( map(Preprocessor.__mean__, zip(*sub_average_precision_make))) cum_average_precision_color.append( map(Preprocessor.__mean__, zip(*sub_average_precision_color))) cum_average_precision_body.append( map(Preprocessor.__mean__, zip(*sub_average_precision_body))) cum_average_precision_model.append( map(Preprocessor.__mean__, zip(*sub_average_precision_model))) print("\n \n \n") print( "-----------------------------------------------------------------------------------" ) print("Average Precision Make-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_make))) print("Average Precision Color-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_color))) print("Average Precision Body-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_body))) print("Average Precision Model-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_model))) writer = csv.writer(open(os.path.join(root_dir, 'results.csv'), 'w')) writer.writerow([ "Make-Wise: Precision at 3", "Make-Wise: Precision at 5", "Make-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_make): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Color-Wise: Precision at 3", "Color-Wise: Precision at 5", "Color-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_color): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Body-Wise: Precision at 3", "Body-Wise: Precision at 5", "Body-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_body): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Model-Wise: Precision at 3", "Model-Wise: Precision at 5", "Model-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_model): writer.writerow(row) writer.writerow('\n')
def _main(args): def predict(sess, image): # Preprocess your image image, image_data = preprocess_image(image, model_image_size=(416, 416)) # Run the session with the correct tensors and choose the correct placeholders in the feed_dict. # You'll need to use feed_dict={yolo_model.input: ... , K.learning_phase(): 0}) out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 }) # Print predictions info print('Found {} boxes'.format(len(out_boxes))) # Generate colors for drawing bounding boxes. colors = generate_colors(class_names) # Draw bounding boxes on the image file out_image = draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) return out_image, out_scores, out_boxes, out_classes video_path = os.path.join(INPUT_PATH, args.input) input = cv2.VideoCapture(video_path) if (input.isOpened() == False): print("Error opening video file.") return height = input.get(cv2.CAP_PROP_FRAME_HEIGHT) width = input.get(cv2.CAP_PROP_FRAME_WIDTH) fps = input.get(cv2.CAP_PROP_FPS) if not os.path.exists(OUTPUT_PATH): os.mkdir(OUTPUT_PATH) output_file_path = os.path.join(OUTPUT_PATH, args.input) output = cv2.VideoWriter(output_file_path, cv2.VideoWriter_fourcc(*'MP4V'), fps, (int(width), int(height))) image_shape = (height, width) sess = K.get_session() class_names = read_classes(CLASS_LIST_PATH) anchors = read_anchors(YOLO_ANCHORS_PATH) yolo_model, model = create_model(anchors, class_names) yolo_model.load_weights(args.weights) yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) boxes, scores, classes = yolo_eval(yolo_outputs, image_shape, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) n_frame = 0 while input.isOpened(): ret, original_frame = input.read() if ret: n_frame = n_frame + 1 print("processing frame {} ...".format(n_frame)) process_frame = cv2.cvtColor(original_frame, cv2.COLOR_BGR2RGB) process_frame = Image.fromarray(process_frame) process_frame, out_scores, out_boxes, out_classes = predict( sess, process_frame) process_frame = cv2.cvtColor(process_frame, cv2.COLOR_BGR2RGB) output.write(process_frame) else: break print("Finished. Output file: ", output_file_path) input.release() output.release() cv2.destroyAllWindows()
hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=score_threshold, iou_threshold=iou_threshold) def draw_pred(image): if type(image) == np.ndarray: image = Image.fromarray(image) elif not issubclass(type(image),Image.Image): raise Exception('image must be of type PIL.Image.Image') if is_fixed_size: # TODO: When resizing we can use minibatch input. resized_image = image.resize( tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have
# Initiate a session sess = K.get_session() while True: # Capture frame-by-frame try: ret, frame = cap.read() image = frame print("Type of image is: ",type(frame)) width, height = image.size image_shape = (height, width) boxes, scores, classes = yolo_eval(yolo_outputs, image_shape) width = np.array(width, dtype=float) height = np.array(height, dtype=float) image, image_data = preprocess_image(image, model_image_size = (608, 608)) #resized_image = image.resize(tuple(reversed((608,608)), Image.BICUBIC) #image_data = np.array(resized_image, dtype='float32') #image_data /= 255. #image_data = np.expand_dims(image_data, 0) #Run the session #out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],feed_dict={yolo_model.input:image_data,K.learning_phase(): 0}) #Print the results print('Found {} boxes for {}'.format(len(out_boxes), input_image_name)) #Produce the colors for the bounding boxs colors = generate_colors(class_names)
def _main(args): voc_path = os.path.expanduser(args.data_path) classes_path = os.path.expanduser(args.classes_path) anchors_path = os.path.expanduser(args.anchors_path) with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] if os.path.isfile(anchors_path): with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) else: anchors = YOLO_ANCHORS voc = h5py.File(voc_path, 'r') image = PIL.Image.open(io.BytesIO(voc['train/images'][28])) orig_size = np.array([image.width, image.height]) orig_size = np.expand_dims(orig_size, axis=0) # Image preprocessing. image = image.resize((416, 416), PIL.Image.BICUBIC) image_data = np.array(image, dtype=np.float) image_data /= 255. # Box preprocessing. # Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max. boxes = voc['train/boxes'][28] boxes = boxes.reshape((-1, 5)) # Get extents as y_min, x_min, y_max, x_max, class for comparision with # model output. boxes_extents = boxes[:, [2, 1, 4, 3, 0]] # Get box parameters as x_center, y_center, box_width, box_height, class. boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3]) boxes_wh = boxes[:, 3:5] - boxes[:, 1:3] boxes_xy = boxes_xy / orig_size boxes_wh = boxes_wh / orig_size boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1) # Precompute detectors_mask and matching_true_boxes for training. # Detectors mask is 1 for each spatial position in the final conv layer and # anchor that should be active for the given boxes and 0 otherwise. # Matching true boxes gives the regression targets for the ground truth box # that caused a detector to be active or 0 otherwise. detectors_mask_shape = (13, 13, 5, 1) matching_boxes_shape = (13, 13, 5, 5) detectors_mask, matching_true_boxes = preprocess_true_boxes(boxes, anchors, [416, 416]) # Create model input layers. image_input = Input(shape=(416, 416, 3)) boxes_input = Input(shape=(None, 5)) detectors_mask_input = Input(shape=detectors_mask_shape) matching_boxes_input = Input(shape=matching_boxes_shape) print('Boxes:') print(boxes) print('Box corners:') print(boxes_extents) print('Active detectors:') print(np.where(detectors_mask == 1)[:-1]) print('Matching boxes for active detectors:') print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]]) # Create model body. model_body = yolo_body(image_input, len(anchors), len(class_names)) model_body = Model(image_input, model_body.output) # Place model loss on CPU to reduce GPU memory usage. with tf.device('/cpu:0'): # TODO: Replace Lambda with custom Keras layer for loss. model_loss = Lambda( yolo_loss, output_shape=(1, ), name='yolo_loss', arguments={'anchors': anchors, 'num_classes': len(class_names)})([ model_body.output, boxes_input, detectors_mask_input, matching_boxes_input ]) model = Model( [image_input, boxes_input, detectors_mask_input, matching_boxes_input], model_loss) model.compile( optimizer='adam', loss={ 'yolo_loss': lambda y_true, y_pred: y_pred }) # This is a hack to use the custom loss function in the last layer. # Add batch dimension for training. image_data = np.expand_dims(image_data, axis=0) boxes = np.expand_dims(boxes, axis=0) detectors_mask = np.expand_dims(detectors_mask, axis=0) matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0) num_steps = 1000 # TODO: For full training, put preprocessing inside training loop. # for i in range(num_steps): # loss = model.train_on_batch( # [image_data, boxes, detectors_mask, matching_true_boxes], # np.zeros(len(image_data))) model.fit([image_data, boxes, detectors_mask, matching_true_boxes], np.zeros(len(image_data)), batch_size=1, epochs=num_steps) model.save_weights('overfit_weights.h5') # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, score_threshold=.3, iou_threshold=.9) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes, class_names, out_scores) plt.imshow(image_with_boxes, interpolation='nearest') plt.show()
def _main(): voc_path = os.path.expanduser( '/Users/ss/Data/VOCdevkit/pascal_voc_07_07.hdf5') classes_path = os.path.expanduser('model_data/pascal_classes.txt') with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] voc = h5py.File(voc_path, 'r') image = PIL.Image.open(io.BytesIO(voc['train/images'][28])) orig_size = np.array([image.width, image.height]) orig_size = np.expand_dims(orig_size, axis=0) # Image preprocessing. image = image.resize((416, 416), PIL.Image.BICUBIC) image_data = np.array(image, dtype=np.float) image_data /= 255. # Box preprocessing. # Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max. boxes = voc['train/boxes'][28] boxes = boxes.reshape((-1, 5)) # Get extents as y_min, x_min, y_max, x_max, class for comparision with # model output. boxes_extents = boxes[:, [2, 1, 4, 3, 0]] # Get box parameters as x_center, y_center, box_width, box_height, class. boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3]) boxes_wh = boxes[:, 3:5] - boxes[:, 1:3] boxes_xy = boxes_xy / orig_size boxes_wh = boxes_wh / orig_size boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1) # Precompute detectors_mask and matching_true_boxes for training. # Detectors mask is 1 for each spatial position in the final conv layer and # anchor that should be active for the given boxes and 0 otherwise. # Matching true boxes gives the regression targets for the ground truth box # that caused a detector to be active or 0 otherwise. anchors = COCO_ANCHORS detectors_mask_shape = (13, 13, 5, 1) matching_boxes_shape = (13, 13, 5, 5) detectors_mask, matching_true_boxes = preprocess_true_boxes( boxes, anchors, [416, 416]) # Create model input layers. image_input = Input(shape=(416, 416, 3)) boxes_input = Input(shape=(None, 5)) detectors_mask_input = Input(shape=detectors_mask_shape) matching_boxes_input = Input(shape=matching_boxes_shape) print(boxes) print(boxes_extents) print(np.where(detectors_mask == 1)[:-1]) print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]]) # Create model body. model_body = yolo_body(image_input, len(anchors), len(class_names)) model_body = Model(image_input, model_body.output) # Place model loss on CPU to reduce GPU memory usage. with tf.device('/cpu:0'): # TODO: Replace Lambda with custom Keras layer for loss. model_loss = Lambda(yolo_loss, output_shape=(1, ), name='yolo_loss', arguments={ 'anchors': anchors, 'num_classes': len(class_names) })([ model_body.output, boxes_input, detectors_mask_input, matching_boxes_input ]) model = Model( [image_input, boxes_input, detectors_mask_input, matching_boxes_input], model_loss) model.compile( optimizer='adam', loss={ 'yolo_loss': lambda y_true, y_pred: y_pred }) # This is a hack to use the custom loss function in the last layer. # Add batch dimension for training. image_data = np.expand_dims(image_data, axis=0) boxes = np.expand_dims(boxes, axis=0) detectors_mask = np.expand_dims(detectors_mask, axis=0) matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0) num_steps = 1000 # TODO: For full training, put preprocessing inside training loop. # for i in range(num_steps): # loss = model.train_on_batch( # [image_data, boxes, detectors_mask, matching_true_boxes], # np.zeros(len(image_data))) model.fit([image_data, boxes, detectors_mask, matching_true_boxes], np.zeros(len(image_data)), batch_size=1, epochs=num_steps) model.save_weights('overfit_weights.h5') # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=.3, iou_threshold=.9) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes, class_names, out_scores) plt.imshow(image_with_boxes, interpolation='nearest') plt.show()
def _main(args): model_path = os.path.expanduser(args.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(args.anchors_path) classes_path = os.path.expanduser(args.classes_path) test_path = os.path.expanduser(args.test_path) output_path = os.path.expanduser(args.output_path) url = args.ws_addr video = args.video if video: pygame.init() screen = pygame.display.set_mode((720, 720)) x = (screen.get_width() - resolution[0]) / 2 y = (screen.get_height() - resolution[1]) / 2 if url: ws = websocket.create_connection(url) ws.send(json.dumps({'device': 'yolo'})) else: ws = None if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() # TODO: Remove dependence on Tensorflow session. with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) # TODO: Assumes dim ordering is channel last model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags.' print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) with picamera.PiCamera() as camera: output = picamera.array.PiRGBArray(camera) exitFlag = True counter = 0 while (exitFlag): if video: for event in pygame.event.get(): if (event.type is pygame.MOUSEBUTTONDOWN or event.type is pygame.QUIT): exitFlag = False camera.resolution = resolution camera.capture(output, 'rgb') image_pixels = output.array if is_fixed_size: # TODO: When resizing we can use minibatch input. image = Image.fromarray(image_pixels) resized_image = image.resize(tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), counter)) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 output.truncate(0) for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print("{} identified. Uploading profile to the NSA.".format( label)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) # image.save(os.path.join(output_path, str(counter) + '.png'), quality=90) counter += 1 buff = BytesIO() image.save(buff, format="JPEG") ws_image_bytes = buff.getvalue() del draw if ws: send_data = { 'type': 'yolo', 'data': base64.b64encode(ws_image_bytes).decode('utf-8') } ws.send(json.dumps(send_data)) image_bytes = image.tobytes()[0:resolution[0] * resolution[1] * 3] py_image = pygame.image.fromstring(image_bytes, camera.resolution, 'RGB') if video: screen.fill(0) if py_image: screen.blit(py_image, (x, y)) pygame.display.update() sess.close()
def yoloThread(): global frames,times model_path = scriptFolder+"tiny.h5" anchors_path = scriptFolder+"anchors.txt" sess = K.get_session() print("[PiCam] Loading anchors file...") anchors = [1.08,1.19,3.42,4.41,6.63,11.38,9.42,5.11,16.62,10.52] anchors = np.array(anchors).reshape(-1, 2) print("[PiCam] Loading yolo model ({})...".format(scriptFolder+"tiny.h5")) yolo_model = load_model(model_path) num_anchors = len(anchors) print('[PiCam] YOLO model loaded !'.format(model_path)) model_image_size = yolo_model.layers[0].input_shape[1:3] yolo_outputs = yolo_head(yolo_model.output, anchors, 20) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs,input_image_shape,score_threshold=0.3,iou_threshold=0.4) num = 0 #Photo name old_time = 0.0 #Latest time print("[PiCam] YOLO Thread started!") ### Loop: while True: if len(frames) != 0: try: cv2.waitKey(17) mat = frames[0] mat = cv2.resize(mat,(model_image_size[0],model_image_size[1])) in_mat = np.array(mat,dtype='float32') in_mat /= 255. in_mat = np.expand_dims(in_mat, 0) if (times[0] - old_time) > time_chunck: out_boxes, out_scores, out_classes = sess.run([boxes, scores, classes],feed_dict={yolo_model.input: in_mat,input_image_shape: [mat.shape[1], mat.shape[0]],K.learning_phase(): 0}) if len(out_boxes) > 0: writ = False xs,ys = [],[] for i, c in reversed(list(enumerate(out_classes))): if c == 14: #14 is the label for persons writ = True box = out_boxes[i] top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(mat.shape[1], np.floor(bottom + 0.5).astype('int32')) right = min(mat.shape[0], np.floor(right + 0.5).astype('int32')) # cv2.rectangle(mat,(left+i,top+i),(right-i,bottom-i),(10,230,10),1) xs.append(left+i) xs.append(right-i) ys.append(top+i) ys.append(bottom-i) if writ: img_name = scriptFolder+"imgs/{}.png".format(num) cv2.imwrite(img_name,mat[min(ys):max(ys),min(xs):max(xs)]) #we only save the part of the images where we have found persons out_s = "[{}] Detected person (taken {}s)!\n".format(time.strftime("%H:%M:%S"),round(time.time()-times[0])) print(out_s) flog.write(out_s) flog.flush() #system("telegram-cli -W -e \"send_photo {} {} \"".format(telegram_user,img_name)) try: system("telegram-cli -W -e \"send_photo {} {} \"".format(telegram_user,img_name)) except Exception as exc: print("[PiCam] Some error occured in YOLO Thread ({}) :".format(time.strftime("%H:%M:%S")),exc) num += 1 old_time = times[0] except Exception as ex: print("[PiCam] Some error occured in YOLO Thread ({}) :".format(time.strftime("%H:%M:%S")),ex) del times[0] del frames[0] cv2.waitKey(50)
def _main(args): model_path = os.path.expanduser(args.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(args.anchors_path) classes_path = os.path.expanduser(args.classes_path) debug = os.path.expanduser(args.debug) debugging = False if (debug != 'off'): debugging = True sess = K.get_session() with open(classes_path) as f: class_names = f.readlines() class_names = [c.strip() for c in class_names] with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] anchors = np.array(anchors).reshape(-1, 2) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), \ 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and ' \ '--classes_path flags. Should be in model_data folder.' # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) if debugging: print( "Note that debugging mode will make processing time slower due to camera preview time." ) cachedposition = (-1000, -1000) cachedprediction = ['nothing', 'nothing'] with picamera.PiCamera(sensor_mode=6) as camera: camera.resolution = (800, 640) #camera.video_stabilization = True #camera.shutter_speed = 800 print("\nPreloading success! Now running...") while True: stream = io.BytesIO() if debugging: #camera.start_preview() #time.sleep(2) #camera.stop_preview() start_time = time.time() camera.capture(stream, 'jpeg') if debugging: print("Capture time : %.3f" % (time.time() - start_time)) #stream.seek(0) if debugging: opening = time.time() image = Image.open(stream) if debugging: print("Open Image time : %.3f" % (time.time() - opening)) if is_fixed_size: resized_image = image.resize(tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), iZmage.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. if debugging: time1 = time.time() out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [800, 640], #input_image_shape: [640, 480], K.learning_phase(): 0 }) if debugging: print("Sess Run time : %.3f" % (time.time() - time1)) #font = ImageFont.truetype( # font='font/FiraMono-Medium.otf', # size=2 * np.floor(2e-2 * image.size[1] + 0.5).astype('int32')) #thickness = 2 * (image.size[0] + image.size[1]) // 300 message = '' message_head = "\n[INFO] Predicted scene: User is" label_result = [] match_pair = [] dic = { 'person': 'next to another person!', 'motorbike': 'doing workout with a Gym Bicycle!', 'bicycle': 'doing workout with a Gym Bicycle!', 'sports ball': 'doing workout with a Yoga Ball!', 'apple': 'doing workout with a Yoga Ball!', 'mouse': 'doing workout with a Yoga Ball!', 'chair': 'taking a rest!', 'sofa': 'taking a rest!' } weights = { 'apple': 0.1, 'person': 0.7, 'bicycle': 0.1, 'sports ball': 0.1, 'motorbike': 0.1, 'mouse': 0.1, 'chair': 0.1, 'sofa': 0.1 } for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) label_result.append(predicted_class) #draw = ImageDraw.Draw(image) #label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) # w and h of the box w = bottom - top h = right - left x = top y = right # print(label, (left, top), (right, bottom)) # add to match_pair for comparison if predicted_class in [ 'person', 'bicycle', 'sports ball', 'apple', 'motorbike', 'mouse', 'chair', 'sofa' ]: match_pair.append([c, [x, y, w, h]]) # now the matched_pair contains only person and predefined items #if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) #else: text_origin = np.array([left, top + 1]) #if predicted_class in ['person']: # for i in range(thickness): # draw.rectangle( # [left + i, top + i, right - i, bottom - i], # outline=colors[c]) message_origin = np.array([0, 10]) person_box, ball_box, chair_box, bicycle_box = [], [], [], [] match_pair = sorted(match_pair) founduser = False for i in range(1, len(match_pair)): try: test_pair = match_pair[i][0] test_class = class_names[test_pair] test_dic = dic[test_class] if test_pair == match_pair[0][0]: continue if intersect_area(match_pair[0][1], match_pair[i][1]) > 0: if test_class in ['bicycle', 'motorbike']: message = "Predicted scenery: Person is " + test_dic print(message_head, test_dic, "\n[INFO] User Pos:", body_pos(match_pair[0][1])) #draw.text(message_origin, message, fill=(255, 0, 0), font=font) cachedprediction[0] = cachedprediction[1] cachedprediction[1] = 'bicycle' break if test_class in ['sports ball', 'apple', 'mouse']: message = "Predicted scenery: Person is " + test_dic print(message_head, test_dic, "\n[INFO] User Pos:", body_pos(match_pair[0][1])) #draw.text(message_origin, message, fill=(255, 0, 0), font=font) cachedprediction[0] = cachedprediction[1] cachedprediction[1] = 'yoga' break if test_class in ['chair', 'sofa']: message = "Predicted scenery: Person is " + test_dic print(message_head, test_dic, "\n[INFO] User Pos:", body_pos(match_pair[0][1])) #draw.text(message_origin, message, fill=(255, 0, 0), font=font) cachedprediction[0] = cachedprediction[1] cachedprediction[1] = 'chair' break else: founduser = True if cachedprediction[1] != 'nothing': printpred(cachedprediction[1]) elif cachedprediction[0] != 'nothing': printpred(cachedprediction[0]) elif (distance(body_pos(match_pair[0][1]), cachedposition) < 4): print( "\n[INF0] Predicted scene: User is taking a rest!" ) cachedprediction[0] = cachedprediction[1] cachedprediction[1] = 'chair' else: cachedprediction[0] = cachedprediction[1] cachedprediction[1] = 'nothing' print("[INFO] User Pos:", body_pos(match_pair[0][1])) cachedposition = body_pos(match_pair[0][1]) break except KeyError: continue if not founduser: cachedposition = (-1000, -1000) label_result = [] # cache if debugging: print("Loop time : %.3f\n" % (time.time() - start_time)) # -2 for the 2 second camera preview time sess.close()
def _main(args): test_path = os.path.expanduser(args.test_path) output_path = os.path.expanduser(args.output_path) if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.makedirs(output_path) if args.gpu: # https://github.com/keras-team/keras/issues/3685 os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu yolo_model, class_names, anchors = load_model(args.model_path, args.classes_path, args.anchors_path) print('{} model, anchors, and classes loaded.'.format(args.model_path)) sess = K.get_session() # TODO: Remove dependence on Tensorflow session. # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate colors for drawing bounding boxes. hsv_tuples = [(x / len(class_names), 1., 1.) for x in range(len(class_names))] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list( map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(10101) # Fixed seed for consistent colors across runs. random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes. random.seed(None) # Reset seed to default. # Generate output tensor targets for filtered bounding boxes. # TODO: Wrap these backend operations with Keras layers. yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval( yolo_outputs, input_image_shape, max_boxes=20, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold) for image_file in sorted(os.listdir(test_path)): try: image_type = imghdr.what(os.path.join(test_path, image_file)) if not image_type: continue except (IsADirectoryError, PermissionError): continue image = Image.open(os.path.join(test_path, image_file)) if image.mode != 'RGB': image = image.convert(mode='RGB') if is_fixed_size: # TODO: When resizing we can use minibatch input. resized_image = image.resize( tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_file)) font = ImageFont.truetype( font=os.path.join(os.path.dirname(__file__), 'font', 'FiraMono-Medium.otf'), size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw image.save(os.path.join(output_path, image_file), quality=90) if args.text: # TODO - factor out! name, _ = os.path.splitext(image_file) text_file = name + '.txt' with open(os.path.join(output_path, text_file), 'w') as detection_file: for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) detection_file.write(f'{predicted_class} {score:.6f} {left} {top} {right} {bottom}\n') sess.close()
def draw(self, test_model_path=None, image_set='validation', weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): """ Draw bounding boxes on image data """ if test_model_path is None: self.model_body.load_weights(weights_name) else: self.model_body = load_model(test_model_path) if self.overfit_single_image: partition_eval = self.partition["train"][[0, 0]] else: partition_eval = self.partition[image_set] # load validation data # only annotate 100 images max if len(partition_eval) > 100: partition_eval = np.random.choice(partition_eval, (100, )) files_to_load = self.file_list[partition_eval] print(files_to_load.shape) image_data = [np.load(file)['image'] for file in files_to_load] image_data = process_data(image_data) image_data = np.array( [np.expand_dims(image, axis=0) for image in image_data]) # Create output variables for prediction. yolo_outputs = yolo_head(self.model_body.output, self.anchors, len(self.class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.5, iou_threshold=0.5) # Run prediction images. sess = K.get_session() if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ self.model_body.input: image_data[i], input_image_shape: [image_data.shape[2], image_data.shape[3]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[i][0], out_boxes, out_classes, self.class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path, str(i) + '.tif'))
def draw(model_body, class_names, anchors, image_data, image_data_original, truth_boxes, image_set='val', weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' image_size_orig = tuple(reversed( image_data_original.shape[1:3])) # width, height image_data_size = tuple(reversed(image_data.shape[1:3])) # width, height print(image_size_orig) print(image_data_size) plotter = BoxPlotter(image_size_orig) if image_set == 'train': image_data = np.array([ np.expand_dims(image, axis=0) for image in image_data[:int(len(image_data) * .9)] ]) elif image_set == 'val': image_data = np.array([ np.expand_dims(image, axis=0) for image in image_data[int(len(image_data) * .9):] ]) elif image_set == 'all': image_data = np.array( [np.expand_dims(image, axis=0) for image in image_data]) else: ValueError("draw argument image_set must be 'train', 'val', or 'all'") # model.load_weights(weights_name) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.07, iou_threshold=0.5) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. if not os.path.exists(out_path): os.makedirs(out_path) for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data[i], input_image_shape: [image_size_orig[1], image_size_orig[0]], # height, width K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) # outboxes = [ymin, xmin, ymax, xmax] out_boxes2 = out_boxes[:, [1, 0, 3, 2]] image = image_data_original[i] image = image / np.max(image) * 255 image = image.astype(np.uint8) y = plotter.package_data(truth_boxes[i][:, 1:], truth_boxes[i][:, 0].astype(np.int)) yhat = plotter.package_data(out_boxes2, out_classes.astype(np.int), out_scores) plotter.comparison(y, yhat, image) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data_original[i], out_boxes, out_classes, class_names, out_scores) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path, str(i) + '.png')) # To display (pauses the program): # plt.imshow(image_with_boxes, interpolation='nearest') # plt.show() input("Enter for next image")
def predict_draw(model_body, class_names, anchors, img_path, weights_name='trained_stage_3_best.h5', out_path="output_images", save_all=True): ''' Draw bounding boxes on image data ''' head, tail = ntpath.split(img_path) image = PIL.Image.open(img_path) resized_image = image.resize(tuple(image_shape), PIL.Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') image_data /= 255. image_data = np.expand_dims(image_data, 0) print(image_data.shape) # model.load_weights(weights_name) # print(image_data.shape) model_body.load_weights(weights_name) # Create output variables for prediction. yolo_outputs = yolo_head(model_body.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, max_boxes=3, score_threshold=.7, iou_threshold=0.05) # Run prediction on overfit image. sess = K.get_session() # TODO: Remove dependence on Tensorflow session. logging = TensorBoard() if not os.path.exists(out_path): os.makedirs(out_path) #for i in range(len(image_data)): out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ model_body.input: image_data, input_image_shape: [image_data.shape[1], image_data.shape[2]], K.learning_phase(): 0 }) print('Found {} boxes for image.'.format(len(out_boxes))) print(out_boxes) # Plot image with predicted boxes. image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes, class_names, out_scores, out_path + "\\" + tail) # Save the image: if save_all or (len(out_boxes) > 0): image = PIL.Image.fromarray(image_with_boxes) image.save(os.path.join(out_path, tail + '.png')) # To display (pauses the program): plt.imshow(image_with_boxes, interpolation='nearest') plt.show()