def detection(detection_graph, category_index, score, expand): print("Building Graph") # Session Config: allow seperate GPU/CPU adressing and limit memory allocation config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=log_device) config.gpu_options.allow_growth = allow_memory_growth cur_frames = 0 with detection_graph.as_default(): with tf.Session(graph=detection_graph, config=config) as sess: # Define Input and Ouput tensors image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0') detection_scores = detection_graph.get_tensor_by_name('detection_scores:0') detection_classes = detection_graph.get_tensor_by_name('detection_classes:0') num_detections = detection_graph.get_tensor_by_name('num_detections:0') if split_model: score_out = detection_graph.get_tensor_by_name('Postprocessor/convert_scores:0') expand_out = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1:0') score_in = detection_graph.get_tensor_by_name('Postprocessor/convert_scores_1:0') expand_in = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1_1:0') # Threading gpu_worker = SessionWorker("GPU", detection_graph, config) cpu_worker = SessionWorker("CPU", detection_graph, config) gpu_opts = [score_out, expand_out] cpu_opts = [detection_boxes, detection_scores, detection_classes, num_detections] gpu_counter = 0 cpu_counter = 0 # Start Video Stream and FPS calculation fps = FPS2(fps_interval).start() video_stream = WebcamVideoStream(video_input, width, height).start() cur_frames = 0 print("Press 'q' to Exit") print('Starting Detection') while video_stream.isActive(): # actual Detection if split_model: # split model in seperate gpu and cpu session threads if gpu_worker.is_sess_empty(): # read video frame, expand dimensions and convert to rgb image = video_stream.read() image_expanded = np.expand_dims(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), axis=0) # put new queue gpu_feeds = {image_tensor: image_expanded} if visualize: gpu_extras = image # for visualization frame else: gpu_extras = None gpu_worker.put_sess_queue(gpu_opts, gpu_feeds, gpu_extras) g = gpu_worker.get_result_queue() if g is None: # gpu thread has no output queue. ok skip, let's check cpu thread. gpu_counter += 1 else: # gpu thread has output queue. gpu_counter = 0 score, expand, image = g["results"][0], g["results"][1], g["extras"] if cpu_worker.is_sess_empty(): # When cpu thread has no next queue, put new queue. # else, drop gpu queue. cpu_feeds = {score_in: score, expand_in: expand} cpu_extras = image cpu_worker.put_sess_queue(cpu_opts, cpu_feeds, cpu_extras) c = cpu_worker.get_result_queue() if c is None: # cpu thread has no output queue. ok, nothing to do. continue cpu_counter += 1 time.sleep(0.005) continue # If CPU RESULT has not been set yet, no fps update else: cpu_counter = 0 boxes, scores, classes, num, image = c["results"][0], c["results"][1], c["results"][2], \ c["results"][3], c["extras"] else: # default session image = video_stream.read() image_expanded = np.expand_dims(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), axis=0) boxes, scores, classes, num = sess.run( [detection_boxes, detection_scores, detection_classes, num_detections], feed_dict={image_tensor: image_expanded}) # Visualization of the results of a detection. if visualize: vis_util.visualize_boxes_and_labels_on_image_array( image, np.squeeze(boxes), np.squeeze(classes).astype(np.int32), np.squeeze(scores), category_index, use_normalized_coordinates=True, line_thickness=8) if vis_text: cv2.putText(image, "fps: {}".format(fps.fps_local()), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2) cv2.imshow('object_detection', image) # Exit Option if cv2.waitKey(1) & 0xFF == ord('q'): break else: cur_frames += 1 # Exit after max frames if no visualization for box, score, _class in zip(np.squeeze(boxes), np.squeeze(scores), np.squeeze(classes)): if cur_frames % det_interval == 0 and score > det_th: label = category_index[_class]['name'] print("label: {}\nscore: {}\nbox: {}".format(label, score, box)) if cur_frames >= max_frames: break fps.update() # End everything gpu_worker.stop() cpu_worker.stop() fps.stop() video_stream.stop() cv2.destroyAllWindows() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand): print("Building Graph") # Session Config: allow seperate GPU/CPU adressing and limit memory allocation config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=log_device) config.gpu_options.allow_growth = allow_memory_growth cur_frames = 0 with detection_graph.as_default(): with tf.Session(graph=detection_graph, config=config) as sess: # Define Input and Ouput tensors image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') detection_boxes = detection_graph.get_tensor_by_name( 'detection_boxes:0') detection_scores = detection_graph.get_tensor_by_name( 'detection_scores:0') detection_classes = detection_graph.get_tensor_by_name( 'detection_classes:0') num_detections = detection_graph.get_tensor_by_name( 'num_detections:0') if split_model: score_out = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores:0') expand_out = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1:0') score_in = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores_1:0') expand_in = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1_1:0') # Threading gpu_worker = SessionWorker("GPU", detection_graph, config) cpu_worker = SessionWorker("CPU", detection_graph, config) gpu_opts = [score_out, expand_out] cpu_opts = [ detection_boxes, detection_scores, detection_classes, num_detections ] gpu_counter = 0 cpu_counter = 0 # Start Video Stream and FPS calculation fps = FPS2(fps_interval).start() #video_stream = WebcamVideoStream(video_input,width,height).start() cap = cv2.VideoCapture(video_input) cur_frames = 0 print("Press 'q' to Exit") print('Starting Detection') kcf = False box_to_color_map = collections.defaultdict(str) box_to_display_str_map = collections.defaultdict(list) tracker = KCF.kcftracker(False, True, False, False) count = 0 ret1, image = cap.read() im_height, im_width = image.shape[:2] im_height /= 3 im_width /= 3 print im_height, im_width fourcc = cv2.VideoWriter_fourcc(*'XVID') out = cv2.VideoWriter('output.avi', fourcc, 30.0, (im_width, im_height)) while True: count += 1 if count == 500: break if kcf and use_kcf: start = timer() ret1, image = cap.read() image = cv2.resize(image, (im_width, im_height)) image_expanded = np.expand_dims(cv2.cvtColor( image, cv2.COLOR_BGR2RGB), axis=0) for box, color in box_to_color_map.items(): label = box_to_display_str_map[box] ymin, xmin, ymax, xmax = box xmin = (int)(xmin * im_width) ymin = (int)(ymin * im_height) xmax = (int)(xmax * im_width) ymax = (int)(ymax * im_height) tracker.init([xmin, ymin, xmax - xmin, ymax - ymin], preframe) boundingbox = tracker.update(image) boundingbox = map(int, boundingbox) cv2.rectangle(image, (boundingbox[0], boundingbox[1]), (boundingbox[0] + boundingbox[2], boundingbox[1] + boundingbox[3]), (0, 255, 255), 2) cv2.putText( image, label[0], (boundingbox[0], boundingbox[1] + boundingbox[3]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (77, 255, 9), 2) #print 'hello' if vis_text: cv2.putText(image, "fps: {}".format(fps.fps_local()), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2) out.write(image) cv2.imshow('object_detection', image) cv2.waitKey(1) fps.update() kcf = False time_elapsed = timer() - start print( "Detection time in frame using KCF: {:.4f} sec fps {:.4f}" .format(time_elapsed, 1 / time_elapsed)) else: start = timer() kcf = True # actual Detection if split_model: # split model in seperate gpu and cpu session threads if gpu_worker.is_sess_empty(): # read video frame, expand dimensions and convert to rgb #image = video_stream.read() ret1, image = cap.read() preframe = image image_expanded = np.expand_dims(cv2.cvtColor( image, cv2.COLOR_BGR2RGB), axis=0) # put new queue gpu_feeds = {image_tensor: image_expanded} if visualize: gpu_extras = image # for visualization frame else: gpu_extras = None gpu_worker.put_sess_queue(gpu_opts, gpu_feeds, gpu_extras) g = gpu_worker.get_result_queue() if g is None: # gpu thread has no output queue. ok skip, let's check cpu thread. gpu_counter += 1 else: # gpu thread has output queue. gpu_counter = 0 score, expand, image = g["results"][0], g[ "results"][1], g["extras"] if cpu_worker.is_sess_empty(): # When cpu thread has no next queue, put new queue. # else, drop gpu queue. cpu_feeds = { score_in: score, expand_in: expand } cpu_extras = image cpu_worker.put_sess_queue( cpu_opts, cpu_feeds, cpu_extras) c = cpu_worker.get_result_queue() if c is None: # cpu thread has no output queue. ok, nothing to do. continue cpu_counter += 1 time.sleep(0.005) continue else: cpu_counter = 0 boxes, scores, classes, num, image = c["results"][ 0], c["results"][1], c["results"][2], c[ "results"][3], c["extras"] else: # default session #image = video_stream.read() ret1, image = cap.read() image = cv2.resize(image, (im_width, im_height)) preframe = image image_expanded = np.expand_dims(cv2.cvtColor( image, cv2.COLOR_BGR2RGB), axis=0) (boxes, scores, classes, num) = sess.run( [ detection_boxes, detection_scores, detection_classes, num_detections ], feed_dict={image_tensor: image_expanded}) # Visualization of the results of a detection. if visualize: box_to_color_map, box_to_display_str_map = vis_util.visualize_boxes_and_labels_on_image_array( image, np.squeeze(boxes), np.squeeze(classes).astype(np.int32), np.squeeze(scores), category_index, use_normalized_coordinates=True, line_thickness=8) for box, color in box_to_color_map.items(): label = box_to_display_str_map[box] ymin, xmin, ymax, xmax = box xmin = (int)(xmin * im_width) ymin = (int)(ymin * im_height) xmax = (int)(xmax * im_width) ymax = (int)(ymax * im_height) cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 255), 2) cv2.putText(image, label[0], (xmin, ymax), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (77, 255, 9), 2) if vis_text: cv2.putText(image, "fps: {}".format(fps.fps_local()), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2) out.write(image) cv2.imshow('object_detection', image) # Exit Option if cv2.waitKey(1) & 0xFF == ord('q'): break time_elapsed = timer() - start print( "Detection time in frame using SSD: {:.4f} sec fps {:.4f}" .format(time_elapsed, 1 / time_elapsed)) else: cur_frames += 1 # Exit after max frames if no visualization for box, score, _class in zip(np.squeeze(boxes), np.squeeze(scores), np.squeeze(classes)): if cur_frames % det_interval == 0 and score > det_th: label = category_index[_class]['name'] print("label: {}\nscore: {}\nbox: {}".format( label, score, box)) if cur_frames >= max_frames: break fps.update() # End everything if split_model: gpu_worker.stop() cpu_worker.stop() fps.stop() out.release() cap.release() #video_stream.stop() cv2.destroyAllWindows() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand): print("Building Graph") # Session Config: allow seperate GPU/CPU adressing and limit memory allocation config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=log_device) config.gpu_options.allow_growth = allow_memory_growth with detection_graph.as_default(): with tf.Session(graph=detection_graph, config=config) as sess: # Define Input and Ouput tensors image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') detection_boxes = detection_graph.get_tensor_by_name( 'detection_boxes:0') detection_scores = detection_graph.get_tensor_by_name( 'detection_scores:0') detection_classes = detection_graph.get_tensor_by_name( 'detection_classes:0') num_detections = detection_graph.get_tensor_by_name( 'num_detections:0') if split_model: score_out = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores:0') expand_out = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1:0') score_in = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores_1:0') expand_in = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1_1:0') # Threading gpu_worker = SessionWorker("GPU", detection_graph, config) cpu_worker = SessionWorker("CPU", detection_graph, config) gpu_opts = [score_out, expand_out] cpu_opts = [ detection_boxes, detection_scores, detection_classes, num_detections ] gpu_counter = 0 cpu_counter = 0 # Start Video Stream, FPS calculation and Tracker fps = FPS2(fps_interval).start() video_stream = WebcamVideoStream(video_input, width, height).start() #tracker = create_tracker(tracker_type) tracker = KCF.kcftracker(False, True, False, False) real_width = video_stream.real_width real_height = video_stream.real_height tracker_counter = 0 track = False print("Press 'q' to Exit") print('Starting Detection') while video_stream.isActive(): # Detection if not (use_tracker and track): if split_model: # split model in seperate gpu and cpu session threads if gpu_worker.is_sess_empty(): # read video frame, expand dimensions and convert to rgb frame = video_stream.read() frame_expanded = np.expand_dims(cv2.cvtColor( frame, cv2.COLOR_BGR2RGB), axis=0) # put new queue gpu_feeds = {image_tensor: frame_expanded} if visualize: gpu_extras = frame # for visualization frame else: gpu_extras = None gpu_worker.put_sess_queue(gpu_opts, gpu_feeds, gpu_extras) g = gpu_worker.get_result_queue() if g is None: # gpu thread has no output queue. ok skip, let's check cpu thread. gpu_counter += 1 else: # gpu thread has output queue. gpu_counter = 0 score, expand, frame = g["results"][0], g[ "results"][1], g["extras"] if cpu_worker.is_sess_empty(): # When cpu thread has no next queue, put new queue. # else, drop gpu queue. cpu_feeds = { score_in: score, expand_in: expand } cpu_extras = frame cpu_worker.put_sess_queue( cpu_opts, cpu_feeds, cpu_extras) c = cpu_worker.get_result_queue() if c is None: # cpu thread has no output queue. ok, nothing to do. continue cpu_counter += 1 time.sleep(0.005) continue # If CPU RESULT has not been set yet, no fps update else: cpu_counter = 0 boxes, scores, classes, num, frame = c["results"][ 0], c["results"][1], c["results"][2], c[ "results"][3], c["extras"] else: # default session frame = video_stream.read() frame_expanded = np.expand_dims(cv2.cvtColor( frame, cv2.COLOR_BGR2RGB), axis=0) (boxes, scores, classes, num) = sess.run( [ detection_boxes, detection_scores, detection_classes, num_detections ], feed_dict={image_tensor: frame_expanded}) # reformat detection num = int(num) boxes = np.squeeze(boxes) classes = np.squeeze(classes).astype(np.int32) scores = np.squeeze(scores) # visualize detection vis = visualize_detection(frame, boxes, classes, scores, category_index, fps) if not vis: break # Activate Tracker if use_tracker and num <= num_trackers: tracker_frame = frame track = True first_track = True # Tracking else: frame = video_stream.read() if first_track: trackers = [] tracker_boxes = boxes for box in boxes[~np.all(boxes == 0, axis=1)]: tracker.init( conv_detect2track(box, real_width, real_height), tracker_frame) trackers.append(tracker) first_track = False #print ("A: {}".format(boxes[~np.all(boxes == 0, axis=1)])) i = 0 for tracker in trackers: tracker_box = tracker.update(frame) #print ("B: {}".format(tracker_box)) tracker_boxes[i, :] = conv_track2detect( tracker_box, real_width, real_height) i += 1 #p1 = (tracker_box[0], tracker_box[1]) #p2 = (tracker_box[0] + tracker_box[2], tracker_box[1] + tracker_box[3]) #cv2.rectangle(frame, p1, p2, (255,0,0), 2) #cv2.imshow('object_detection', frame) #print ("C: {}".format(tracker_boxes[~np.all(tracker_boxes == 0, axis=1)])) vis = visualize_detection(frame, tracker_boxes, classes, scores, category_index, fps) if not vis: break tracker_counter += 1 #tracker_frame = frame if tracker_counter >= tracker_frames: track = False tracker_counter = 0 fps.update() # End everything if split_model: gpu_worker.stop() cpu_worker.stop() fps.stop() video_stream.stop() cv2.destroyAllWindows() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(model): # Tracker if USE_TRACKER: import sys sys.path.append(os.getcwd() + '/stuff/kcf') import KCF tracker = KCF.kcftracker(False, True, False, False) tracker_counter = 0 track = False print("> Building Graph") # tf Session Config config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True detection_graph = model.detection_graph category_index = model.category_index with detection_graph.as_default(): with tf.Session(graph=detection_graph, config=config) as sess: # start Videostream vs = WebcamVideoStream(VIDEO_INPUT, WIDTH, HEIGHT).start() # Define Input and Ouput tensors tensor_dict = model.get_tensordict([ 'num_detections', 'detection_boxes', 'detection_scores', 'detection_classes', 'detection_masks' ]) image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') # Mask Transformations if 'detection_masks' in tensor_dict: # Reframe is required to translate mask from box coordinates to image coordinates and fit the image size. detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0]) detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0]) real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32) detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1]) detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1]) detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, vs.real_height, vs.real_width) detection_masks_reframed = tf.cast( tf.greater(detection_masks_reframed, 0.5), tf.uint8) # Follow the convention by adding back the batch dimension tensor_dict['detection_masks'] = tf.expand_dims( detection_masks_reframed, 0) if SPLIT_MODEL: score_out = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores:0') expand_out = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1:0') score_in = detection_graph.get_tensor_by_name( 'Postprocessor/convert_scores_1:0') expand_in = detection_graph.get_tensor_by_name( 'Postprocessor/ExpandDims_1_1:0') # Threading score = model.score expand = model.expand gpu_worker = SessionWorker("GPU", detection_graph, config) cpu_worker = SessionWorker("CPU", detection_graph, config) gpu_opts = [score_out, expand_out] cpu_opts = [ tensor_dict['detection_boxes'], tensor_dict['detection_scores'], tensor_dict['detection_classes'], tensor_dict['num_detections'] ] gpu_counter = 0 cpu_counter = 0 fps = FPS(FPS_INTERVAL).start() print('> Starting Detection') while vs.isActive(): # Detection if not (USE_TRACKER and track): if SPLIT_MODEL: # split model in seperate gpu and cpu session threads masks = None # No Mask Detection possible yet if gpu_worker.is_sess_empty(): # read video frame, expand dimensions and convert to rgb frame = vs.read() # put new queue gpu_feeds = {image_tensor: vs.expanded()} if VISUALIZE: gpu_extras = frame # for visualization frame else: gpu_extras = None gpu_worker.put_sess_queue(gpu_opts, gpu_feeds, gpu_extras) g = gpu_worker.get_result_queue() if g is None: # gpu thread has no output queue. ok skip, let's check cpu thread. gpu_counter += 1 else: # gpu thread has output queue. gpu_counter = 0 score, expand, frame = g["results"][0], g[ "results"][1], g["extras"] if cpu_worker.is_sess_empty(): # When cpu thread has no next queue, put new queue. # else, drop gpu queue. cpu_feeds = { score_in: score, expand_in: expand } cpu_extras = frame cpu_worker.put_sess_queue( cpu_opts, cpu_feeds, cpu_extras) c = cpu_worker.get_result_queue() if c is None: # cpu thread has no output queue. ok, nothing to do. continue cpu_counter += 1 continue # If CPU RESULT has not been set yet, no fps update else: cpu_counter = 0 boxes, scores, classes, num, frame = c["results"][ 0], c["results"][1], c["results"][2], c[ "results"][3], c["extras"] else: # default session frame = vs.read() output_dict = sess.run( tensor_dict, feed_dict={image_tensor: vs.expanded()}) num = output_dict['num_detections'][0] classes = output_dict['detection_classes'][0] boxes = output_dict['detection_boxes'][0] scores = output_dict['detection_scores'][0] if 'detection_masks' in output_dict: masks = output_dict['detection_masks'][0] else: masks = None # reformat detection num = int(num) boxes = np.squeeze(boxes) classes = np.squeeze(classes).astype(np.uint8) scores = np.squeeze(scores) # Visualization vis = vis_detection(frame, VISUALIZE, boxes, classes, scores, masks, category_index, DET_INTERVAL, DET_TH, MAX_FRAMES, fps) if not vis: break # Activate Tracker if USE_TRACKER and num <= NUM_TRACKERS: tracker_frame = frame track = True first_track = True # Tracking else: frame = vs.read() if first_track: trackers = [] tracker_boxes = boxes for box in boxes[~np.all(boxes == 0, axis=1)]: tracker.init( conv_detect2track(box, vs.real_width, vs.real_height), tracker_frame) trackers.append(tracker) first_track = False for idx, tracker in enumerate(trackers): tracker_box = tracker.update(frame) tracker_boxes[idx, :] = conv_track2detect( tracker_box, vs.real_width, vs.real_height) vis = vis_detection(frame, VISUALIZE, tracker_boxes, classes, scores, masks, category_index, DET_INTERVAL, DET_TH, MAX_FRAMES, fps) if not vis: break tracker_counter += 1 #tracker_frame = frame if tracker_counter >= TRACKER_FRAMES: track = False tracker_counter = 0 fps.update() # End everything vs.stop() fps.stop() if SPLIT_MODEL: gpu_worker.stop() cpu_worker.stop()