def segmentation(detection_graph):
vs = WebcamVideoStream(0, 1280, 720).start()
resize_ratio = 1.0 * 513 / max(vs.real_width, vs.real_height)
target_size = (int(resize_ratio * vs.real_width),
int(resize_ratio * vs.real_height))
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
fps = FPS2(5).start()
# background_image = cv2.imread('b.jpg')
# resized_background_image = cv2.resize(background_image, target_size) # (384,513)
print("Starting...")
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
while vs.isActive():
image = cv2.resize(vs.read(), target_size)
batch_seg_map = sess.run('SemanticPredictions:0',
feed_dict={'ImageTensor:0': [cv2.cvtColor(image, cv2.COLOR_BGR2RGB)]})
# visualization
seg_map = batch_seg_map[0]
seg_map[seg_map != 15] = 0
# bg_copy = resized_background_image.copy()
mask = (seg_map == 15)
# car=render(image)
car = cv2.stylization(image, sigma_s=60, sigma_r=0.07)
# gray0 = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
# gray0 = cv2.cvtColor(gray0, cv2.COLOR_GRAY2RGB)
print(car.shape)
car[mask] = image[mask]
# create_colormap(seg_map).astype(np.uint8)
seg_image = np.stack(
(seg_map, seg_map, seg_map), axis=-1).astype(np.uint8)
gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)[1]
cnts, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
try:
cv2.drawContours(
car, cnts, -1, (randint(0, 255), randint(0, 255), randint(0, 255)), 2)
except:
pass
# ir=cv2.resize(car,(vs.real_width,vs.real_height))
ir = car
cv2.imshow('segmentation', ir)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
fps.update()
fps.stop()
vs.stop()
cv2.destroyAllWindows()
def segmentation(model):
detection_graph = model.detection_graph
# fixed input sizes as model needs resize either way
vs = WebcamVideoStream(VIDEO_INPUT, 640, 480).start()
resize_ratio = 1.0 * 513 / max(vs.real_width, vs.real_height)
target_size = (int(resize_ratio * vs.real_width),
int(resize_ratio * vs.real_height))
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
fps = FPS(FPS_INTERVAL).start()
print("> Starting Segmentaion")
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
while vs.isActive():
frame = vs.resized(target_size)
batch_seg_map = sess.run(
'SemanticPredictions:0',
feed_dict={
'ImageTensor:0':
[cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)]
})
# visualization
if VISUALIZE:
seg_map = batch_seg_map[0]
seg_image = create_colormap(seg_map).astype(np.uint8)
cv2.addWeighted(seg_image, ALPHA, frame, 1 - ALPHA, 0,
frame)
vis_text(frame, "fps: {}".format(fps.fps_local()),
(10, 30))
# boxes (ymin, xmin, ymax, xmax)
if BBOX:
map_labeled = measure.label(seg_map, connectivity=1)
for region in measure.regionprops(map_labeled):
if region.area > MINAREA:
box = region.bbox
p1 = (box[1], box[0])
p2 = (box[3], box[2])
cv2.rectangle(frame, p1, p2, (77, 255, 9), 2)
vis_text(
frame, LABEL_NAMES[seg_map[tuple(
region.coords[0])]],
(p1[0], p1[1] - 10))
cv2.imshow('segmentation', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
fps.update()
fps.stop()
vs.stop()
def segmentation(detection_graph,label_names):
# fixed input sizes as model needs resize either way
vs = WebcamVideoStream(VIDEO_INPUT,640,480).start()
resize_ratio = 1.0 * 513 / max(vs.real_width,vs.real_height)
target_size = (int(resize_ratio * vs.real_width), int(resize_ratio * vs.real_height))
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth=True
fps = FPS2(FPS_INTERVAL).start()
print("> Starting Segmentaion")
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
while vs.isActive():
image = cv2.resize(vs.read(),target_size)
batch_seg_map = sess.run('SemanticPredictions:0',
feed_dict={'ImageTensor:0': [cv2.cvtColor(image, cv2.COLOR_BGR2RGB)]})
# visualization
seg_map = batch_seg_map[0]
seg_image = create_colormap(seg_map).astype(np.uint8)
cv2.addWeighted(seg_image,ALPHA,image,1-ALPHA,0,image)
vis_text(image,"fps: {}".format(fps.fps_local()),(10,30))
# boxes (ymin, xmin, ymax, xmax)
if BBOX:
map_labeled = measure.label(seg_map, connectivity=1)
for region in measure.regionprops(map_labeled):
if region.area > MINAREA:
box = region.bbox
p1 = (box[1], box[0])
p2 = (box[3], box[2])
cv2.rectangle(image, p1, p2, (77,255,9), 2)
vis_text(image,label_names[seg_map[tuple(region.coords[0])]],(p1[0],p1[1]-10))
cv2.imshow('segmentation',image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
fps.update()
fps.stop()
vs.stop()
cv2.destroyAllWindows()
def object_detection(video_input,visualize,max_frames,width,height,fps_interval,bbox_thickness, \
allow_memory_growth,det_intervall,det_th,model_name):
config = tf.ConfigProto()
config.gpu_options.allow_growth = allow_memory_growth
cur_frames = 0
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Definite input and output Tensors for detection_graph
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Each box represents a part of the image where a particular object was detected.
detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
# Each score represent how level of confidence for each of the objects.
# Score is shown on the result image, together with the class label.
detection_scores = detection_graph.get_tensor_by_name(
'detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
# fps calculation
fps = FPS2(fps_interval).start()
# Start Video Stream
video_stream = WebcamVideoStream(video_input, width,
height).start()
print("Press 'q' to Exit")
while video_stream.isActive():
image_np = video_stream.read()
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(boxes, scores, classes,
num) = sess.run([
detection_boxes, detection_scores, detection_classes,
num_detections
],
feed_dict={image_tensor: image_np_expanded})
if visualize:
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
CATEGORY_INDEX,
min_score_thresh=MINIMUM_CONFIDENCE,
use_normalized_coordinates=True,
line_thickness=bbox_thickness)
cv2.imshow('object_detection', image_np)
# print(boxes)
# print bounding corners of boxes when confidence is > minimum confidence (the ones you're drawing boxes around)
# print("NEW FRAME")
for i, box in enumerate(np.squeeze(boxes)):
if (np.squeeze(scores)[i] > MINIMUM_CONFIDENCE):
# This uses actual coordinates based on size of image - remove height and width to use normalized coordinates
ymin = box[0] * height
xmin = box[1] * width
ymax = box[2] * height
xmax = box[3] * width
#normalized/percentage
nymin = box[0] * 100
nxmin = box[1] * 100
nymax = box[2] * 100
nxmax = box[3] * 100
#TODO: should pass through image size at the very beginning
print('Top left')
print("(" + str(nxmin) + "%," + str(nymin) + "%)")
print('Bottom right')
print("(" + str(nxmax) + "%," + str(nymax) + "%)")
print()
# Exit Option--BROKEN
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
cur_frames += 1
for box, score, _class in zip(np.squeeze(boxes),
np.squeeze(scores),
np.squeeze(classes)):
if cur_frames % det_intervall == 0 and score > det_th:
label = category_index[_class]['name']
print(label, score, box)
if cur_frames >= max_frames:
break
# fps calculation
fps.update()
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand):
print("Building Graph")
# Session Config: allow seperate GPU/CPU adressing and limit memory allocation
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=log_device)
config.gpu_options.allow_growth = allow_memory_growth
cur_frames = 0
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Define Input and Ouput tensors
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
if split_model:
score_out = detection_graph.get_tensor_by_name('Postprocessor/convert_scores:0')
expand_out = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1:0')
score_in = detection_graph.get_tensor_by_name('Postprocessor/convert_scores_1:0')
expand_in = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1_1:0')
# Threading
gpu_worker = SessionWorker("GPU", detection_graph, config)
cpu_worker = SessionWorker("CPU", detection_graph, config)
gpu_opts = [score_out, expand_out]
cpu_opts = [detection_boxes, detection_scores, detection_classes, num_detections]
gpu_counter = 0
cpu_counter = 0
# Start Video Stream and FPS calculation
fps = FPS2(fps_interval).start()
video_stream = WebcamVideoStream(video_input, width, height).start()
cur_frames = 0
print("Press 'q' to Exit")
print('Starting Detection')
while video_stream.isActive():
# actual Detection
if split_model:
# split model in seperate gpu and cpu session threads
if gpu_worker.is_sess_empty():
# read video frame, expand dimensions and convert to rgb
image = video_stream.read()
image_expanded = np.expand_dims(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), axis=0)
# put new queue
gpu_feeds = {image_tensor: image_expanded}
if visualize:
gpu_extras = image # for visualization frame
else:
gpu_extras = None
gpu_worker.put_sess_queue(gpu_opts, gpu_feeds, gpu_extras)
g = gpu_worker.get_result_queue()
if g is None:
# gpu thread has no output queue. ok skip, let's check cpu thread.
gpu_counter += 1
else:
# gpu thread has output queue.
gpu_counter = 0
score, expand, image = g["results"][0], g["results"][1], g["extras"]
if cpu_worker.is_sess_empty():
# When cpu thread has no next queue, put new queue.
# else, drop gpu queue.
cpu_feeds = {score_in: score, expand_in: expand}
cpu_extras = image
cpu_worker.put_sess_queue(cpu_opts, cpu_feeds, cpu_extras)
c = cpu_worker.get_result_queue()
if c is None:
# cpu thread has no output queue. ok, nothing to do. continue
cpu_counter += 1
time.sleep(0.005)
continue # If CPU RESULT has not been set yet, no fps update
else:
cpu_counter = 0
boxes, scores, classes, num, image = c["results"][0], c["results"][1], c["results"][2], \
c["results"][3], c["extras"]
else:
# default session
image = video_stream.read()
image_expanded = np.expand_dims(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), axis=0)
boxes, scores, classes, num = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_expanded})
# Visualization of the results of a detection.
if visualize:
vis_util.visualize_boxes_and_labels_on_image_array(
image,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
category_index,
use_normalized_coordinates=True,
line_thickness=8)
if vis_text:
cv2.putText(image, "fps: {}".format(fps.fps_local()), (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2)
cv2.imshow('object_detection', image)
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
cur_frames += 1
# Exit after max frames if no visualization
for box, score, _class in zip(np.squeeze(boxes), np.squeeze(scores), np.squeeze(classes)):
if cur_frames % det_interval == 0 and score > det_th:
label = category_index[_class]['name']
print("label: {}\nscore: {}\nbox: {}".format(label, score, box))
if cur_frames >= max_frames:
break
fps.update()
# End everything
gpu_worker.stop()
cpu_worker.stop()
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand):
outputs = [
'num_detections', 'detection_boxes', 'detection_scores',
'detection_classes', 'detection_masks'
]
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=log_device)
config.gpu_options.allow_growth = allow_memory_growth
cur_frames = 0
print('Starting detection')
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Start Video Stream
video_stream = WebcamVideoStream(video_input, width,
height).start()
print("Press 'q' to Exit")
# Get handles to input and output tensors
tensor_dict = get_tensordict(detection_graph, outputs)
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
if 'detection_masks' in tensor_dict:
#real_width, real_height = get_image_shape()
# Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
detection_boxes = tf.squeeze(tensor_dict['detection_boxes'],
[0])
detection_masks = tf.squeeze(tensor_dict['detection_masks'],
[0])
real_num_detection = tf.cast(tensor_dict['num_detections'][0],
tf.int32)
detection_boxes = tf.slice(detection_boxes, [0, 0],
[real_num_detection, -1])
detection_masks = tf.slice(detection_masks, [0, 0, 0],
[real_num_detection, -1, -1])
detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, video_stream.real_height,
video_stream.real_width)
detection_masks_reframed = tf.cast(
tf.greater(detection_masks_reframed, 0.5), tf.uint8)
# Follow the convention by adding back the batch dimension
tensor_dict['detection_masks'] = tf.expand_dims(
detection_masks_reframed, 0)
if split_model:
score_out = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores:0')
expand_out = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1:0')
score_in = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores_1:0')
expand_in = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1_1:0')
# fps calculation
fps = FPS2(fps_interval).start()
cur_frames = 0
while video_stream.isActive():
image = video_stream.read()
fps.update()
# read video frame and expand dimensions
if convert_rgb:
try:
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
cvt = True
except:
print("Error converting BGR2RGB")
cvt = False
# detection
if split_model:
(score, expand) = sess.run(
[score_out, expand_out],
feed_dict={image_tensor: np.expand_dims(image, 0)})
output_dict = sess.run(tensor_dict,
feed_dict={
score_in: score,
expand_in: expand
})
else:
output_dict = sess.run(
tensor_dict,
feed_dict={image_tensor: np.expand_dims(image, 0)})
#num = int(output_dict['num_detections'][0])
classes = output_dict['detection_classes'][0].astype(np.uint8)
boxes = output_dict['detection_boxes'][0]
scores = output_dict['detection_scores'][0]
if 'detection_masks' in output_dict:
output_dict['detection_masks'] = output_dict[
'detection_masks'][0]
# Visualization of the results of a detection.
if visualize:
if convert_rgb and cvt:
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
vis_util.visualize_boxes_and_labels_on_image_array(
image,
boxes,
classes,
scores,
category_index,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
line_thickness=8)
if vis_text:
cv2.putText(image, "fps: {}".format(fps.fps_local()),
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,
(77, 255, 9), 2)
cv2.imshow('object_detection', image)
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
# Exit after max frames if no visualization
cur_frames += 1
for box, score, _class in zip(boxes, scores, classes):
if cur_frames % det_interval == 0 and score > det_th:
label = category_index[_class]['name']
print("label: {}\nscore: {}\nbox: {}".format(
label, score, box))
if cur_frames >= max_frames:
break
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def segmentation(detection_graph):
vs = WebcamVideoStream(0, 640, 480).start()
resize_ratio = 1.0 * 513 / max(vs.real_width, vs.real_height)
target_size = (int(resize_ratio * vs.real_width),
int(resize_ratio * vs.real_height))
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
fps = FPS2(5).start()
filelist = [
file for file in os.listdir('backgrounds') if file.endswith('.jpg')
]
num_files = len(filelist)
background_image = []
resized_background_image = []
for x in filelist:
background_image.append(cv2.imread(x))
resized_background_image.append(
cv2.resize(background_image[-1], target_size))
# fff = 0
# background_image = cv2.imread('b.jpg')
# resized_background_image = cv2.resize(
# background_image, target_size) # (384,513)
# background_image2 = cv2.imread('b2.jpg')
# resized_background_image2 = cv2.resize(
# background_image2, target_size) # (384,513)
# background_image3 = cv2.imread('b3.jpg')
# resized_background_image3 = cv2.resize(
# background_image3, target_size) # (384,513)
mike_background_image = cv2.imread('mike.png')
mike_background_image = cv2.resize(
mike_background_image,
(int(resize_ratio * vs.real_width / 3),
int(resize_ratio * vs.real_height * 3 / 4))) # (384,513)
# Uncomment to save output
# out = cv2.VideoWriter('outpy.avi', cv2.VideoWriter_fourcc(
# 'M', 'J', 'P', 'G'), 1, (vs.real_height, vs.real_width))#CHANGE
print("Starting...")
cv2.namedWindow(
'segmentation',
16) # 16 means WINDOW_GUI_NORMAL, to disable right click context menu
cv2.setMouseCallback('segmentation', next_bg)
global img_num, mike_flag
img_num = 0
mike_flag = False
with detection_graph.as_default():
with tf.Session(graph=detection_graph) as sess:
while vs.isActive():
image = cv2.resize(vs.read(), target_size)
batch_seg_map = sess.run(
'SemanticPredictions:0',
feed_dict={
'ImageTensor:0':
[cv2.cvtColor(image, cv2.COLOR_BGR2RGB)]
})
# visualization
seg_map = batch_seg_map[0]
seg_map[seg_map != 15] = 0
bg_copy = resized_background_image[img_num % num_files].copy()
# if fff == 0:
# bg_copy = resized_background_image.copy()
# elif fff == 1:
# bg_copy = resized_background_image2.copy()
# elif fff == 2:
# bg_copy = resized_background_image3.copy()
mask = (seg_map == 15)
bg_copy[mask] = image[mask]
# create_colormap(seg_map).astype(np.uint8)
seg_image = np.stack((seg_map, seg_map, seg_map),
axis=-1).astype(np.uint8)
gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)[1]
_, cnts, hierarchy = cv2.findContours(thresh.copy(),
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
try:
cv2.drawContours(
bg_copy, cnts, -1,
(randint(0, 255), randint(0, 255), randint(0, 255)), 2)
except:
pass
if mike_flag:
x_offset = 150
y_offset = 95
bg_copy[y_offset:y_offset + mike_background_image.shape[0],
x_offset:x_offset +
mike_background_image.shape[1]][
mike_background_image !=
0] = mike_background_image[
mike_background_image != 0]
ir = cv2.resize(bg_copy, (vs.real_width, vs.real_height))
cv2.imshow('segmentation', ir)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
elif cv2.waitKey(1) & 0xFF == ord('a'):
fff = 0
elif cv2.waitKey(1) & 0xFF == ord('b'):
fff = 1
elif cv2.waitKey(1) & 0xFF == ord('c'):
fff = 2
fps.update()
# out.write(ir)
fps.stop()
vs.stop()
# out.release()
cv2.destroyAllWindows()
(boxes, scores, classes, num) = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
if visualize:
for i in range(batch_size):
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image_np_expanded[i],
boxes[i],
classes[i].astype(np.int32),
scores[i],
category_index,
use_normalized_coordinates=True,
line_thickness=bbox_thickness)
cv2.imshow('object_detection', image_np_expanded[i])
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
cur_frames += batch_size
if cur_frames >= max_frames:
break
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand):
print("Building Graph")
# Session Config: allow seperate GPU/CPU adressing and limit memory allocation
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=log_device)
config.gpu_options.allow_growth = allow_memory_growth
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Define Input and Ouput tensors
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name(
'detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
if split_model:
score_out = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores:0')
expand_out = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1:0')
score_in = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores_1:0')
expand_in = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1_1:0')
# Threading
gpu_worker = SessionWorker("GPU", detection_graph, config)
cpu_worker = SessionWorker("CPU", detection_graph, config)
gpu_opts = [score_out, expand_out]
cpu_opts = [
detection_boxes, detection_scores, detection_classes,
num_detections
]
gpu_counter = 0
cpu_counter = 0
# Start Video Stream, FPS calculation and Tracker
fps = FPS2(fps_interval).start()
video_stream = WebcamVideoStream(video_input, width,
height).start()
#tracker = create_tracker(tracker_type)
tracker = KCF.kcftracker(False, True, False, False)
real_width = video_stream.real_width
real_height = video_stream.real_height
tracker_counter = 0
track = False
print("Press 'q' to Exit")
print('Starting Detection')
while video_stream.isActive():
# Detection
if not (use_tracker and track):
if split_model:
# split model in seperate gpu and cpu session threads
if gpu_worker.is_sess_empty():
# read video frame, expand dimensions and convert to rgb
frame = video_stream.read()
frame_expanded = np.expand_dims(cv2.cvtColor(
frame, cv2.COLOR_BGR2RGB),
axis=0)
# put new queue
gpu_feeds = {image_tensor: frame_expanded}
if visualize:
gpu_extras = frame # for visualization frame
else:
gpu_extras = None
gpu_worker.put_sess_queue(gpu_opts, gpu_feeds,
gpu_extras)
g = gpu_worker.get_result_queue()
if g is None:
# gpu thread has no output queue. ok skip, let's check cpu thread.
gpu_counter += 1
else:
# gpu thread has output queue.
gpu_counter = 0
score, expand, frame = g["results"][0], g[
"results"][1], g["extras"]
if cpu_worker.is_sess_empty():
# When cpu thread has no next queue, put new queue.
# else, drop gpu queue.
cpu_feeds = {
score_in: score,
expand_in: expand
}
cpu_extras = frame
cpu_worker.put_sess_queue(
cpu_opts, cpu_feeds, cpu_extras)
c = cpu_worker.get_result_queue()
if c is None:
# cpu thread has no output queue. ok, nothing to do. continue
cpu_counter += 1
time.sleep(0.005)
continue # If CPU RESULT has not been set yet, no fps update
else:
cpu_counter = 0
boxes, scores, classes, num, frame = c["results"][
0], c["results"][1], c["results"][2], c[
"results"][3], c["extras"]
else:
# default session
frame = video_stream.read()
frame_expanded = np.expand_dims(cv2.cvtColor(
frame, cv2.COLOR_BGR2RGB),
axis=0)
(boxes, scores, classes, num) = sess.run(
[
detection_boxes, detection_scores,
detection_classes, num_detections
],
feed_dict={image_tensor: frame_expanded})
# reformat detection
num = int(num)
boxes = np.squeeze(boxes)
classes = np.squeeze(classes).astype(np.int32)
scores = np.squeeze(scores)
# visualize detection
vis = visualize_detection(frame, boxes, classes, scores,
category_index, fps)
if not vis:
break
# Activate Tracker
if use_tracker and num <= num_trackers:
tracker_frame = frame
track = True
first_track = True
# Tracking
else:
frame = video_stream.read()
if first_track:
trackers = []
tracker_boxes = boxes
for box in boxes[~np.all(boxes == 0, axis=1)]:
tracker.init(
conv_detect2track(box, real_width,
real_height), tracker_frame)
trackers.append(tracker)
first_track = False
#print ("A: {}".format(boxes[~np.all(boxes == 0, axis=1)]))
i = 0
for tracker in trackers:
tracker_box = tracker.update(frame)
#print ("B: {}".format(tracker_box))
tracker_boxes[i, :] = conv_track2detect(
tracker_box, real_width, real_height)
i += 1
#p1 = (tracker_box[0], tracker_box[1])
#p2 = (tracker_box[0] + tracker_box[2], tracker_box[1] + tracker_box[3])
#cv2.rectangle(frame, p1, p2, (255,0,0), 2)
#cv2.imshow('object_detection', frame)
#print ("C: {}".format(tracker_boxes[~np.all(tracker_boxes == 0, axis=1)]))
vis = visualize_detection(frame, tracker_boxes, classes,
scores, category_index, fps)
if not vis:
break
tracker_counter += 1
#tracker_frame = frame
if tracker_counter >= tracker_frames:
track = False
tracker_counter = 0
fps.update()
# End everything
if split_model:
gpu_worker.stop()
cpu_worker.stop()
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph,
category_index,
video_input=0,
visualize=True,
max_frames=500,
width=300,
height=300,
fps_interval=3,
allow_memory_growth=True,
det_intervall=75,
det_th=0.5):
# Session Config: Limit GPU Memory Usage
config = tf.ConfigProto()
config.gpu_options.allow_growth = allow_memory_growth
cur_frames = 0
# Detection
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Definite input and output Tensors for detection_graph
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Each box represents a part of the image where a particular object was detected.
detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
# Each score represent how level of confidence for each of the objects.
# Score is shown on the result image, together with the class label.
detection_scores = detection_graph.get_tensor_by_name(
'detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
# fps calculation
fps = FPS2(fps_interval).start()
# Start Video Stream
video_stream = WebcamVideoStream(video_input, width,
height).start()
print("Press 'q' to Exit")
while video_stream.isActive():
image_np = video_stream.read()
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(boxes, scores, classes,
num) = sess.run([
detection_boxes, detection_scores, detection_classes,
num_detections
],
feed_dict={image_tensor: image_np_expanded})
if visualize:
# Visualization of the results of a detection.
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
category_index,
use_normalized_coordinates=True,
line_thickness=8)
cv2.imshow('object_detection', image_np)
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
cur_frames += 1
for box, score, _class in zip(np.squeeze(boxes),
np.squeeze(scores),
np.squeeze(classes)):
if cur_frames % det_intervall == 0 and score > det_th:
label = category_index[_class]['name']
print(label, score, box)
if cur_frames >= max_frames:
break
# fps calculation
fps.update()
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index, score, expand):
print("Building Graph")
# Session Config: allow seperate GPU/CPU adressing and limit memory allocation
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=log_device)
config.gpu_options.allow_growth=allow_memory_growth
cur_frames = 0
with detection_graph.as_default():
with tf.Session(graph=detection_graph,config=config) as sess:
# Define Input and Ouput tensors
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
if split_model:
score_out = detection_graph.get_tensor_by_name('Postprocessor/convert_scores:0')
expand_out = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1:0')
score_in = detection_graph.get_tensor_by_name('Postprocessor/convert_scores_1:0')
expand_in = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1_1:0')
# Start Video Stream and FPS calculation
fps = FPS2(fps_interval).start()
video_stream = WebcamVideoStream(video_input,width,height).start()
cur_frames = 0
print ("Press 'q' to Exit")
print('Starting Detection')
while video_stream.isActive():
# read video frame, convert color and expand dimensions
image = video_stream.read()
fps.update()
if convert_rgb:
try:
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
cvt = True
except:
print("Error converting BGR2RGB")
cvt = False
image_expanded = np.expand_dims(image, axis=0)
# actual Detection
if split_model:
# Split Detection in two sessions.
(score, expand) = sess.run([score_out, expand_out], feed_dict={image_tensor: image_expanded})
(boxes, scores, classes, num) = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={score_in:score, expand_in: expand})
else:
# default session
(boxes, scores, classes, num) = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_expanded})
# Visualization of the results of a detection.
if visualize:
if convert_rgb and cvt:
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
vis_util.visualize_boxes_and_labels_on_image_array(
image,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
category_index,
use_normalized_coordinates=True,
line_thickness=8)
if vis_text:
cv2.putText(image,"fps: {}".format(fps.fps_local()), (10,30),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2)
cv2.imshow('object_detection', image)
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
# Exit after max frames if no visualization
cur_frames += 1
for box, score, _class in zip(np.squeeze(boxes), np.squeeze(scores), np.squeeze(classes)):
if cur_frames%det_interval==0 and score > det_th:
label = category_index[_class]['name']
print("label: {}\nscore: {}\nbox: {}".format(label, score, box))
if cur_frames >= max_frames:
break
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(detection_graph, category_index):
# Session Config: allow seperate GPU/CPU adressing and limit memory allocation
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = allow_memory_growth
cur_frames = 0
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# Define Input and Ouput tensors
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name(
'detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
# Start Video Stream
video_stream = WebcamVideoStream(video_input, width,
height).start()
print("Press 'q' to Exit")
# fps calculation
fps = FPS2(fps_interval).start()
while video_stream.isActive():
image_np = video_stream.read()
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
# Actual detection.
(boxes, scores, classes,
num) = sess.run([
detection_boxes, detection_scores, detection_classes,
num_detections
],
feed_dict={image_tensor: image_np_expanded})
# Visualization of the results of a detection.
if visualize:
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(boxes),
np.squeeze(classes).astype(np.int32),
np.squeeze(scores),
category_index,
use_normalized_coordinates=True,
line_thickness=8)
# print fps on visualization screen
if vis_text:
cv2.putText(image_np,
"fps: {}".format(fps.fps_local()),
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75,
(77, 255, 9), 2)
cv2.imshow('object_detection', image_np)
# Exit Option
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
# Exit after max frames if no visualization
cur_frames += 1
for box, score, _class in zip(np.squeeze(boxes),
np.squeeze(scores),
np.squeeze(classes)):
if cur_frames % det_interval == 0 and score > det_th:
label = category_index[_class]['name']
print(label, score, box)
if cur_frames >= max_frames:
break
fps.update()
# End everything
fps.stop()
video_stream.stop()
cv2.destroyAllWindows()
print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))
print('[INFO] approx. FPS: {:.2f}'.format(fps.fps()))
def detection(model):
# Tracker
if USE_TRACKER:
import sys
sys.path.append(os.getcwd() + '/stuff/kcf')
import KCF
tracker = KCF.kcftracker(False, True, False, False)
tracker_counter = 0
track = False
print("> Building Graph")
# tf Session Config
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
detection_graph = model.detection_graph
category_index = model.category_index
with detection_graph.as_default():
with tf.Session(graph=detection_graph, config=config) as sess:
# start Videostream
vs = WebcamVideoStream(VIDEO_INPUT, WIDTH, HEIGHT).start()
# Define Input and Ouput tensors
tensor_dict = model.get_tensordict([
'num_detections', 'detection_boxes', 'detection_scores',
'detection_classes', 'detection_masks'
])
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Mask Transformations
if 'detection_masks' in tensor_dict:
# Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
detection_boxes = tf.squeeze(tensor_dict['detection_boxes'],
[0])
detection_masks = tf.squeeze(tensor_dict['detection_masks'],
[0])
real_num_detection = tf.cast(tensor_dict['num_detections'][0],
tf.int32)
detection_boxes = tf.slice(detection_boxes, [0, 0],
[real_num_detection, -1])
detection_masks = tf.slice(detection_masks, [0, 0, 0],
[real_num_detection, -1, -1])
detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, vs.real_height,
vs.real_width)
detection_masks_reframed = tf.cast(
tf.greater(detection_masks_reframed, 0.5), tf.uint8)
# Follow the convention by adding back the batch dimension
tensor_dict['detection_masks'] = tf.expand_dims(
detection_masks_reframed, 0)
if SPLIT_MODEL:
score_out = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores:0')
expand_out = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1:0')
score_in = detection_graph.get_tensor_by_name(
'Postprocessor/convert_scores_1:0')
expand_in = detection_graph.get_tensor_by_name(
'Postprocessor/ExpandDims_1_1:0')
# Threading
score = model.score
expand = model.expand
gpu_worker = SessionWorker("GPU", detection_graph, config)
cpu_worker = SessionWorker("CPU", detection_graph, config)
gpu_opts = [score_out, expand_out]
cpu_opts = [
tensor_dict['detection_boxes'],
tensor_dict['detection_scores'],
tensor_dict['detection_classes'],
tensor_dict['num_detections']
]
gpu_counter = 0
cpu_counter = 0
fps = FPS(FPS_INTERVAL).start()
print('> Starting Detection')
while vs.isActive():
# Detection
if not (USE_TRACKER and track):
if SPLIT_MODEL:
# split model in seperate gpu and cpu session threads
masks = None # No Mask Detection possible yet
if gpu_worker.is_sess_empty():
# read video frame, expand dimensions and convert to rgb
frame = vs.read()
# put new queue
gpu_feeds = {image_tensor: vs.expanded()}
if VISUALIZE:
gpu_extras = frame # for visualization frame
else:
gpu_extras = None
gpu_worker.put_sess_queue(gpu_opts, gpu_feeds,
gpu_extras)
g = gpu_worker.get_result_queue()
if g is None:
# gpu thread has no output queue. ok skip, let's check cpu thread.
gpu_counter += 1
else:
# gpu thread has output queue.
gpu_counter = 0
score, expand, frame = g["results"][0], g[
"results"][1], g["extras"]
if cpu_worker.is_sess_empty():
# When cpu thread has no next queue, put new queue.
# else, drop gpu queue.
cpu_feeds = {
score_in: score,
expand_in: expand
}
cpu_extras = frame
cpu_worker.put_sess_queue(
cpu_opts, cpu_feeds, cpu_extras)
c = cpu_worker.get_result_queue()
if c is None:
# cpu thread has no output queue. ok, nothing to do. continue
cpu_counter += 1
continue # If CPU RESULT has not been set yet, no fps update
else:
cpu_counter = 0
boxes, scores, classes, num, frame = c["results"][
0], c["results"][1], c["results"][2], c[
"results"][3], c["extras"]
else:
# default session
frame = vs.read()
output_dict = sess.run(
tensor_dict,
feed_dict={image_tensor: vs.expanded()})
num = output_dict['num_detections'][0]
classes = output_dict['detection_classes'][0]
boxes = output_dict['detection_boxes'][0]
scores = output_dict['detection_scores'][0]
if 'detection_masks' in output_dict:
masks = output_dict['detection_masks'][0]
else:
masks = None
# reformat detection
num = int(num)
boxes = np.squeeze(boxes)
classes = np.squeeze(classes).astype(np.uint8)
scores = np.squeeze(scores)
# Visualization
vis = vis_detection(frame, VISUALIZE, boxes, classes,
scores, masks, category_index,
DET_INTERVAL, DET_TH, MAX_FRAMES, fps)
if not vis:
break
# Activate Tracker
if USE_TRACKER and num <= NUM_TRACKERS:
tracker_frame = frame
track = True
first_track = True
# Tracking
else:
frame = vs.read()
if first_track:
trackers = []
tracker_boxes = boxes
for box in boxes[~np.all(boxes == 0, axis=1)]:
tracker.init(
conv_detect2track(box, vs.real_width,
vs.real_height),
tracker_frame)
trackers.append(tracker)
first_track = False
for idx, tracker in enumerate(trackers):
tracker_box = tracker.update(frame)
tracker_boxes[idx, :] = conv_track2detect(
tracker_box, vs.real_width, vs.real_height)
vis = vis_detection(frame, VISUALIZE, tracker_boxes,
classes, scores, masks, category_index,
DET_INTERVAL, DET_TH, MAX_FRAMES, fps)
if not vis:
break
tracker_counter += 1
#tracker_frame = frame
if tracker_counter >= TRACKER_FRAMES:
track = False
tracker_counter = 0
fps.update()
# End everything
vs.stop()
fps.stop()
if SPLIT_MODEL:
gpu_worker.stop()
cpu_worker.stop()