class ResNet50ExtractorAPI:
def __init__(self, graph_prefix='', flush_pipe_on_read=False):
self.__flush_pipe_on_read = flush_pipe_on_read
self.__feature_dim = (2048)
self.__image_shape = (224, 224, 3)
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
if not graph_prefix:
self.__graph_prefix = ''
else:
self.__graph_prefix = graph_prefix + '/'
def __preprocess(self, original):
preprocessed = cv2.resize(original,
tuple(self.__image_shape[:2][::-1]))
preprocessed = resnet50.preprocess_input(preprocessed)
return preprocessed
def __in_pipe_process(self, inference):
images = inference.get_input()
if type(images) is not list:
images = [images]
data = []
for img in images:
data.append(self.__preprocess(img))
data = np.array(data)
inference.set_data(data)
return inference
def __out_pipe_process(self, result):
result, inference = result
inference.set_result(result)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
K.set_session(session_runner.get_session())
self.__tf_sess = K.get_session()
with self.__tf_sess.as_default():
self.__model = resnet50.ResNet50(weights='imagenet',
include_top=False,
pooling='avg')
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
(self.__job, inference))
else:
self.__in_pipe.wait()
def __job(self, inference):
try:
self.__out_pipe.push(
(self.__model.predict(inference.get_data()), inference))
except:
self.__out_pipe.push((np.zeros((0, self.__feature_dim),
np.float32), inference))
class TFObjectDetectionAPI(ObjectDetectorInterface):
outPipe = None
inPipe = None
objectTypes = {
0: None,
1: 'person',
2: 'bicycle',
3: 'car',
4: 'motorcycle',
5: 'airplane',
6: 'bus',
7: 'train',
8: 'truck',
9: 'boat',
10: 'traffic light',
11: 'fire hydrant',
13: 'stop sign',
14: 'parking meter',
15: 'bench',
16: 'bird',
17: 'cat',
18: 'dog',
19: 'horse',
20: 'sheep',
21: 'cow',
22: 'elephant',
23: 'bear',
24: 'zebra',
25: 'giraffe',
27: 'backpack',
28: 'umbrella',
31: 'handbag',
32: 'tie',
33: 'suitcase',
34: 'frisbee',
35: 'skis',
36: 'snowboard',
37: 'sports ball',
38: 'kite',
39: 'baseball bat',
40: 'baseball glove',
41: 'skateboard',
42: 'surfboard',
43: 'tennis racket',
44: 'bottle',
46: 'wine glass',
47: 'cup',
48: 'fork',
49: 'knife',
50: 'spoon',
51: 'bowl',
52: 'banana',
53: 'apple',
54: 'sandwich',
55: 'orange',
56: 'broccoli',
57: 'carrot',
58: 'hot dog',
59: 'pizza',
60: 'donut',
61: 'cake',
62: 'chair',
63: 'couch',
64: 'potted plant',
65: 'bed',
67: 'dining table',
70: 'toilet',
72: 'tv',
73: 'laptop',
74: 'mouse',
75: 'remote',
76: 'keyboard',
77: 'cell phone',
78: 'microwave',
79: 'oven',
80: 'toaster',
81: 'sink',
82: 'refrigerator',
84: 'book',
85: 'clock',
86: 'vase',
87: 'scissors',
88: 'teddy bear',
89: 'hair drier',
90: 'toothbrush',
}
thread = None
def __init__(self, model_name=PRETRAINED_ssd_mobilenet_v1_coco_2017_11_17):
self.dir_path = dirname(realpath(__file__))
self.model_path = self.dir_path + '/object_detection/pretrained/'
self.model_file = model_name + '.tar.gz'
self.download_base = 'http://download.tensorflow.org/models/object_detection/'
self.path_to_frozen_graph = self.model_path + model_name + '/frozen_inference_graph.pb'
path_to_labels = os.path.join(self.dir_path + '/object_detection/data',
'mscoco_label_map.pbtxt')
self.class_count = 90
if not path.exists(self.path_to_frozen_graph):
self.__download()
self.__load()
self.label_map = label_map_util.load_labelmap(path_to_labels)
self.categories = label_map_util.convert_label_map_to_categories(
self.label_map,
max_num_classes=self.class_count,
use_display_name=True)
self.category_index = label_map_util.create_category_index(
self.categories)
self.inPipe = Pipe()
self.outPipe = Pipe()
def start(self):
if self.thread is None:
self.thread = Thread(target=self.__start)
self.thread.start()
def stop(self):
if self.thread is not None:
self.thread.stop()
self.thread = None
def getInPipe(self):
return self.inPipe
def getOutPipe(self):
return self.outPipe
def __download(self):
try:
os.mkdir(self.model_path)
except:
pass
opener = urllib.request.URLopener()
opener.retrieve(self.download_base + self.model_file,
self.model_path + self.model_file)
tar_file = tarfile.open(self.model_path + self.model_file)
for file in tar_file.getmembers():
file_name = os.path.basename(file.name)
if 'frozen_inference_graph.pb' in file_name:
tar_file.extract(file, self.model_path)
def __load(self):
self.detection_graph = tf.Graph()
with self.detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(self.path_to_frozen_graph, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
def __start(self, types=None):
with self.detection_graph.as_default():
with tf.Session(graph=self.detection_graph) as sess:
#######################################################Crowd#######################################
# new_saver = tf.train.import_meta_graph("/home/allahbaksh/crowd_count-tf-B/src/model.ckpt.meta")
# new_saver.restore(sess, tf.train.latest_checkpoint("/home/allahbaksh/crowd_count-tf-B/src/"))
# graph = tf.get_default_graph()
# op_to_restore = graph.get_tensor_by_name("add_12:0")
# x = graph.get_tensor_by_name('Placeholder:0')
#
# fps = 25
#
# counter = 0
avg = 0
#
# crowd_count = []
#######################################################Crowd#######################################
while True:
ret, image_np = self.inPipe.pull()
if not ret:
continue
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
image_tensor = self.detection_graph.get_tensor_by_name(
'image_tensor:0')
# Each box represents a part of the image where a particular object was detected.
boxes = self.detection_graph.get_tensor_by_name(
'detection_boxes:0')
# Each score represent how level of confidence for each of the objects.
# Score is shown on the result image, together with the class label.
scores = self.detection_graph.get_tensor_by_name(
'detection_scores:0')
classes = self.detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = self.detection_graph.get_tensor_by_name(
'num_detections:0')
# Actual detection.
(boxes, scores, classes, num_detections) = sess.run(
[boxes, scores, classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
decisionInstances = []
height, width = image_np.shape[0], image_np.shape[1]
boxes = np.squeeze(boxes)
classes = np.squeeze(classes)
scores = np.squeeze(scores)
for i in range(len(classes)):
y_tl, x_tl, y_br, x_br = boxes[i] * [
height, width, height, width
]
try:
objType = self.objectTypes[classes[i]]
if types is not None:
if objType not in types:
continue
except:
if types is not None:
continue
objType = 'undefined'
decisionInstances.append(
DecisionInstance(
InstanceType(objType, classes[i]), scores[i],
InferenceBounds(x_tl, y_tl, x_br, y_br)))
annotatedImage = image_np.copy()
vis_util.visualize_boxes_and_labels_on_image_array(
annotatedImage,
boxes,
classes.astype(np.int32),
scores,
self.category_index,
use_normalized_coordinates=True,
line_thickness=2)
#######################################################Crowd#######################################
# img = image_np.copy()
# img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# img = np.array(img)
# img = (img - 127.5) / 128
#
# x_in = np.reshape(img, (1, img.shape[0], img.shape[1], 1))
# x_in = np.float32(x_in)
# y_pred = []
# y_pred = sess.run(op_to_restore, feed_dict={x: x_in})
# sum = np.absolute(np.int32(np.sum(y_pred)))
#
# crowd_count.append(sum)
# if len(crowd_count) > 20:
# avg = int(np.average(crowd_count) * 0.5)
# crowd_count = []
#
# # if counter <= fps:
# # avg += sum
# # else:
# # counter = 0
# # avg = np.int32(avg / fps)
# # print("AVG ###########################################", avg)
# # avg = 0
# # counter+=1
#######################################################Crowd#######################################
inference = Inference(image_np, decisionInstances,
annotatedImage, avg)
self.outPipe.push(inference)
class MarsExtractorAPI:
def __init__(self, graph_prefix='', flush_pipe_on_read=False):
self.__flush_pipe_on_read = flush_pipe_on_read
self.__model_path = pretrained_path.get() + "/mars-small128.pb"
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
if not graph_prefix:
self.__graph_prefix = ''
else:
self.__graph_prefix = graph_prefix + '/'
def get_input_shape(self):
return self.__image_shape
def __preprocess(self, original):
preprocessed = cv2.resize(original,
tuple(self.__image_shape[:2][::-1]))
# preprocessed = resnet50.preprocess_input(preprocessed)
return preprocessed
def __in_pipe_process(self, inference):
images = inference.get_input()
if type(images) is not list:
images = [images]
data = []
for img in images:
data.append(self.__preprocess(img))
# data = np.array(data)
inference.set_data(data)
return inference
def __out_pipe_process(self, result):
result, inference = result
inference.set_result(result)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
self.__tf_sess = session_runner.get_session()
with self.__tf_sess.graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(self.__model_path, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name=self.__graph_prefix)
self.__input_var = tf.get_default_graph().get_tensor_by_name(
self.__graph_prefix + "images:0")
self.__output_var = tf.get_default_graph().get_tensor_by_name(
self.__graph_prefix + "features:0")
assert len(self.__output_var.get_shape()) == 2
assert len(self.__input_var.get_shape()) == 4
self.__feature_dim = self.__output_var.get_shape().as_list()[-1]
self.__image_shape = self.__input_var.get_shape().as_list()[1:]
# print(self.__image_shape)
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
SessionRunnable(self.__job, inference))
else:
self.__in_pipe.pull_wait()
def __job(self, inference):
x = inference.get_data()
if len(x) > 0:
self.__out_pipe.push(
(self.__tf_sess.run(self.__output_var,
feed_dict={self.__input_var:
x}), inference))
else:
self.__out_pipe.push((np.zeros((0, self.__feature_dim),
np.float32), inference))
class AgeDetection(object):
'''
Path to the trained model path for age_detection
'''
WRN_WEIGHTS_PATH = age_model_path.get() + '/age_model.h5'
def __init__(self, flush_pipe_on_read=False):
self.__detector = None
self.__face_size = 64
self.__flush_pipe_on_read = flush_pipe_on_read
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
def crop_face(self, imgarray, section, margin=40, size=64):
"""
:param imgarray: full image
:param section: face detected area (x, y, w, h)
:param margin: add some margin to the face detected area to include a full head
:param size: the result image resolution with be (size x size)
:return: resized image in numpy array with shape (size x size x 3)
"""
img_h, img_w, _ = imgarray.shape
if section is None:
section = [0, 0, img_w, img_h]
(x, y, w, h) = section
margin = int(min(w, h) * margin / 100)
x_a = x - margin
y_a = y - margin
x_b = x + w + margin
y_b = y + h + margin
if x_a < 0:
x_b = min(x_b - x_a, img_w - 1)
x_a = 0
if y_a < 0:
y_b = min(y_b - y_a, img_h - 1)
y_a = 0
if x_b > img_w:
x_a = max(x_a - (x_b - img_w), 0)
x_b = img_w
if y_b > img_h:
y_a = max(y_a - (y_b - img_h), 0)
y_b = img_h
cropped = imgarray[y_a:y_b, x_a:x_b]
resized_img = cv2.resize(cropped, (size, size),
interpolation=cv2.INTER_AREA)
resized_img = np.array(resized_img)
return resized_img, (x_a, y_a, x_b - x_a, y_b - y_a)
def __in_pipe_process(self, inference):
'''
:param inference: inference object with input frame
:return: inference: inference object with age_inference object containing face bbox details,
data condaining np_array of all faces detected
'''
frame = inference.get_input()
faces = self.__detector.detect_faces(frame)
faces = list_filter(faces, 0.90)
# print("faces " ,faces)
bboxes = []
face_imgs = np.empty(
(len(faces), self.__face_size, self.__face_size, 3))
for i, face in enumerate(faces):
face_img, cropped = self.crop_face(frame,
face[:4],
margin=40,
size=self.__face_size)
(x, y, w, h) = cropped
bboxes.append([x, y, x + w, y + h])
# face_img = (face_img/255)-0.5
# cv2.rectangle(self.image_bounding_boxes, (x, y), (x + w, y + h), (255, 200, 0), 2)
face_imgs[i, :, :, :] = face_img
age_inference = AgeInference(frame, bboxes=bboxes)
inference.set_result(age_inference)
inference.set_data(face_imgs)
return inference
def __out_pipe_process(self, inference):
'''
:param inference: inference object with faces np array in data,
age_inference object with bboxes in result
:return: inference: inference object with age_inference object in result after processing
face np array from data for age, gender and ethnicity
'''
# placeholder for cropped faces
results, inference = inference
age_inference = inference.get_result()
if results is not None:
predicted_genders = results[0]
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages).flatten()
predicted_ethnicity = np.argmax(results[2], axis=1)
age_inference.set_ages(predicted_ages)
age_inference.set_genders(predicted_genders)
age_inference.set_ethnicity(predicted_ethnicity)
inference.set_result(age_inference)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def use_threading(self, run_on_thread=True):
self.__run_session_on_thread = run_on_thread
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
K.set_session(self.__session_runner.get_session())
self.__tf_sess = K.get_session()
self.__detector = MTCNN(session_runner=self.__session_runner)
with self.__tf_sess.as_default():
self.__model = WideResNet(self.__face_size)()
self.__model.load_weights(AgeDetection.WRN_WEIGHTS_PATH)
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
SessionRunnable(self.__job, inference))
else:
self.__in_pipe.pull_wait()
def __job(self, inference):
'''
:param inference: run the model on data from inference object and push it to out_pipe
:return:
'''
try:
self.__out_pipe.push(
(self.__model.predict(inference.get_data()), inference))
except:
self.__out_pipe.push((None, inference))
class ImageEncoder:
@staticmethod
def __fetch_model_path(model_name):
return "../resources/networks/" + model_name
def __init__(self,
session_runner,
model_name=PRETRAINED_mars_small128,
graph_prefix=None):
self.__path_to_frozen_graph = self.__fetch_model_path(model_name)
if not graph_prefix:
self.__graph_prefix = ''
else:
self.__graph_prefix = graph_prefix + '/'
self.__session_runner = session_runner
self.__tf_sess = session_runner.get_session()
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
self.init_graph()
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def __in_pipe_process(self, obj):
return obj
def __out_pipe_process(self, obj):
return obj
def init_graph(self):
with self.__tf_sess.graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(self.__path_to_frozen_graph, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name=self.__graph_prefix)
self.input_var = self.__tf_sess.graph.get_tensor_by_name(
self.__graph_prefix + 'images:0')
self.output_var = self.__tf_sess.graph.get_tensor_by_name(
self.__graph_prefix + 'features:0')
assert len(self.output_var.get_shape()) == 2
assert len(self.input_var.get_shape()) == 4
self.feature_dim = self.output_var.get_shape().as_list()[-1]
self.image_shape = self.input_var.get_shape().as_list()[1:]
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, data_x = self.__in_pipe.pull()
if ret:
self.data_x = data_x
self.__session_runner.add_job(self.__job())
else:
self.__in_pipe.pull_wait()
def __job(self):
__tracking_features = np.zeros((len(self.data_x), self.feature_dim),
np.float32)
self._run_in_batches(
lambda x: self.__tf_sess.run(self.output_var, feed_dict=x),
{self.input_var: self.data_x},
__tracking_features,
batch_size=32)
self.__out_pipe.push(__tracking_features)
def _run_in_batches(self, f, data_dict, out, batch_size):
data_len = len(out)
num_batches = int(data_len / batch_size)
s, e = 0, 0
for i in range(num_batches):
s, e = i * batch_size, (i + 1) * batch_size
batch_data_dict = {k: v[s:e] for k, v in data_dict.items()}
out[s:e] = f(batch_data_dict)
if e < len(out):
batch_data_dict = {k: v[e:] for k, v in data_dict.items()}
out[e:] = f(batch_data_dict)
class YOLOObjectDetectionAPI():
_defaults = {
"model_path": pretrained_path.get() + '/yolo_v3.h5',
"anchors_path": data_path.get() + '/yolo_anchors.txt',
"classes_path": data_path.get() + '/coco.names',
"score": 0.3,
"iou": 0.45,
"model_image_size": (416, 416),
"gpu_num": 1,
}
@staticmethod
def __get_dir_path():
return dirname(realpath(__file__))
@classmethod
def get_defaults(cls, n):
if n in cls._defaults:
return cls._defaults[n]
else:
return "Unrecognized attribute name '" + n + "'"
def __init__(self, graph_prefix=None, flush_pipe_on_read=False):
self.__dict__.update(self._defaults) # set up default values
# self.__dict__.update(kwargs) # and update with user overrides
YOLOObjectDetectionAPI.class_names, self.__category_dict = self._get_class(
)
YOLOObjectDetectionAPI.anchors = self._get_anchors()
self.__graph_prefix = graph_prefix
self.__flush_pipe_on_read = flush_pipe_on_read
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
self.__run_session_on_thread = False
def use_threading(self, run_on_thread=True):
self.__run_session_on_thread = run_on_thread
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
K.set_session(session_runner.get_session())
self.__tf_sess = K.get_session()
self.boxes, self.scores, self.classes = self.generate()
def __in_pipe_process(self, inference):
image = inference.get_input()
pil_image = Image.fromarray(image)
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
pil_image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (pil_image.width - (pil_image.width % 32),
pil_image.height - (pil_image.height % 32))
boxed_image = letterbox_image(pil_image, new_image_size)
data = np.array(boxed_image, dtype='float32')
data /= 255.
data = np.expand_dims(data, 0) # Add batch dimension.
inference.set_data(data)
inference.get_meta_dict()['PIL'] = pil_image
return inference
# return image
def __out_pipe_process(self, result):
(out_boxes, out_classes, out_scores), inference = result
out_boxes = np.array([[0 if y < 0 else y for y in x]
for x in out_boxes])
result = InferedDetections(inference.get_input(),
len(out_boxes),
out_boxes,
out_classes,
out_scores,
masks=None,
is_normalized=False,
get_category_fnc=self.get_category,
annotator=self.annotate)
inference.set_result(result)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def _get_class(self):
dir_path = YOLOObjectDetectionAPI.__get_dir_path()
classes_path = os.path.expanduser(self.classes_path)
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
category_dict = {}
for id, name in enumerate(class_names):
category_dict[id] = name
category_dict[name] = id
return class_names, category_dict
def _get_anchors(self):
dir_path = YOLOObjectDetectionAPI.__get_dir_path()
anchors_path = os.path.expanduser(self.anchors_path)
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
return np.array(anchors).reshape(-1, 2)
def generate(self):
dir_path = YOLOObjectDetectionAPI.__get_dir_path()
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith(
'.h5'), 'Keras model or weights must be a .h5 file.'
# Load model, or construct model and load weights.
num_anchors = len(self.anchors)
num_classes = len(self.class_names)
is_tiny_version = num_anchors == 6 # default setting
try:
self.yolo_model = load_model(model_path, compile=False)
except:
self.yolo_model = tiny_yolo_body(Input(shape=(None, None, 3)), num_anchors // 2, num_classes) \
if is_tiny_version else yolo_body(Input(shape=(None, None, 3)), num_anchors // 3, num_classes)
self.yolo_model.load_weights(
self.model_path) # make sure model, anchors and classes match
else:
assert self.yolo_model.layers[-1].output_shape[-1] == \
num_anchors / len(self.yolo_model.output) * (num_classes + 5), \
'Mismatch between model and given anchor and class sizes'
# print('{} model, anchors, and classes loaded.'.format(model_path))
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
YOLOObjectDetectionAPI.colors = list(
map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
YOLOObjectDetectionAPI.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
YOLOObjectDetectionAPI.colors))
np.random.seed(10101) # Fixed seed for consistent colors across runs.
np.random.shuffle(YOLOObjectDetectionAPI.colors
) # Shuffle colors to decorrelate adjacent classes.
np.random.shuffle(YOLOObjectDetectionAPI.colors
) # Shuffle colors to decorrelate adjacent classes.
np.random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
self.input_image_shape = K.placeholder(shape=(2, ))
if self.gpu_num >= 2:
self.yolo_model = multi_gpu_model(self.yolo_model,
gpus=self.gpu_num)
boxes, scores, classes = yolo_eval(self.yolo_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score,
iou_threshold=self.iou)
return boxes, scores, classes
def freeze_session(self,
session,
keep_var_names=None,
output_names=None,
clear_devices=True):
"""
Freezes the state of a session into a pruned computation graph.
Creates a new computation graph where variable nodes are replaced by
constants taking their current value in the session. The new graph will be
pruned so subgraphs that are not necessary to compute the requested
outputs are removed.
@param session The TensorFlowsion(detected_boxes, confidence_threshold=FLAGS.conf_threshold,
iou_threshold=FLAGS.iou_threshold)
draw_boxes(filtered_boxes, img, classes, (FLAGS.size, FLAGS.size))
img.save(FLAGS.output_img) session to be frozen.
@param keep_var_names A list of variable names that should not be frozen,
or None to freeze all the variables in the graph.
@param output_names Names of the relevant graph outputs.
@param clear_devices Remove the device directives from the graph for better portability.
@return The frozen graph definition.
"""
from tensorflow.python.framework.graph_util import convert_variables_to_constants
graph = session.graph
with graph.as_default():
freeze_var_names = list(
set(v.op.name
for v in tf.global_variables()).difference(keep_var_names
or []))
output_names = output_names or []
output_names += [v.op.name for v in tf.global_variables()]
input_graph_def = graph.as_graph_def()
if clear_devices:
for node in input_graph_def.node:
node.device = ""
frozen_graph = convert_variables_to_constants(
session, input_graph_def, output_names, freeze_var_names)
return frozen_graph
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
SessionRunnable(
self.__job,
inference,
run_on_thread=self.__run_session_on_thread))
else:
self.__in_pipe.pull_wait()
def __job(self, inference):
data = inference.get_data()
pil_image = inference.get_meta_dict()['PIL']
# image = inference.get_input()
out_boxes, out_scores, out_classes = self.__tf_sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: data,
self.input_image_shape: [pil_image.size[1], pil_image.size[0]],
})
self.__out_pipe.push(((out_boxes, out_classes, out_scores), inference))
def close_session(self):
self.__tf_sess.close()
@staticmethod
def annotate(inference):
annotated = inference.image.copy()
image = Image.fromarray(inference.get_image())
font = ImageFont.truetype(font='arial.ttf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = 1 #(image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(inference.get_classes()))):
predicted_class = YOLOObjectDetectionAPI.class_names[c]
box = inference.get_boxes_tlbr(normalized=False)[i]
score = inference.get_scores()[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
# print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=YOLOObjectDetectionAPI.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=YOLOObjectDetectionAPI.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
return np.array(image)
def get_category(self, category):
return self.__category_dict[category]
class TFObjectDetectionAPI:
@staticmethod
def __get_dir_path():
return dirname(realpath(__file__))
@staticmethod
def __download_model(model_path, download_base, model_file):
print("downloading model...", model_path)
try:
os.mkdir(model_path)
except:
pass
opener = urllib.request.URLopener()
opener.retrieve(download_base + model_file, model_path + model_file)
print("finished downloading. extracting...")
tar_file = tarfile.open(model_path + model_file)
for file in tar_file.getmembers():
file_name = os.path.basename(file.name)
if 'frozen_inference_graph.pb' in file_name:
tar_file.extract(file, model_path)
print("finished extracting.")
@staticmethod
def __fetch_model_path(model_name):
dir_path = pretrained_path.get()
model_path = dir_path + '/'
model_file = model_name + '.tar.gz'
download_base = 'http://download.tensorflow.org/models/object_detection/'
path_to_frozen_graph = model_path + model_name + '/frozen_inference_graph.pb'
if not path.exists(path_to_frozen_graph):
TFObjectDetectionAPI.__download_model(model_path, download_base, model_file)
return path_to_frozen_graph
@staticmethod
def __fetch_category_indices():
dir_path = TFObjectDetectionAPI.__get_dir_path()
path_to_labels = os.path.join(data_path.get(), 'mscoco_label_map.pbtxt')
class_count = 90
label_map = label_map_util.load_labelmap(path_to_labels)
categories = label_map_util.convert_label_map_to_categories(label_map,
max_num_classes=class_count,
use_display_name=True)
category_index = label_map_util.create_category_index(categories)
category_dict = {}
for item in category_index.values():
category_dict[item['id']] = item['name']
category_dict[item['name']] = item['id']
return category_index, category_dict
def __init__(self, model_name=PRETRAINED_ssd_mobilenet_v1_coco_2017_11_17, image_shape=None,
graph_prefix=None, flush_pipe_on_read=False):
self.__category_index, self.__category_dict = self.__fetch_category_indices()
self.__path_to_frozen_graph = self.__fetch_model_path(model_name)
self.__flush_pipe_on_read = flush_pipe_on_read
self.__image_shape = image_shape
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
self.__run_session_on_thread = False
if not graph_prefix:
self.__graph_prefix = ''
else:
self.__graph_prefix = graph_prefix + '/'
def __in_pipe_process(self, inference):
image = inference.get_input()
data = np.expand_dims(image, axis=0)
inference.set_data(data)
return inference
def __out_pipe_process(self, result):
result, inference = result
num_detections = int(result['num_detections'][0])
detection_classes = result['detection_classes'][0][:num_detections].astype(np.uint8)
detection_boxes = result['detection_boxes'][0][:num_detections]
detection_scores = result['detection_scores'][0][:num_detections]
if 'detection_masks' in result:
detection_masks = result['detection_masks'][0][:num_detections]
else:
detection_masks = None
result = InferedDetections(inference.get_input(), num_detections, detection_boxes, detection_classes,
detection_scores,
masks=detection_masks, is_normalized=True, get_category_fnc=self.get_category,
annotator=self.annotate)
inference.set_result(result)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def use_threading(self, run_on_thread=True):
self.__run_session_on_thread = run_on_thread
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
self.__tf_sess = session_runner.get_session()
with self.__tf_sess.graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(self.__path_to_frozen_graph, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name=self.__graph_prefix)
tf_default_graph = self.__tf_sess.graph
self.__image_tensor = tf_default_graph.get_tensor_by_name(self.__graph_prefix + 'image_tensor:0')
tensor_names = {output.name for op in tf_default_graph.get_operations() for output in op.outputs}
self.__tensor_dict = {}
for key in ['num_detections', 'detection_boxes', 'detection_classes', 'detection_scores', 'detection_masks']:
tensor_name = self.__graph_prefix + key + ':0'
if tensor_name in tensor_names:
self.__tensor_dict[key] = tf_default_graph.get_tensor_by_name(
tensor_name)
if 'detection_masks' in self.__tensor_dict:
# The following processing is only for single image
detection_boxes = tf.squeeze(self.__tensor_dict['detection_boxes'], [0])
detection_masks = tf.squeeze(self.__tensor_dict['detection_masks'], [0])
# Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
real_num_detection = tf.cast(self.__tensor_dict['num_detections'][0], tf.int32)
detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])
detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])
detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, self.__image_shape[0], self.__image_shape[1])
detection_masks_reframed = tf.cast(
tf.greater(detection_masks_reframed, 0.5), tf.uint8)
# Follow the convention by adding back the batch dimension
self.__tensor_dict['detection_masks'] = tf.expand_dims(
detection_masks_reframed, 0)
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
SessionRunnable(self.__job, inference, run_on_thread=self.__run_session_on_thread))
else:
self.__in_pipe.wait()
def __job(self, inference):
self.__out_pipe.push(
(self.__tf_sess.run(self.__tensor_dict, feed_dict={self.__image_tensor: inference.get_data()}), inference))
def get_category(self, category):
return self.__category_dict[category]
@staticmethod
def annotate(inference):
annotated = inference.image.copy()
vis_util.visualize_boxes_and_labels_on_image_array(
annotated,
inference.get_boxes_tlbr(),
inference.get_classes().astype(np.int32),
inference.get_scores(),
TFObjectDetectionAPI.__fetch_category_indices()[0],
instance_masks=inference.get_masks(),
use_normalized_coordinates=True,
line_thickness=1)
return annotated
class YOLO(object):
def __init__(self,
input_size,
labels,
max_box_per_image,
anchors,
flush_pipe_on_read=False):
self.__flush_pipe_on_read = flush_pipe_on_read
self.__model_path = model_path.get()
self.__model = None
self.__input_size = input_size
self.__labels = list(labels)
self.__nb_class = len(self.__labels)
self.__nb_box = len(anchors) // 2
self.__class_wt = np.ones(self.__nb_class, dtype='float32')
self.__anchors = anchors
self.__max_box_per_image = max_box_per_image
self.__session_runner = None
self.__tf_sess = None
self.__thread = None
self.__in_pipe = Pipe(self.__in_pipe_process)
self.__out_pipe = Pipe(self.__out_pipe_process)
self.__run_session_on_thread = False
def use_threading(self, run_on_thread=True):
self.__run_session_on_thread = run_on_thread
def __normalize(self, image):
return image / 255.
def __preprocess(self, original, warp_points):
img_shape = original.shape
test1 = np.array(warp_points['point_set_1'], dtype="float32")
test2 = np.array(warp_points['point_set_2'], dtype="float32")
# print(test2)
# points = [warp_points['point_set_2'][0],warp_points['point_set_2'][2]]
# print(points)
# np.array([[0, 0], [img_shape[0], 0], [img_shape[0], img_shape[1]], [0, img_shape[1]]], dtype=np.float32)
M = cv2.getPerspectiveTransform(test1, test2)
warped_image = cv2.warpPerspective(original,
M, (img_shape[1], img_shape[0]),
flags=cv2.INTER_LINEAR)
# warped_image = warped_image[points[0][1]:points[1][1],points[0][0]:points[1][0],:]
return warped_image
def __prepare_data(self, image):
image_h, image_w, _ = image.shape
preprocessed = cv2.resize(image,
(self.__input_size, self.__input_size))
preprocessed = self.__normalize(preprocessed)
preprocessed = preprocessed[:, :, ::-1]
preprocessed = np.expand_dims(preprocessed, 0)
return preprocessed
def __in_pipe_process(self, inference):
image = inference.get_input()
warp_points = inference.get_meta_dict()['warp_points']
dummy_array = np.zeros((1, 1, 1, 1, self.__max_box_per_image, 4))
warped_image = self.__preprocess(image, warp_points)
data = (self.__prepare_data(warped_image), dummy_array)
inference.set_data(data)
inference.get_meta_dict()['warped_image'] = warped_image
return inference
def __out_pipe_process(self, result):
result, inference = result
image = inference.get_meta_dict()['warped_image']
i_dets = RetailInference(image, self.__labels)
i_dets.decode_netout(result, self.__anchors, self.__nb_class)
inference.set_result(i_dets)
return inference
def get_in_pipe(self):
return self.__in_pipe
def get_out_pipe(self):
return self.__out_pipe
def use_session_runner(self, session_runner):
self.__session_runner = session_runner
K.set_session(self.__session_runner.get_session())
self.__tf_sess = K.get_session()
with self.__tf_sess.as_default():
with self.__tf_sess.graph.as_default():
self.__model = load_model(self.__model_path + '/check1.h5',
custom_objects={"tf": tf})
print("Successful load of yolo")
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
if self.__in_pipe.is_closed():
self.__out_pipe.close()
return
ret, inference = self.__in_pipe.pull(self.__flush_pipe_on_read)
if ret:
self.__session_runner.get_in_pipe().push(
SessionRunnable(
self.__job,
inference,
run_on_thread=self.__run_session_on_thread))
else:
self.__in_pipe.pull_wait()
def __job(self, inference):
data = inference.get_data()
self.__out_pipe.push((self.__model.predict([data[0],
data[1]])[0], inference))
exit_time_str = datetime.fromtimestamp(entry_time).strftime(
'%Y:%m:%d:%H:%M:%S')
entry_time_str = datetime.fromtimestamp(entry_time).strftime(
'%Y:%m:%d:%H:%M:%S')
exit_zone['rack_id'] = zone.get_id()
exit_zone['person_id'] = trk.get_id()
exit_zone['entry_time'] = entry_time_str
exit_zone['exit_time'] = exit_time_str
if exit_time - entry_time > 10:
# req = requests.post(URL, json=exit_zone)
# print(req)
payload.append(exit_zone)
print("Exiting: ", exit_zone)
zones.extend(trk.get_trail().get_current_zones())
zone_detection_in_pipe.push(zones)
overlay = frame.copy()
for z in tracker.get_zones():
cv2.polylines(overlay, [np.int32(z.get_coords())], 1,
(0, 255, 255), 2)
frame = cv2.addWeighted(overlay, 0.4, frame, 0.6, 0)
# # count+=1
# video_writer.write(frame)
# if patches:
# for i, patch in enumerate(patches):
# cv2.imshow("patch" + str(i), patch)
class FLIRCamera(object):
def __init__(self, index=None):
if index is None:
index = 0
self.__system = PySpin.System.GetInstance()
self.__cam_list = self.__system.GetCameras()
self.__cam = self.__cam_list.GetByIndex(index)
# self.__pixel_format = self.__cam.PixelFormat
# print(self.__pixel_format.GetCurrentEntry())
# self.__pixel_format = PySpin.PixelFormat_BGR8
# print(self.__pixel_format)
# self.__access_mode = self.__cam.PixelFormat.GetAccessMode
self.__cam.Init()
self.__nodemap = self.__cam.GetNodeMap()
self.configure_custom_image_settings(self.__nodemap)
# self.__cam.DeInit()
# self.__cam.Init()
self.__cam.AcquisitionMode.SetValue(PySpin.AcquisitionMode_Continuous)
print(self.__cam.EncoderMode)
self.__node_acquisition_framerate = PySpin.CFloatPtr(
self.__nodemap.GetNode('AcquisitionFrameRate'))
self.__framerate_to_set = self.__node_acquisition_framerate.GetValue()
print(self.__framerate_to_set)
self.__cam.BeginAcquisition()
self.__out_pipe = Pipe(self.__out_pipe_process)
self.__thread = None
def __del__(self):
self.close()
def __out_pipe_process(self, image_primary):
image_primary = image_primary.Convert(PySpin.PixelFormat_BGR8,
PySpin.HQ_LINEAR)
image_array = image_primary.GetNDArray()
return image_array
def close(self):
self.__thread = None
self.__cam.EndAcquisition()
self.__cam.DeInit()
del self.__cam
del self.__cam_list
del self.__system
self.__out_pipe.close()
def run(self):
if self.__thread is None:
self.__thread = Thread(target=self.__run)
self.__thread.start()
def __run(self):
while self.__thread:
try:
self.__job(self.__cam.GetNextImage())
except:
self.close()
def __job(self, image_primary):
self.__out_pipe.push(image_primary)
def get_out_pipe(self):
return self.__out_pipe
def configure_custom_image_settings(self, nodemap):
"""
Configures a number of settings on the camera including offsets X and Y, width,
height, and pixel format. These settings must be applied before BeginAcquisition()
is called; otherwise, they will be read only. Also, it is important to note that
settings are applied immediately. This means if you plan to reduce the width and
move the x offset accordingly, you need to apply such changes in the appropriate order.
:param nodemap: GenICam nodemap.
:type nodemap: INodeMap
:return: True if successful, False otherwise.
:rtype: bool
"""
try:
result = True
# Apply mono 8 pixel format
#
# *** NOTES ***
# Enumeration nodes are slightly more complicated to set than other
# nodes. This is because setting an enumeration node requires working
# with two nodes instead of the usual one.
#
# As such, there are a number of steps to setting an enumeration node:
# retrieve the enumeration node from the nodemap, retrieve the desired
# entry node from the enumeration node, retrieve the integer value from
# the entry node, and set the new value of the enumeration node with
# the integer value from the entry node.
#
# Retrieve the enumeration node from the nodemap
node_pixel_format = PySpin.CEnumerationPtr(
nodemap.GetNode('PixelFormat'))
if PySpin.IsAvailable(node_pixel_format) and PySpin.IsWritable(
node_pixel_format):
# Retrieve the desired entry node from the enumeration node
node_pixel_format_mono8 = PySpin.CEnumEntryPtr(
node_pixel_format.GetEntryByName('RGB8'))
print(node_pixel_format.GetEntries())
if PySpin.IsAvailable(
node_pixel_format_mono8) and PySpin.IsReadable(
node_pixel_format_mono8):
# Retrieve the integer value from the entry node
pixel_format_mono8 = node_pixel_format_mono8.GetValue()
# Set integer as new value for enumeration node
node_pixel_format.SetIntValue(pixel_format_mono8)
print('Pixel format set to %s...' %
node_pixel_format.GetCurrentEntry().GetSymbolic())
else:
print('Pixel format mono 8 not available...')
else:
print('Pixel format not available...')
# Apply minimum to offset X
#
# *** NOTES ***
# Numeric nodes have both a minimum and maximum. A minimum is retrieved
# with the method GetMin(). Sometimes it can be important to check
# minimums to ensure that your desired value is within range.
node_offset_x = PySpin.CIntegerPtr(nodemap.GetNode('OffsetX'))
if PySpin.IsAvailable(node_offset_x) and PySpin.IsWritable(
node_offset_x):
node_offset_x.SetValue(node_offset_x.GetMin())
print('Offset X set to %i...' % node_offset_x.GetMin())
else:
print('Offset X not available...')
# Apply minimum to offset Y
#
# *** NOTES ***
# It is often desirable to check the increment as well. The increment
# is a number of which a desired value must be a multiple of. Certain
# nodes, such as those corresponding to offsets X and Y, have an
# increment of 1, which basically means that any value within range
# is appropriate. The increment is retrieved with the method GetInc().
node_offset_y = PySpin.CIntegerPtr(nodemap.GetNode('OffsetY'))
if PySpin.IsAvailable(node_offset_y) and PySpin.IsWritable(
node_offset_y):
node_offset_y.SetValue(node_offset_y.GetMin())
print('Offset Y set to %i...' % node_offset_y.GetMin())
else:
print('Offset Y not available...')
# Set maximum width
#
# *** NOTES ***
# Other nodes, such as those corresponding to image width and height,
# might have an increment other than 1. In these cases, it can be
# important to check that the desired value is a multiple of the
# increment. However, as these values are being set to the maximum,
# there is no reason to check against the increment.
node_width = PySpin.CIntegerPtr(nodemap.GetNode('Width'))
if PySpin.IsAvailable(node_width) and PySpin.IsWritable(
node_width):
width_to_set = node_width.GetMax()
node_width.SetValue(width_to_set)
print('Width set to %i...' % node_width.GetValue())
else:
print('Width not available...')
# Set maximum height
#
# *** NOTES ***
# A maximum is retrieved with the method GetMax(). A node's minimum and
# maximum should always be a multiple of its increment.
node_height = PySpin.CIntegerPtr(nodemap.GetNode('Height'))
if PySpin.IsAvailable(node_height) and PySpin.IsWritable(
node_height):
height_to_set = node_height.GetMax()
node_height.SetValue(height_to_set)
print('Height set to %i...' % node_height.GetValue())
else:
print('Height not available...')
except PySpin.SpinnakerException as ex:
print('Error: %s' % ex)
return False
return result