def inference(self, resized_rgb_image):
"""
inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding detection output which is used for creating result
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
"""
input_image = np.expand_dims(resized_rgb_image, axis=0)
input_tensor = tf.convert_to_tensor(input_image)
t_begin = time.perf_counter()
output_dict = self.detection_model(input_tensor)
inference_time = time.perf_counter() - t_begin # Seconds
# Calculate Frames rate (fps)
self.fps = convert_infr_time_to_fps(inference_time)
boxes = output_dict['detection_boxes']
labels = output_dict['detection_classes']
scores = output_dict['detection_scores']
class_id = int(self.model_variables['ClassID'])
score_threshold = float(self.model_variables['MinScore'])
result = []
for i in range(boxes.shape[1]): # number of boxes
if labels[0, i] == class_id and scores[0, i] > score_threshold:
result.append({"id": str(class_id) + '-' + str(i), "bbox": boxes[0, i, :].numpy(), "score": scores[0, i]})
return result
def inference(self, resized_rgb_images) -> list:
"""
Inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding class id output which is used for creating result
Args:
resized_rgb_images: Array of images with shape (no_images, img_height, img_width, channels)
Returns:
result: List of class id for each input image. ex: [0, 0, 1, 1, 0]
scores: The classification confidence for each class. ex: [.99, .75, .80, 1.0]
"""
if np.shape(resized_rgb_images)[0] == 0:
return [], []
# input_image = np.expand_dims(resized_rgb_images, axis=0)
t_begin = time.perf_counter()
output_dict = self.classifier_model.predict(resized_rgb_images)
inference_time = time.perf_counter() - t_begin # Seconds
# Calculate Frames rate (fps)
self.fps = convert_infr_time_to_fps(inference_time)
result = list(np.argmax(output_dict, axis=1)) # returns class id
# TODO: optimized without for
scores = []
for i, itm in enumerate(output_dict):
scores.append(itm[result[i]])
return result, scores
def inference(self, resized_rgb_images) -> list:
"""
Inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding class id output which is used for creating result
Args:
resized_rgb_images: Array of images with shape (no_images, img_height, img_width, channels)
Returns:
result: List of class id for each input image. ex: [0, 0, 1, 1, 0]
scores: The classification confidence for each class. ex: [.99, .75, .80, 1.0]
"""
if np.shape(resized_rgb_images)[0] == 0:
return [], []
resized_rgb_images = (resized_rgb_images * 255).astype("uint8")
result = []
net_results = []
for img in resized_rgb_images:
img = np.expand_dims(img, axis=0)
self.interpreter.set_tensor(self.input_details[0]["index"], img)
t_begin = time.perf_counter()
self.interpreter.invoke()
inference_time = time.perf_counter() - t_begin # Second
self.fps = convert_infr_time_to_fps(inference_time)
net_output = self.interpreter.get_tensor(
self.output_details[0]['index'])[0]
net_results.append(net_output)
result.append(np.argmax(net_output)) # returns class id
# TODO: optimized without for
scores = []
for i, itm in enumerate(net_results):
scores.append((itm[result[i]] - 1) / 255.0)
return result, scores
def inference(self, resized_rgb_image):
img, orig_im, dim = self.prep_image(resized_rgb_image, self._inp_dim)
im_dim = torch.FloatTensor(dim).repeat(1, 2)
if self._CUDA:
im_dim = im_dim.cuda()
img = img.cuda()
# start calculate fps
t_begin = time.perf_counter()
with torch.no_grad():
output = self._model(Variable(img), self._CUDA)
output = write_results(output,
self.confidence,
self._num_classes,
nms=True,
nms_conf=self.nms_threshold)
inference_time = time.perf_counter() - t_begin
self.fps = convert_infr_time_to_fps(inference_time)
im_dim = im_dim.repeat(output.size(0), 1)
scaling_factor = torch.min(self._inp_dim / im_dim, 1)[0].view(-1, 1)
output[:, [1, 3]] -= (self._inp_dim -
scaling_factor * im_dim[:, 0].view(-1, 1)) / 2
output[:, [2, 4]] -= (self._inp_dim -
scaling_factor * im_dim[:, 1].view(-1, 1)) / 2
output[:, 1:5] /= scaling_factor
for i in range(output.shape[0]):
output[i, [1, 3]] = torch.clamp(output[i, [1, 3]], 0.0, im_dim[i,
0])
output[i, [2, 4]] = torch.clamp(output[i, [2, 4]], 0.0, im_dim[i,
1])
result = []
for i, pred in enumerate(output):
c1 = pred[1:3].cpu().int().numpy() # unormalized [xmin, ymin]
c2 = pred[3:5].cpu().int().numpy() # unormalized [xmax, ymax]
cls = int(pred[-1].cpu())
score = float(pred[5].cpu())
if cls == 0: # person class index is '0' at coco dataset
bbox_dict = {
"id":
"1-" + str(i),
"bbox": [
c1[1] / self.h, c1[0] / self.w, c2[1] / self.h,
c2[0] / self.w
],
"score":
score,
"face":
None
}
result.append(bbox_dict)
return result
def inference(self, resized_rgb_images):
"""
Inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding class id output which is used for creating result
Args:
resized_rgb_images: Array of images with shape (no_images, img_height, img_width, channels)
Returns:
result: List of class id for each input image. ex: [0, 0, 1, 1, 0]
scores: The classification confidence for each class. ex: [.99, .75, .80, 1.0]
"""
bindings = self.bindings
host_inputs = self.host_inputs
host_outputs = self.host_outputs
cuda_inputs = self.cuda_inputs
cuda_outputs = self.cuda_outputs
stream = self.stream
t_begin = time.perf_counter()
result = []
scores = []
for img in resized_rgb_images:
img = np.expand_dims(img, axis=0)
img = img.astype(np.float32)
host_inputs[0] = np.ravel(np.zeros_like(img))
self.cuda_context.push()
np.copyto(host_inputs[0], img.ravel())
cuda.memcpy_htod_async(cuda_inputs[0], host_inputs[0], stream)
self.engine_context.execute_async(batch_size=1,
bindings=bindings,
stream_handle=stream.handle)
cuda.memcpy_dtoh_async(host_outputs[0], cuda_outputs[0], stream)
stream.synchronize()
output_dict = host_outputs[0]
pred = list(np.argmax(host_outputs, axis=1))
# TODO: optimized without for
for i, itm in enumerate(host_outputs):
scores.append(itm[pred[i]])
result.append(pred[0])
self.cuda_context.pop()
inference_time = float(time.perf_counter() - t_begin)
if len(resized_rgb_images) != 0:
inference_time = inference_time / len(resized_rgb_images)
self.fps = convert_infr_time_to_fps(inference_time)
return result, scores
def inference(self, resized_rgb_image):
"""
inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding detection output which is used for creating result
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
"""
required_image_size = (544, 320)
input_image = cv.resize(resized_rgb_image, required_image_size)
input_image = input_image.transpose(2, 0, 1)
input_image = np.expand_dims(input_image, axis=0)
t_begin = time.perf_counter()
output = self.detection_model.infer(
inputs={self.input_layer: input_image}
)['detection_out']
inference_time = time.perf_counter() - t_begin # Seconds
# Calculate Frames rate (fps)
self.fps = convert_infr_time_to_fps(inference_time)
class_id = int(self.config.get_section_dict('Detector')['ClassID'])
score_threshold = float(self.config.get_section_dict('Detector')['MinScore'])
result = []
for i, (_, label, score, x_min, y_min, x_max, y_max) in enumerate(output[0][0]):
box = [y_min, x_min, y_max, x_max]
if label == class_id and score > score_threshold:
result.append({"id": str(class_id) + '-' + str(i), "bbox": box, "score": score})
return result
def inference(self, resized_rgb_image):
"""
This method will perform inference and return the detected bounding boxes
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
"""
image = resized_rgb_image
image = cv2.resize(image, self.model_input_size)
pil_im = PIL.Image.fromarray(image)
preprocess = None
data = openpifpaf.datasets.PilImageList([pil_im],
preprocess=preprocess)
loader = torch.utils.data.DataLoader(
data,
batch_size=1,
shuffle=False,
pin_memory=True,
collate_fn=openpifpaf.datasets.collate_images_anns_meta)
for images_batch, _, __ in loader:
np_img = images_batch.numpy()
bindings = self.bindings
host_inputs = self.host_inputs
host_outputs = self.host_outputs
cuda_inputs = self.cuda_inputs
cuda_outputs = self.cuda_outputs
stream = self.stream
host_inputs[0] = np.ravel(np.zeros_like(np_img))
self.cuda_context.push()
t_begin = time.perf_counter()
np.copyto(host_inputs[0], np.ravel(np_img))
cuda.memcpy_htod_async(cuda_inputs[0], host_inputs[0], stream)
self.engine_context.execute_async(batch_size=1,
bindings=bindings,
stream_handle=stream.handle)
cif = [None] * 1
caf = [None] * 1
cif_names = ['cif']
caf_names = ['caf']
for i in range(1, self.engine.num_bindings):
cuda.memcpy_dtoh_async(host_outputs[i - 1], cuda_outputs[i - 1],
stream)
stream.synchronize()
for i in range(1, self.engine.num_bindings):
shape = self.engine.get_binding_shape(i)
name = self.engine.get_binding_name(i)
total_shape = np.prod(shape)
output = host_outputs[i - 1][0:total_shape]
output = np.reshape(output, tuple(shape))
if name in cif_names:
index_n = cif_names.index(name)
tmp = torch.from_numpy(output[0])
cif = tmp.cpu().numpy()
elif name in caf_names:
index_n = caf_names.index(name)
tmp = torch.from_numpy(output[0])
caf = tmp.cpu().numpy()
heads = [cif, caf]
self.cuda_context.pop()
inference_time = time.perf_counter() - t_begin
fields = heads
decoder_time = time.perf_counter()
decoder = CifCafDecoder()
predictions = decoder.decode(fields)
decoder_time = time.perf_counter() - t_begin
self.fps = convert_infr_time_to_fps(inference_time + decoder_time)
result = []
for i, pred_object in enumerate(predictions):
pred = pred_object.data
pred_visible = pred[pred[:, 2] > .2]
xs = pred_visible[:, 0]
ys = pred_visible[:, 1]
if len(xs) == 0 or len(ys) == 0:
continue
x, y, w, h = pred_object.bbox()
x_min = int(x)
x_max = int(x + w)
y_min = int(y)
y_max = int(y + h)
xmin = int(max(x_min - .15 * w, 0))
xmax = int(min(x_max + .15 * w, self.w))
ymin = int(max(y_min - .2 * h, 0))
ymax = int(min(y_max + .05 * h, self.h))
bbox_dict = {
"id":
"1-" + str(i),
"bbox":
[ymin / self.h, xmin / self.w, ymax / self.h, xmax / self.w],
"score":
0.9,
"face":
None
}
# extract face bounding box
if np.all(pred[[0, 1, 2, 5, 6], -1] > 0.15):
x_min_face = int(pred[6, 0])
x_max_face = int(pred[5, 0])
y_max_face = int((pred[5, 1] + pred[6, 1]) / 2)
y_eyes = int((pred[1, 1] + pred[2, 1]) / 2)
y_min_face = 2 * y_eyes - y_max_face
if (y_max_face - y_min_face > 0) and (x_max_face - x_min_face >
0):
h_crop = y_max_face - y_min_face
x_min_face = int(max(0, x_min_face - 0.1 * h_crop))
y_min_face = int(max(0, y_min_face - 0.1 * h_crop))
x_max_face = int(min(self.w, x_min_face + 1.1 * h_crop))
y_max_face = int(min(self.h, y_min_face + 1.1 * h_crop))
bbox_dict["face"] = [
y_min_face / self.h, x_min_face / self.w,
y_max_face / self.h, x_max_face / self.w
]
result.append(bbox_dict)
return result
def inference(self, resized_rgb_image):
"""
This method will perform inference and return the detected bounding boxes
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
"""
pil_im = PIL.Image.fromarray(resized_rgb_image)
preprocess = openpifpaf.transforms.Compose([
openpifpaf.transforms.NormalizeAnnotations(),
openpifpaf.transforms.CenterPadTight(16),
openpifpaf.transforms.EVAL_TRANSFORM,
])
data = openpifpaf.datasets.PilImageList([pil_im], preprocess=preprocess)
loader = torch.utils.data.DataLoader(
data, batch_size=1, pin_memory=True,
collate_fn=openpifpaf.datasets.collate_images_anns_meta)
t_begin = time.perf_counter()
for images_batch, _, __ in loader:
predictions = self.processor.batch(self.net, images_batch, device=self.device)[0]
inference_time = time.perf_counter() - t_begin
self.fps = convert_infr_time_to_fps(inference_time)
result = []
for i, pred in enumerate(predictions):
pred = pred.data
pred_visible = pred[pred[:, 2] > .2]
xs = pred_visible[:, 0]
ys = pred_visible[:, 1]
x_min = int(xs.min())
x_max = int(xs.max())
y_min = int(ys.min())
y_max = int(ys.max())
w = x_max - x_min
h = y_max - y_min
xmin = int(max(x_min - .15 * w, 0))
xmax = int(min(x_max + .15 * w, self.w))
ymin = int(max(y_min - .2 * h, 0))
ymax = int(min(y_max + .05 * h, self.h))
bbox_dict = {"id": "1-" + str(i), "bbox": [ymin / self.h, xmin / self.w, ymax / self.h, xmax / self.w],
"score": 0.9, "face": None}
# extracting face bounding box
if np.all(pred[[0, 1, 2, 5, 6], -1] > 0.15):
x_min_face = int(pred[6, 0])
x_max_face = int(pred[5, 0])
y_max_face = int((pred[5, 1] + pred[6, 1]) / 2)
y_eyes = int((pred[1, 1] + pred[2, 1]) / 2)
y_min_face = 2 * y_eyes - y_max_face
if (y_max_face - y_min_face > 0) and (x_max_face - x_min_face > 0):
h_crop = y_max_face - y_min_face
x_min_face = int(max(0, x_min_face - 0.1 * h_crop))
y_min_face = int(max(0, y_min_face - 0.1 * h_crop))
x_max_face = int(min(self.w, x_min_face + 1.1 * h_crop))
y_max_face = int(min(self.h, y_min_face + 1.1 * h_crop))
bbox_dict["face"] = [y_min_face / self.h, x_min_face / self.w, y_max_face / self.h, x_max_face / self.w]
result.append(bbox_dict)
return result