def append_regions(self, t, regions):
"""Append a list of regions given a timestamp
If tstamp exists in frame_anns, appends regions, otherwise, creates new ImageAnns
Args:
t (float): tstamp
region (Region): region
"""
assert isinstance(t, float)
assert isinstance(regions, list)
for region in regions:
assert isinstance(region, type(Region()))
if t in self._tstamp2frameannsidx:
log.debug(f"t: {t} in frame_anns, extending Regions")
frame_anns_idx = self._tstamp2frameannsidx[t]
self._response_internal["media_annotation"]["frames_annotation"][
frame_anns_idx].extend(regions)
else:
log.debug(f"t: {t} NOT in frame_anns, appending ImageAnn")
ia = ImageAnn(t=t, regions=regions)
self._response_internal["media_annotation"][
"frames_annotation"].append(ia)
self._tstamp2frameannsidx[t] = len(self.frame_anns) - 1
def process_images(self, images, tstamps, detections_of_interest=None):
image_tensor = self._detection_graph.get_tensor_by_name(
'image_tensor:0')
detection_boxes = self._detection_graph.get_tensor_by_name(
'detection_boxes:0')
detection_scores = self._detection_graph.get_tensor_by_name(
'detection_scores:0')
detection_classes = self._detection_graph.get_tensor_by_name(
'detection_classes:0')
num_pred_detections = self._detection_graph.get_tensor_by_name(
'num_detections:0')
for frame, tstamp in zip(images, tstamps):
frame_np = np.array(frame)
frame_np = frame_np[..., ::-1]
frame_np_expanded = np.expand_dims(frame_np, axis=0)
(detections, scores, classes,
num) = self._sess.run([
detection_boxes, detection_scores, detection_classes,
num_pred_detections
],
feed_dict={image_tensor: frame_np_expanded})
num = int(num[0])
for det_idx in range(num):
try:
confidence = float(scores[0, det_idx])
if confidence < CONFIDENCE_MIN:
continue
xmin = float(detections[0, det_idx, 1])
ymin = float(detections[0, det_idx, 0])
xmax = float(detections[0, det_idx, 3])
ymax = float(detections[0, det_idx, 2])
contour = [
Point(xmin, ymin),
Point(xmax, ymin),
Point(xmax, ymax),
Point(xmin, ymax)
]
prop = Property(property_type=self.prop_type,
server=self.server_name,
ver=self.version,
value=str(self.label_map[str(
int(classes[0, det_idx]))]),
confidence=confidence,
confidence_min=CONFIDENCE_MIN)
region = Region(contour, [prop])
self.response.append_region(t=tstamp, region=region)
except Exception as e:
log.error(traceback.format_exc())
def process_images(self, images, tstamps, prev_regions=None):
for frame, tstamp in zip(images, tstamps):
predictions = inference_detector(self.model, frame)
for pred_idx, pred in enumerate(predictions):
try:
for xmin, ymin, xmax, ymax, confidence in pred:
if confidence < self.confidence_min:
continue
label = self.model.CLASSES[pred_idx]
xmin_n = xmin / frame.shape[1]
xmax_n = xmax / frame.shape[1]
ymin_n = ymin / frame.shape[0]
ymax_n = ymax / frame.shape[0]
contour = create_bbox_contour_from_points(xmin_n,
ymin_n,
xmax_n,
ymax_n,
bound=True)
area = compute_box_area(contour)
prop = Property(
confidence=float(confidence),
confidence_min=self.confidence_min,
ver=self.version,
server=self.name,
value=label,
property_type=self.prop_type,
fraction=area,
)
self.response.append_region(t=tstamp,
region=Region(
contour=contour,
props=[prop]))
except Exception:
log.error(traceback.format_exc())
def process_images(self, images, tstamps, prev_detections=None):
for frame, tstamp in zip(images, tstamps):
im = self.transformer.preprocess("data", frame)
self.net.blobs["data"].data[...] = im
predictions = self.net.forward()[LAYER_NAME]
for pred_idx in range(predictions.shape[2]):
try:
confidence = float(predictions[0, 0, pred_idx, 2])
if confidence < self.confidence_min:
continue
index = int(predictions[0, 0, pred_idx, 1])
label = self.labelmap[index]
xmin = float(predictions[0, 0, pred_idx, 3])
ymin = float(predictions[0, 0, pred_idx, 4])
xmax = float(predictions[0, 0, pred_idx, 5])
ymax = float(predictions[0, 0, pred_idx, 6])
contour = create_bbox_contour_from_points(xmin,
ymin,
xmax,
ymax,
bound=True)
area = compute_box_area(contour)
prop = Property(
confidence=confidence,
confidence_min=self.confidence_min,
ver=self.version,
server=self.name,
value=label,
property_type=self.prop_type,
fraction=area,
)
self.response.append_region(t=tstamp,
region=Region(contour=contour,
props=[prop]))
except Exception:
log.error(traceback.format_exc())
def process_images(self, images, tstamps, prev_regions):
# log.debug("Processing images")
# log.debug("tstamps: " + str(tstamps))
assert len(images) == len(tstamps) == len(prev_regions)
for i, (frame, tstamp,
prev_region) in enumerate(zip(images, tstamps, prev_regions)):
log.debug("caffe classifier tstamp: " + str(tstamp))
try:
if prev_region is not None:
frame = crop_image_from_bbox_contour(
frame, prev_region.get("contour"))
im = self.transformer.preprocess("data", frame)
self.net.blobs["data"].data[...] = im
# TODO : clean this up
probs = self.net.forward()[self.layer_name]
# log.debug("probs: " + str(probs))
# log.debug("probs.shape: " + str(probs.shape))
target_shape = (1, len(self.labels))
if (probs.shape == target_shape) is False:
log.debug("Changing shape " + str(probs.shape) + "->" +
str(target_shape))
probs = np.reshape(probs, target_shape)
props = []
for p in probs:
# log.debug('p: ' + str(p))
p_indexes = np.argsort(p)
p_indexes = np.flip(p_indexes, 0)
while True:
if len(p_indexes) == 1:
break
index = p_indexes[0]
label = self.labels[index]
log.debug("label: " + str(label))
if label in LOGOEXCLUDE:
p_indexes = np.delete(p_indexes, 0)
else:
break
p_indexes = p_indexes[:self.top_n]
# log.debug("p_indexes: " + str(p_indexes))
for i, property_id in enumerate(p_indexes):
if i == self.top_n:
break
index = p_indexes[i]
label = self.labels[index]
confidence = p[index]
# TODO remove this unknown
if confidence < self.confidence_min:
label = "Unknown"
prop = Property(
server=self.name,
ver=self.version,
value=label,
property_type=self.prop_type,
confidence=float(confidence),
confidence_min=float(self.confidence_min),
)
if prev_region is not None:
prev_region.get("props").append(prop)
else:
props.append(prop)
if prev_region is None:
self.response.append_region(t=tstamp,
region=Region(props=props))
except Exception as e:
log.error(traceback.print_exc())
log.error(e)