def __init__(self, model, weights):
self._model = model
self._weights = weights
IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH")
if not IE_PLUGINS_PATH:
raise OSError(
"Inference engine plugin path env not found in the system.")
plugin = make_plugin()
network = make_network(self._model, self._weights)
supported_layers = plugin.get_supported_layers(network)
not_supported_layers = [
l for l in network.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
raise Exception(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(plugin.device, ", ".join(not_supported_layers)))
self._input_blob_name = next(iter(network.inputs))
self._output_blob_name = next(iter(network.outputs))
self._net = plugin.load(network=network, num_requests=2)
input_type = network.inputs[self._input_blob_name]
self._input_layout = input_type if isinstance(
input_type, list) else input_type.shape
def run_inference_engine_annotation(image_list, labels_mapping, treshold):
from cvat.apps.auto_annotation.inference_engine import make_plugin_or_core, make_network
def _normalize_box(box, w, h, dw, dh):
xmin = min(int(box[0] * dw * w), w)
ymin = min(int(box[1] * dh * h), h)
xmax = min(int(box[2] * dw * w), w)
ymax = min(int(box[3] * dh * h), h)
return xmin, ymin, xmax, ymax
result = {}
MODEL_PATH = os.environ.get('TF_ANNOTATION_MODEL_PATH')
if MODEL_PATH is None:
raise OSError('Model path env not found in the system.')
core_or_plugin = make_plugin_or_core()
network = make_network('{}.xml'.format(MODEL_PATH), '{}.bin'.format(MODEL_PATH))
input_blob_name = next(iter(network.inputs))
output_blob_name = next(iter(network.outputs))
if getattr(core_or_plugin, 'load_network', False):
executable_network = core_or_plugin.load_network(network, 'CPU')
else:
executable_network = core_or_plugin.load(network=network)
job = rq.get_current_job()
del network
try:
for image_num, im_name in enumerate(image_list):
job.refresh()
if 'cancel' in job.meta:
del job.meta['cancel']
job.save()
return None
job.meta['progress'] = image_num * 100 / len(image_list)
job.save_meta()
image = Image.open(im_name)
width, height = image.size
image.thumbnail((600, 600), Image.ANTIALIAS)
dwidth, dheight = 600 / image.size[0], 600 / image.size[1]
image = image.crop((0, 0, 600, 600))
image_np = load_image_into_numpy(image)
image_np = np.transpose(image_np, (2, 0, 1))
prediction = executable_network.infer(inputs={input_blob_name: image_np[np.newaxis, ...]})[output_blob_name][0][0]
for obj in prediction:
obj_class = int(obj[1])
obj_value = obj[2]
if obj_class and obj_class in labels_mapping and obj_value >= treshold:
label = labels_mapping[obj_class]
if label not in result:
result[label] = []
xmin, ymin, xmax, ymax = _normalize_box(obj[3:7], width, height, dwidth, dheight)
result[label].append([image_num, xmin, ymin, xmax, ymax])
finally:
del executable_network
del plugin
return result
def __init__(self, model, weights):
self._model = model
self._weights = weights
core_or_plugin = make_plugin_or_core()
network = make_network(self._model, self._weights)
if getattr(core_or_plugin, 'get_supported_layers', False):
supported_layers = core_or_plugin.get_supported_layers(network)
not_supported_layers = [
l for l in network.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
raise Exception(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(core_or_plugin.device,
", ".join(not_supported_layers)))
iter_inputs = iter(network.inputs)
self._input_blob_name = next(iter_inputs)
self._input_info_name = ''
self._output_blob_name = next(iter(network.outputs))
self._require_image_info = False
info_names = ('image_info', 'im_info')
# NOTE: handeling for the inclusion of `image_info` in OpenVino2019
if any(s in network.inputs for s in info_names):
self._require_image_info = True
self._input_info_name = set(
network.inputs).intersection(info_names)
self._input_info_name = self._input_info_name.pop()
if self._input_blob_name in info_names:
self._input_blob_name = next(iter_inputs)
if getattr(core_or_plugin, 'load_network', False):
self._net = core_or_plugin.load_network(network,
"CPU",
num_requests=2)
else:
self._net = core_or_plugin.load(network=network, num_requests=2)
input_type = network.inputs[self._input_blob_name]
self._input_layout = input_type if isinstance(
input_type, list) else input_type.shape
def __init__(self, model, weights):
self._model = model
self._weights = weights
IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH")
if not IE_PLUGINS_PATH:
raise OSError(
"Inference engine plugin path env not found in the system.")
plugin = make_plugin()
network = make_network(self._model, self._weights)
supported_layers = plugin.get_supported_layers(network)
not_supported_layers = [
l for l in network.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
raise Exception(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(plugin.device, ", ".join(not_supported_layers)))
iter_inputs = iter(network.inputs)
self._input_blob_name = next(iter_inputs)
self._input_info_name = ''
self._output_blob_name = next(iter(network.outputs))
self._require_image_info = False
info_names = ('image_info', 'im_info')
# NOTE: handeling for the inclusion of `image_info` in OpenVino2019
if any(s in network.inputs for s in info_names):
self._require_image_info = True
self._input_info_name = set(
network.inputs).intersection(info_names)
self._input_info_name = self._input_info_name.pop()
if self._input_blob_name in info_names:
self._input_blob_name = next(iter_inputs)
self._net = plugin.load(network=network, num_requests=2)
input_type = network.inputs[self._input_blob_name]
self._input_layout = input_type if isinstance(
input_type, list) else input_type.shape
def handle(self, db_data, frame, points):
# Lazy initialization
if not self._plugin:
self._plugin = make_plugin_or_core()
self._network = make_network(
os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml'),
os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin'))
self._input_blob = next(iter(self._network.inputs))
self._output_blob = next(iter(self._network.outputs))
if getattr(self._plugin, 'load_network', False):
self._exec_network = self._plugin.load_network(
self._network, 'CPU')
else:
self._exec_network = self._plugin.load(network=self._network)
frame_provider = FrameProvider(db_data)
image = frame_provider.get_frame(frame,
frame_provider.Quality.ORIGINAL)
image = PIL.Image.open(image[0])
numpy_image = np.array(image)
points = np.asarray([[int(p["x"]), int(p["y"])] for p in points],
dtype=int)
bounding_box = (max(min(points[:, 0]) - _DEXTR_PADDING,
0), max(min(points[:, 1]) - _DEXTR_PADDING, 0),
min(
max(points[:, 0]) + _DEXTR_PADDING,
numpy_image.shape[1] - 1),
min(
max(points[:, 1]) + _DEXTR_PADDING,
numpy_image.shape[0] - 1))
# Prepare an image
numpy_cropped = np.array(image.crop(bounding_box))
resized = cv2.resize(numpy_cropped, (_DEXTR_SIZE, _DEXTR_SIZE),
interpolation=cv2.INTER_CUBIC).astype(np.float32)
# Make a heatmap
points = points - [min(points[:, 0]),
min(points[:, 1])
] + [_DEXTR_PADDING, _DEXTR_PADDING]
points = (points * [
_DEXTR_SIZE / numpy_cropped.shape[1],
_DEXTR_SIZE / numpy_cropped.shape[0]
]).astype(int)
heatmap = np.zeros(shape=resized.shape[:2], dtype=np.float64)
for point in points:
gaussian_x_axis = np.arange(0, _DEXTR_SIZE, 1, float) - point[0]
gaussian_y_axis = np.arange(0, _DEXTR_SIZE, 1,
float)[:, np.newaxis] - point[1]
gaussian = np.exp(
-4 * np.log(2) *
((gaussian_x_axis**2 + gaussian_y_axis**2) / 100)).astype(
np.float64)
heatmap = np.maximum(heatmap, gaussian)
cv2.normalize(heatmap, heatmap, 0, 255, cv2.NORM_MINMAX)
# Concat an image and a heatmap
input_dextr = np.concatenate(
(resized, heatmap[:, :, np.newaxis].astype(resized.dtype)), axis=2)
input_dextr = input_dextr.transpose((2, 0, 1))
pred = self._exec_network.infer(
inputs={self._input_blob: input_dextr[np.newaxis, ...]})[
self._output_blob][0, 0, :, :]
pred = cv2.resize(pred,
tuple(reversed(numpy_cropped.shape[:2])),
interpolation=cv2.INTER_CUBIC)
result = np.zeros(numpy_image.shape[:2])
result[bounding_box[1]:bounding_box[1] + pred.shape[0],
bounding_box[0]:bounding_box[0] +
pred.shape[1]] = pred > _DEXTR_TRESHOLD
# Convert a mask to a polygon
result = np.array(result, dtype=np.uint8)
cv2.normalize(result, result, 0, 255, cv2.NORM_MINMAX)
contours = None
if int(cv2.__version__.split('.')[0]) > 3:
contours = cv2.findContours(result, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_TC89_KCOS)[0]
else:
contours = cv2.findContours(result, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_TC89_KCOS)[1]
contours = max(contours, key=lambda arr: arr.size)
if contours.shape.count(1):
contours = np.squeeze(contours)
if contours.size < 3 * 2:
raise Exception(
'Less then three point have been detected. Can not build a polygon.'
)
result = ""
for point in contours:
result += "{},{} ".format(int(point[0]), int(point[1]))
result = result[:-1]
return result