def set_cpu_extensions(core: IECore, cpu_ext: str):
core.add_extension(cpu_ext, "CPU")
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Loading Inference Engine")
ie = IECore()
# ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format ---
model = args.model
log.info(f"Loading network:\n\t{model}")
net = ie.read_network(model=model)
func = ng.function_from_cnn(net)
ops = func.get_ordered_ops()
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
# -----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
print("inputs number: " + str(len(net.input_info.keys())))
for input_key in net.input_info:
print("input shape: " +
str(net.input_info[input_key].input_data.shape))
print("input key: " + input_key)
if len(net.input_info[input_key].input_data.layout) == 4:
n, c, h, w = net.input_info[input_key].input_data.shape
images = np.ndarray(shape=(n, c, h, w))
images_hw = []
for i in range(n):
image = cv2.imread(args.input)
ih, iw = image.shape[:-1]
images_hw.append((ih, iw))
log.info("File was added: ")
log.info(" {}".format(args.input[i]))
if (ih, iw) != (h, w):
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = cv2.resize(image, (w, h))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.input_info.keys()) == 1 or len(net.input_info.keys())
== 2), "Sample supports topologies only with 1 or 2 inputs"
out_blob = next(iter(net.outputs))
input_name, input_info_name = "", ""
for input_key in net.input_info:
if len(net.input_info[input_key].layout) == 4:
input_name = input_key
net.input_info[input_key].precision = 'U8'
elif len(net.input_info[input_key].layout) == 2:
input_info_name = input_key
net.input_info[input_key].precision = 'FP32'
if net.input_info[input_key].input_data.shape[1] != 3 and net.input_info[input_key].input_data.shape[1] != 6 or \
net.input_info[input_key].input_data.shape[0] != 1:
log.error(
'Invalid input info. Should be 3 or 6 values length.')
data = {}
data[input_name] = images
if input_info_name != "":
infos = np.ndarray(shape=(n, c), dtype=float)
for i in range(n):
infos[i, 0] = h
infos[i, 1] = w
infos[i, 2] = 1.0
data[input_info_name] = infos
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
output_ops = {op.friendly_name : op for op in ops \
if op.friendly_name in net.outputs and op.get_type_name() == "DetectionOutput"}
if len(output_ops) != 0:
output_name, output_info = output_ops.popitem()
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
log.info("Creating infer request and starting inference")
res = exec_net.infer(inputs=data)
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info("Processing output blobs")
res = res[out_blob]
boxes, classes = {}, {}
data = res[0][0]
for number, proposal in enumerate(data):
if proposal[2] > 0:
imid = np.int(proposal[0])
ih, iw = images_hw[imid]
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(iw * proposal[3])
ymin = np.int(ih * proposal[4])
xmax = np.int(iw * proposal[5])
ymax = np.int(ih * proposal[6])
print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}" \
.format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="")
if proposal[2] > 0.5:
print(" WILL BE PRINTED!")
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
if not imid in classes.keys():
classes[imid] = []
classes[imid].append(label)
else:
print()
for imid in classes:
tmp_image = cv2.imread(args.input[imid])
for box in boxes[imid]:
cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]),
(232, 35, 244), 2)
cv2.imwrite("out.bmp", tmp_image)
log.info("Image out.bmp created!")
# -----------------------------------------------------------------------------------------------------
log.info("Execution successful\n")
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool"
)
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self.net = None
self.plugin = None
self.input_blob = None
self.out_blob = None
self.net_plugin = None
self.infer_request_handle = None
def load_model(self,
model,
device,
input_size,
output_size,
num_requests,
cpu_extension=None,
plugin=None):
"""
Loads a network and an image to the Inference Engine plugin.
:param model: .xml file of pre trained model
:param cpu_extension: extension for the CPU device
:param device: Target device
:param input_size: Number of input layers
:param output_size: Number of output layers
:param num_requests: Index of Infer request value. Limited to device capabilities.
:param plugin: Plugin for specified device
:return: Shape of input layer
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device
# and load extensions library if specified
if not plugin:
log.info("Initializing plugin for {} device...".format(device))
self.plugin = IECore()
else:
self.plugin = plugin
if cpu_extension and 'CPU' in device:
self.plugin.add_extension(cpu_extension, "CPU")
# Read IR
log.info("Reading IR...")
self.net = IENetwork(model=model_xml, weights=model_bin)
log.info("Loading IR to the plugin...")
if "CPU" in device:
supported_layers = self.plugin.query_network(self.net, "CPU")
not_supported_layers = \
[l for l in self.net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by "
"the plugin for specified device {}:\n {}".format(
device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path"
" in command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
if num_requests == 0:
# Loads network read from IR to the plugin
self.net_plugin = self.plugin.load_network(network=self.net,
device_name=device)
else:
self.net_plugin = self.plugin.load_network(
network=self.net,
num_requests=num_requests,
device_name=device)
# log.error("num_requests != 0")
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
assert len(self.net.inputs.keys()) == input_size, \
"Supports only {} input topologies".format(len(self.net.inputs))
assert len(self.net.outputs) == output_size, \
"Supports only {} output topologies".format(len(self.net.outputs))
return self.plugin, self.get_input_shape()
def get_input_shape(self):
"""
Gives the shape of the input layer of the network.
:return: None
"""
return self.net.inputs[self.input_blob].shape
def performance_counter(self, request_id):
"""
Queries performance measures per layer to get feedback of what is the
most time consuming layer.
:param request_id: Index of Infer request value. Limited to device capabilities
:return: Performance of the layer
"""
perf_count = self.net_plugin.requests[request_id].get_perf_counts()
return perf_count
def exec_net(self, request_id, frame):
"""
Starts asynchronous inference for specified request.
:param request_id: Index of Infer request value. Limited to device capabilities.
:param frame: Input image
:return: Instance of Executable Network class
"""
self.infer_request_handle = self.net_plugin.start_async(
request_id=request_id, inputs={self.input_blob: frame})
return self.net_plugin
def wait(self, request_id):
"""
Waits for the result to become available.
:param request_id: Index of Infer request value. Limited to device capabilities.
:return: Timeout value
"""
wait_process = self.net_plugin.requests[request_id].wait(-1)
return wait_process
def get_output(self, request_id, output=None):
"""
Gives a list of results for the output layer of the network.
:param request_id: Index of Infer request value. Limited to device capabilities.
:param output: Name of the output layer
:return: Results for the specified request
"""
if output:
res = self.infer_request_handle.outputs[output]
else:
res = self.net_plugin.requests[request_id].outputs[self.out_blob]
return res
def clean(self):
"""
Deletes all the instances
:return: None
"""
del self.net_plugin
del self.plugin
del self.net
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.net_plugin = None
self.infer_request = None
self.async_infer_handler = None
def load_model(self, model, device="CPU", cpu_extension=None, console_output= False):
### TODO: Load the model ###
# Initialize the plugin
self.plugin = IECore()
# Read the IR as a IENetwork
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.network = IENetwork(model=model_xml, weights=model_bin)
### TODO: Check for supported layers ###
#get supported layers
supported_layers = self.plugin.query_network(self.network, device)
# Check for any unsupported layers, and let the user
# know if anything is missing. Exit the program, if so.
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(unsupported_layers) != 0:
# log.error("Unsupported layers found: {}".format(unsupported_layers))
# log.error("Check whether extensions are available to add to IECore.")
# exit(1)
### TODO: Add any necessary extensions ###
# Add a CPU extension, if applicable
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Load the IENetwork into the plugin
self.net_plugin = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
input_shapes = {input:self.network.inputs[input].shape for input in self.network.inputs}
return input_shapes
def exec_net(self, request_id, net_input):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.async_infer_handler = self.net_plugin.start_async(
request_id=request_id, inputs=net_input)
return
def wait(self, request_id):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.net_plugin.requests[request_id].wait(-1)
def get_output(self, request_id, output=None):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
if output:
return self.async_infer_handler.outputs[output]
else:
return self.net_plugin.requests[request_id].outputs[self.output_blob]
def clean(self):
"""
Deletes all the instances
:return: None
"""
del self.net_plugin
del self.plugin
del self.network
def main():
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# Plugin initialization for specified device and load extensions library if specified.
log.info('Creating Inference Engine...')
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, 'CPU')
# Read IR
log.info('Loading Mask-RCNN network')
mask_rcnn_net = ie.read_network(
args.mask_rcnn_model,
os.path.splitext(args.mask_rcnn_model)[0] + '.bin')
log.info('Loading encoder part of text recognition network')
text_enc_net = ie.read_network(
args.text_enc_model,
os.path.splitext(args.text_enc_model)[0] + '.bin')
log.info('Loading decoder part of text recognition network')
text_dec_net = ie.read_network(
args.text_dec_model,
os.path.splitext(args.text_dec_model)[0] + '.bin')
if 'CPU' in args.device:
supported_layers = ie.query_network(mask_rcnn_net, 'CPU')
not_supported_layers = [
l for l in mask_rcnn_net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
'Following layers are not supported by the plugin for specified device {}:\n {}'
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
required_input_keys = {'im_data', 'im_info'}
assert required_input_keys == set(mask_rcnn_net.inputs.keys()), \
'Demo supports only topologies with the following input keys: {}'.format(', '.join(required_input_keys))
required_output_keys = {
'boxes', 'scores', 'classes', 'raw_masks', 'text_features'
}
assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \
'Demo supports only topologies with the following output keys: {}'.format(', '.join(required_output_keys))
n, c, h, w = mask_rcnn_net.inputs['im_data'].shape
assert n == 1, 'Only batch 1 is supported by the demo application'
log.info('Loading IR to the plugin...')
mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net,
device_name=args.device,
num_requests=2)
text_enc_exec_net = ie.load_network(network=text_enc_net,
device_name=args.device)
text_dec_exec_net = ie.load_network(network=text_dec_net,
device_name=args.device)
hidden_shape = text_dec_net.inputs[args.trd_input_prev_hidden].shape
del mask_rcnn_net
del text_enc_net
del text_dec_net
try:
input_source = int(args.input_source)
except ValueError:
input_source = args.input_source
if os.path.isdir(input_source):
cap = FolderCapture(input_source)
else:
cap = cv2.VideoCapture(input_source)
if not cap.isOpened():
log.error('Failed to open "{}"'.format(args.input_source))
if isinstance(cap, cv2.VideoCapture):
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
if args.no_track:
tracker = None
else:
tracker = StaticIOUTracker()
visualizer = Visualizer(['__background__', 'text'],
show_boxes=args.show_boxes,
show_scores=args.show_scores)
render_time = 0
log.info('Starting inference...')
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
if not args.keep_aspect_ratio:
# Resize the image to a target size.
scale_x = w / frame.shape[1]
scale_y = h / frame.shape[0]
input_image = cv2.resize(frame, (w, h))
else:
# Resize the image to keep the same aspect ratio and to fit it to a window of a target size.
scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1])
input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y)
input_image_size = input_image.shape[:2]
input_image = np.pad(input_image,
((0, h - input_image_size[0]),
(0, w - input_image_size[1]), (0, 0)),
mode='constant',
constant_values=0)
# Change data layout from HWC to CHW.
input_image = input_image.transpose((2, 0, 1))
input_image = input_image.reshape((n, c, h, w)).astype(np.float32)
input_image_info = np.asarray(
[[input_image_size[0], input_image_size[1], 1]], dtype=np.float32)
# Run the net.
inf_start = time.time()
outputs = mask_rcnn_exec_net.infer({
'im_data': input_image,
'im_info': input_image_info
})
# Parse detection results of the current request
boxes = outputs['boxes']
scores = outputs['scores']
classes = outputs['classes'].astype(np.uint32)
raw_masks = outputs['raw_masks']
text_features = outputs['text_features']
# Filter out detections with low confidence.
detections_filter = scores > args.prob_threshold
scores = scores[detections_filter]
classes = classes[detections_filter]
boxes = boxes[detections_filter]
raw_masks = raw_masks[detections_filter]
text_features = text_features[detections_filter]
boxes[:, 0::2] /= scale_x
boxes[:, 1::2] /= scale_y
masks = []
for box, cls, raw_mask in zip(boxes, classes, raw_masks):
raw_cls_mask = raw_mask[cls, ...]
mask = segm_postprocess(box, raw_cls_mask, frame.shape[0],
frame.shape[1])
masks.append(mask)
texts = []
for feature in text_features:
feature = text_enc_exec_net.infer({'input': feature})['output']
feature = np.reshape(feature,
(feature.shape[0], feature.shape[1], -1))
feature = np.transpose(feature, (0, 2, 1))
hidden = np.zeros(hidden_shape)
prev_symbol_index = np.ones((1, )) * SOS_INDEX
text = ''
for i in range(MAX_SEQ_LEN):
decoder_output = text_dec_exec_net.infer({
args.trd_input_prev_symbol:
prev_symbol_index,
args.trd_input_prev_hidden:
hidden,
args.trd_input_encoder_outputs:
feature
})
symbols_distr = decoder_output[args.trd_output_symbols_distr]
prev_symbol_index = int(np.argmax(symbols_distr, axis=1))
if prev_symbol_index == EOS_INDEX:
break
text += args.alphabet[prev_symbol_index]
hidden = decoder_output[args.trd_output_cur_hidden]
texts.append(text)
inf_end = time.time()
inf_time = inf_end - inf_start
render_start = time.time()
if len(boxes) and args.raw_output_message:
log.info('Detected boxes:')
log.info(
' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX '
)
for box, cls, score, mask in zip(boxes, classes, scores, masks):
log.info(
'{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} '
.format(cls, score, *box))
# Get instance track IDs.
masks_tracks_ids = None
if tracker is not None:
masks_tracks_ids = tracker(masks, classes)
# Visualize masks.
frame = visualizer(frame, boxes, classes, scores, masks, texts,
masks_tracks_ids)
# Draw performance stats.
inf_time_message = 'Inference and post-processing time: {:.3f} ms'.format(
inf_time * 1000)
render_time_message = 'OpenCV rendering time: {:.3f} ms'.format(
render_time * 1000)
cv2.putText(frame, inf_time_message, (15, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
cv2.putText(frame, render_time_message, (15, 30),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
# Print performance counters.
if args.perf_counts:
perf_counts = mask_rcnn_exec_net.requests[0].get_perf_counts()
log.info('Performance counters:')
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
'name', 'layer_type', 'exet_type', 'status', 'real_time, us'))
for layer, stats in perf_counts.items():
print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format(
layer, stats['layer_type'], stats['exec_type'],
stats['status'], stats['real_time']))
if not args.no_show:
# Show resulting image.
cv2.imshow('Results', frame)
render_end = time.time()
render_time = render_end - render_start
if not args.no_show:
key = cv2.waitKey(args.delay)
esc_code = 27
if key == esc_code:
break
cv2.destroyAllWindows()
cap.release()
class HeadPoseModel:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model_name = model_name
self.device = device
self.extensions = extensions
self.model_structure = self.model_name
self.model_weights = self.model_name.split('.')[0]+'.bin'
self.core = None
self.network = None
self.exec_net = None
self.input = None
self.output = None
self.mode = 'async'
self.request_id = 0
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.core = IECore()
self.network = self.core.read_network(model=self.model_structure, weights=self.model_weights)
self.exec_net = self.core.load_network(network=self.network, device_name=self.device,num_requests=self.num_requests)
self.input = next(iter(self.network.inputs))
self.output = next(iter(self.network.outputs))
return self.exec_net
def predict(self, image):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
processed_frame = self.preprocess_input(image)
self.exec_net.start_async(request_id=self.request_id,inputs={self.input: processed_frame})
self.exec_net.requests[0].get_perf_counts()
if self.mode == 'async':
self.exec_net.requests[0].wait()
return self.preprocess_output(self.exec_net.requests[0].outputs)
else:
if self.exec_net.requests[0].wait(-1) == 0:
return self.preprocess_output(self.exec_net.requests[0].outputs)
def check_model(self):
supported_layers = self.core.query_network(network=self.network, device_name=self.device)
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
# if len(unsupported_layers) > 0:
# print("Please check extention for these unsupported layers =>" + str(unsupported_layers))
# exit(1)
# print("All layers are supported !!")
if len(unsupported_layers)!=0 and self.device=='CPU':
print("unsupported layers found:{}".format(unsupported_layers))
if not self.extensions==None:
print("Adding cpu_extension")
self.core.add_extension(self.extensions, self.device)
supported_layers = self.core.query_network(network = self.network, device_name=self.device)
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(unsupported_layers)!=0:
print("After adding the extension still unsupported layers found")
exit(1)
print("After adding the extension the issue is resolved")
else:
print("Give the path of cpu extension")
exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
model_input_shape = self.network.inputs[self.input].shape
image_resized = cv2.resize(image, (model_input_shape[3], model_input_shape[2]))
p_frame = np.transpose(np.expand_dims(image_resized,axis=0), (0,3,1,2))
return p_frame
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
return np.array([outputs['angle_y_fc'][0][0], outputs['angle_p_fc'][0][0], outputs['angle_r_fc'][0][0]])
class Model_FaceDetection:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU',extension = None):
self.net = None
self.net_plug = None
self.inp_name = None
self.out_name = None
self.inp_shape = None
self.out_shape = None
self.model = model_name
self.device = device
self.ext = extension
self.weights = self.model.split('.')[0]+'.bin'
def load_model(self,plugin=None):
if not plugin:
self.plugin = IECore()
else:
self.plugin = plugin
self.net = IENetwork(model = self.model, weights = self.weights)
if not self.ext == None and self.device == 'CPU':
print("CPU extension added")
self.plugin.add_extension(self.ext, 'CPU')
if self.device == 'CPU':
supp_layer = self.plugin.query_network(network = self.net, device_name = self.device)
no_supp_layer = [lay for lay in self.net.layers.keys() if lay not in supp_layer]
if len(no_supp_layer) != 0:
print("Non supported layers found! Please add another CPU extension ")
exit(1)
print("Issue resolved after adding extension")
else:
print("CPU extension needed")
exit(1)
self.net_plug = self.plugin.load_network(network = self.net, device_name = self.device, num_requests = 1)
self.inp_name = next(iter(self.net.inputs))
self.out_name = next(iter(self.net.outputs))
self.inp_shape = self.net.inputs[self.inp_name].shape
def predict(self, frame, prob_threshold):
processed_frame = self.preprocess_input(frame.copy())
out = self.net_plug.infer({self.inp_name : processed_frame})
out = self.preprocess_output(out, prob_threshold)
if (len(out) == 0):
return 0, 0
out = out[0]
ht = frame.shape[0]
wd =frame.shape[1]
out = out* np.array([wd, ht, wd, ht])
out = out.astype(np.int32)
cropped = frame[out[1]:out[3], out[0]:out[2]]
return cropped, out
def check_model(self):
pass
def preprocess_input(self, frame):
h = self.inp_shape[2]
w = self.inp_shape[3]
reshaped_frame = cv2.resize(frame, (w, h))
reshaped_frame = reshaped_frame.transpose((2,0,1))
reshaped_frame = reshaped_frame.reshape(1, 3, h, w)
return reshaped_frame
def preprocess_output(self, out, prob_threshold):
dim = []
out = out[self.out_name][0][0]
for cell in out:
conf = cell[2]
if conf >= prob_threshold:
xmin = cell[3]
xmax = cell[5]
ymin = cell[4]
ymax = cell[6]
dim.append([xmin, ymin, xmax, ymax])
return dim
class GazeEstimationModel:
'''
Class for the Gaze Est Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
self.plugin = None
self.network = None
self.exec_network = None
self.input_name = None
self.output_name = None
self.input_shape = None
self.output_shape = None
self.infer_req = None
self.device = device
self.extensions = extensions
self.model_path = model_name
def load_model(self):
model_xml = self.model_path
model_bin = os.path.splitext(self.model_path)[0] + ".bin"
self.plugin = IECore()
# network model
self.network = self.plugin.read_network(model=model_xml,
weights=model_bin)
network_layers = self.network.layers.keys()
supported_layers = self.plugin.query_network(
network=self.network, device_name=self.device).keys()
ext_required = False
for layer in network_layers:
if layer in supported_layers:
pass
else:
ext_required = True
break
# cpu extension added
if self.extensions != None and "CPU" in self.device and ext_required:
self.plugin.add_extension(self.extensions, self.device)
for layer in network_layers:
if layer in supported_layers:
pass
else:
msg = "Layer extension doesn't support all layers"
log.error(msg)
raise Exception(msg)
self.exec_network = self.plugin.load_network(self.network, self.device)
self.input_name = [i for i in self.network.inputs.keys()]
self.input_shape = self.network.inputs[self.input_name[1]].shape
self.output_name = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_name].shape
return
def predict(self, left_eye, right_eye, head_pose, preview=None):
input_left_eye, input_right_eye = self.preprocess_input(
left_eye, right_eye)
input_dict = {
'left_eye_image': input_left_eye,
'right_eye_image': input_right_eye,
'head_pose_angles': head_pose
}
outputs = self.exec_network.infer(input_dict)[self.output_name]
if preview is not None:
return self.preprocess_outputs(outputs[0], head_pose[2], preview)
return self.preprocess_outputs(outputs[0], head_pose[2])
def preprocess_input(self, left_eye, right_eye):
preprocessed_left_eye = cv2.resize(
left_eye, (self.input_shape[3], self.input_shape[2]))
preprocessed_left_eye = preprocessed_left_eye.transpose((2, 0, 1))
preprocessed_right_eye = cv2.resize(
right_eye, (self.input_shape[3], self.input_shape[2]))
preprocessed_right_eye = preprocessed_right_eye.transpose((2, 0, 1))
return preprocessed_left_eye.reshape(
1, *preprocessed_left_eye.shape), preprocessed_right_eye.reshape(
1, *preprocessed_right_eye.shape)
def preprocess_outputs(self, outputs, roll, preview=None):
cos = math.cos(roll * math.pi / 180.0)
sin = math.sin(roll * math.pi / 180.0)
x = outputs[0] * cos + outputs[1] * sin
y = outputs[0] * sin + outputs[1] * cos
if preview is not None:
cv2.putText(
preview, "Gaze Coord: x:{:.2f}|| y:{:.2f} || z:{:.2f}".format(
outputs[0], outputs[1], outputs[2]), (10, 40),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (0, 255, 0), 1)
return (x, y), outputs, preview
return (x, y), outputs
class Network:
'''
Load and store information for working with the Inference Engine,
and any loaded models.
'''
def __init__(self):
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device="CPU", cpu_extension=None):
'''
Load the model given IR files.
Defaults to CPU as device for use in the workspace.
Synchronous requests made within.
'''
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Initialize the plugin
self.plugin = IECore()
# Add a CPU extension, if applicable
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
self.network = IENetwork(model=model_xml, weights=model_bin)
# Load the IENetwork into the plugin
self.exec_network = self.plugin.load_network(self.network,
device,
num_requests=201)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
return
def get_input_shape(self):
'''
Gets the input shape of the network
'''
return self.network.inputs[self.input_blob].shape
def async_inference(self, rqsId, image):
'''
Makes an asynchronous inference request, given an input image.
'''
### TODO: Start asynchronous inference
self.infer_request = self.exec_network.start_async(
rqsId, {self.input_blob: image})
return self.infer_request
def wait(self):
'''
Checks the status of the inference request.
'''
### TODO: Wait for the async request to be complete
status = self.infer_request.wait()
return status
def extract_output(self):
'''
Returns a list of the results for the output layer of the network.
'''
### TODO: Return the outputs of the network from the output_blob
return self.infer_request.outputs['output_blob']
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
#-----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
input_blob = next(iter(net.inputs))
n, c, h, w = net.inputs[input_blob].shape
images = np.ndarray(shape=(n, c, h, w))
images_hw = []
for i in range(n):
image = cv2.imread(args.input[i])
ih, iw = image.shape[:-1]
images_hw.append((ih, iw))
log.info("File was added: ")
log.info(" {}".format(args.input[i]))
if (ih, iw) != (h, w):
image = cv2.resize(image, (w, h))
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1 or len(net.inputs.keys())
== 2), "Sample supports topologies only with 1 or 2 inputs"
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
input_name, input_info_name = "", ""
for input_key in net.inputs:
if len(net.inputs[input_key].layout) == 4:
input_name = input_key
log.info("Batch size is {}".format(net.batch_size))
net.inputs[input_key].precision = 'U8'
elif len(net.inputs[input_key].layout) == 2:
input_info_name = input_key
net.inputs[input_key].precision = 'FP32'
if net.inputs[input_key].shape[1] != 3 and net.inputs[input_key].shape[1] != 6 or \
net.inputs[input_key].shape[0] != 1:
log.error(
'Invalid input info. Should be 3 or 6 values length.')
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
for output_key in net.outputs:
if net.layers[output_key].type == "DetectionOutput":
output_name, output_info = output_key, net.outputs[output_key]
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
log.info("Creating infer request and starting inference")
res = exec_net.infer(inputs={input_blob: images})
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info("Processing output blobs")
res = res[out_blob]
boxes, classes = {}, {}
data = res[0][0]
for number, proposal in enumerate(data):
if proposal[2] > 0:
imid = np.int(proposal[0])
ih, iw = images_hw[imid]
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(iw * proposal[3])
ymin = np.int(ih * proposal[4])
xmax = np.int(iw * proposal[5])
ymax = np.int(ih * proposal[6])
print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}" \
.format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="")
if proposal[2] > 0.5:
print(" WILL BE PRINTED!")
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
if not imid in classes.keys():
classes[imid] = []
classes[imid].append(label)
else:
print()
for imid in classes:
tmp_image = cv2.imread(args.input[imid])
for box in boxes[imid]:
cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]),
(232, 35, 244), 2)
cv2.imwrite("out.bmp", tmp_image)
log.info("Image out.bmp created!")
# -----------------------------------------------------------------------------------------------------
log.info("Execution successful\n")
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool"
)
class GazeEstimationClass:
def __init__(self, model_name, device='CPU', extensions=None):
self.model_weights = model_name + '.bin'
self.model_structure = model_name + '.xml'
self.device = device
self.extension = extensions
self.network = None
self.plugin = None
self.exec_net = None
def load_model(self):
self.plugin = IECore()
self.check_model(self.model_structure, self.model_weights)
if not all_layers_supported(self.plugin, self.network):
self.plugin.add_extension(self.extension, self.device)
# Load model
self.exec_net = self.plugin.load_network(network = self.network, device_name= self.device, num_requests=1)
self.output_name = next(iter(self.network.outputs))
def predict(self, left_eye, right_eye, head_pose_out):
p_left_eye, p_right_eye = self.preprocess_input(left_eye, right_eye)
results = self.exec_net.infer({'head_pose_angles': head_pose_out, 'left_eye_image':p_left_eye, 'right_eye_image': p_right_eye})
mouse_coordinate, gaze_vector = self.preprocess_output(results, head_pose_out)
return mouse_coordinate, gaze_vector
def check_model(self, model_structure, model_weights):
try:
self.network = IENetwork(model_structure, model_weights)
except Exception as e:
raise ValueError("Could not Initialize Network")
def preprocess_input(self, left_eye, right_eye):
# model requires shape [1x3x60x60]
p_left_eye = cv2.resize(left_eye, (60, 60))
p_left_eye = p_left_eye.transpose((2, 0, 1))
p_left_eye = p_left_eye.reshape(1, *p_left_eye.shape)
p_right_eye = cv2.resize(right_eye, (60, 60))
p_right_eye = p_right_eye.transpose((2, 0, 1))
p_right_eye = p_right_eye.reshape(1, *p_right_eye.shape)
return p_left_eye, p_right_eye
def preprocess_output(self, outputs, head_pose_out):
# Calculate x,y for mouse movement
gaze_vector = outputs[self.output_name][0]
angle_r_fc = head_pose_out[2]
cosine = math.cos(angle_r_fc * math.pi / 180)
sine = math.sin(angle_r_fc * math.pi / 180)
x_value = gaze_vector[0] * cosine + gaze_vector[1] * sine
y_value = - gaze_vector[0] * sine + gaze_vector[1] * cosine
return (x_value, y_value), gaze_vector
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.net = None
self.exec_netw = None
self.input_blob = None
self.output_blob = None
self.infer_request = None
def load_model(self, model, device, cpu_extension):
### TODO: Load the model ###
### TODO: Check for supported layers ###
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
model_xml = model
model_bin = model.split('.')[0] + '.bin'
self.plugin = IECore()
if device == 'CPU':
self.plugin.add_extension(cpu_extension, device)
self.net = IENetwork(model=model_xml, weights=model_bin)
self.exec_netw = self.plugin.load_network(self.net, device)
sl = self.plugin.query_network(self.net, device_name=device)
ul = [l for l in self.net.layers.keys() if l not in sl]
if len(ul) != 0:
print('Unsupported layers found:{}'.format(ul))
print(
'Check for any extensions for these unsupported layers available for adding to IECore'
)
exit(1)
self.input_blob = next(iter(self.net.inputs))
self.output_blob = next(iter(self.net.outputs))
return self.exec_netw
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.net.inputs[self.input_blob].shape
def exec_net(self, image):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.exec_netw.start_async(request_id=0,
inputs={self.input_blob: image})
return
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.exec_netw.requests[0].wait(-1)
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
output = self.exec_netw.requests[0].outputs[self.output_blob]
return output
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device and load extensions library if specified
log.info("Creating Inference Engine")
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, "CPU")
# Read IR
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = ie.read_network(model=model_xml, weights=model_bin)
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(
net.inputs.keys()) == 1, "Sample supports only single input topologies"
assert len(
net.outputs) == 1, "Sample supports only single output topologies"
log.info("Preparing input blobs")
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
net.batch_size = len(args.input)
# Read and pre-process input images
n, c, h, w = net.inputs[input_blob].shape
images = np.ndarray(shape=(n, c, h, w))
for i in range(n):
image = cv2.imread(args.input[i])
if image.shape[:-1] != (h, w):
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = cv2.resize(image, (w, h))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
log.info("Batch size is {}".format(n))
# Loading model to the plugin
log.info("Loading model to the plugin")
exec_net = ie.load_network(network=net, device_name=args.device)
# create one inference request for asynchronous execution
request_id = 0
infer_request = exec_net.requests[request_id]
num_iter = 10
request_wrap = InferReqWrap(infer_request, request_id, num_iter)
# Start inference request execution. Wait for last execution being completed
request_wrap.execute("async", {input_blob: images})
# Processing output blob
log.info("Processing output blob")
res = infer_request.outputs[out_blob]
log.info("Top {} results: ".format(args.number_top))
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.split(sep=' ', maxsplit=1)[-1].strip() for x in f]
else:
labels_map = None
classid_str = "classid"
probability_str = "probability"
for i, probs in enumerate(res):
probs = np.squeeze(probs)
top_ind = np.argsort(probs)[-args.number_top:][::-1]
print("Image {}\n".format(args.input[i]))
print(classid_str, probability_str)
print("{} {}".format('-' * len(classid_str),
'-' * len(probability_str)))
for id in top_ind:
det_label = labels_map[id] if labels_map else "{}".format(id)
label_length = len(det_label)
space_num_before = (7 - label_length) // 2
space_num_after = 7 - (space_num_before + label_length) + 2
space_num_before_prob = (11 - len(str(probs[id]))) // 2
print("{}{}{}{}{:.7f}".format(' ' * space_num_before, det_label,
' ' * space_num_after,
' ' * space_num_before_prob,
probs[id]))
print("\n")
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n"
)
class FacialLandmarksDetectionModel:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model_name = model_name
self.device = device
self.extensions = extensions
self.model_structure = self.model_name
self.model_weights = self.model_name.split(".")[0] + '.bin'
self.plugin = None
self.network = None
self.exec_net = None
self.input_name = None
self.input_shape = None
self.output_names = None
self.output_shape = None
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.plugin = IECore()
self.network = self.plugin.read_network(model=self.model_structure,
weights=self.model_weights)
supported_layers = self.plugin.query_network(network=self.network,
device_name=self.device)
unsupported_layers = [
i for i in self.network.layers.keys() if i not in supported_layers
]
if len(unsupported_layers) != 0 and self.device == 'CPU':
print("unsupported layers found:{}".format(unsupported_layers))
if not self.extensions == None:
print("Adding cpu_extension")
self.plugin.add_extension(self.extensions, self.device)
supported_layers = self.plugin.query_network(
network=self.network, device_name=self.device)
unsupported_layers = [
i for i in self.network.layers.keys()
if i not in supported_layers
]
if len(unsupported_layers) != 0:
print(
"After adding the extension still unsupported layers found"
)
exit(1)
print("After adding the extension the issue is resolved")
else:
print("Give the path of cpu extension")
exit(1)
self.exec_net = self.plugin.load_network(network=self.network,
device_name=self.device,
num_requests=1)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_names = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_names].shape
def predict(self, image):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
processed_img = self.preprocess_input(image.copy())
outputs = self.exec_net.infer({self.input_name: processed_img})
coordinates = self.preprocess_output(outputs)
h = image.shape[0]
w = image.shape[1]
coordinates = coordinates * np.array([w, h, w, h])
coordinates = coordinates.astype(
np.int32) #(lefteye_x, lefteye_y, righteye_x, righteye_y)
le_xmin = coordinates[0] - 10
le_ymin = coordinates[1] - 10
le_xmax = coordinates[0] + 10
le_ymax = coordinates[1] + 10
re_xmin = coordinates[2] - 10
re_ymin = coordinates[3] - 10
re_xmax = coordinates[2] + 10
re_ymax = coordinates[3] + 10
left_eye = image[le_ymin:le_ymax, le_xmin:le_xmax]
right_eye = image[re_ymin:re_ymax, re_xmin:re_xmax]
eye_coordinates = [[le_xmin, le_ymin, le_xmax, le_ymax],
[re_xmin, re_ymin, re_xmax, re_ymax]]
return left_eye, right_eye, eye_coordinates
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
image_cvt = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image_resized = cv2.resize(image_cvt,
(self.input_shape[3], self.input_shape[2]))
img_processed = np.transpose(np.expand_dims(image_resized, axis=0),
(0, 3, 1, 2))
return img_processed
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
outs = outputs[self.output_names][0]
ley_x = outs[0].tolist()[0][0]
ley_y = outs[1].tolist()[0][0]
rey_x = outs[2].tolist()[0][0]
rey_y = outs[3].tolist()[0][0]
return (ley_x, ley_y, rey_x, rey_y)
data_fn = os.path.join(args.data_path, args.data_filename)
model_fn = os.path.join(args.output_path, args.inference_filename)
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
# Plugin initialization for specified device and load extensions library if specified
print("check")
#plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir)
ie=IECore()
print(args.device)
if args.cpu_extension and "CPU" in args.device:
#plugin.add_cpu_extension(args.cpu_extension)
ie.add_extension(args.cpu_extension, "CPU")
# Read IR
# If using MYRIAD then we need to load FP16 model version
model_xml, model_bin = load_model()
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
if args.device == "CPU":
supported_layers = ie.query_network(net, args.device)
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
format(args.device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.net_plugin = None
self.infer_request_handle = None
### TODO: Initialize any class variables desired ###
def load_model(self, model, device="CPU", cpu_extension=None):
### TODO: Load the model ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.plugin = IECore()
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
self.network = IENetwork(model=model_xml, weights=model_bin)
### TODO: Check for supported layers ###
supported_layers = self.plugin.query_network(network=self.network,
device_name="CPU")
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(unsupported_layers))
print("Check whether extensions are available to add to IECore.")
exit(1)
### TODO: Return the loaded inference plugin ###
self.net_plugin = self.plugin.load_network(self.network, device)
### Note: You may need to update the function parameters. ###
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, image):
### TODO: Start an asynchronous request ###
self.infer_request_handle = self.net_plugin.start_async(
request_id=0, inputs={self.input_blob: image})
### TODO: Return any necessary information ###
return
### Note: You may need to update the function parameters. ###
def wait(self):
### TODO: Wait for the request to be complete. ###
status = self.net_plugin.requests[0].wait(-1)
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.net_plugin.requests[0].outputs[self.output_blob]
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
return
def load_model(self, model_xml, device="CPU", cpu_extension=None):
### TODO: Load the model ###
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Initialize the plugin
self.plugin = IECore()
### TODO: Add any necessary extensions ###
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
self.network = self.plugin.read_network(model=model_xml, weights=model_bin)
# ### TODO: Check for supported layers ###
# supported_layers = self.plugin.query_network(network=self.network, device_name=device)
# # Check for any unsupported layers, and let the user
# # know if anything is missing. Exit the program, if so.
# unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
# if len(unsupported_layers) != 0:
# print("Unsupported layers found: {}".format(unsupported_layers))
# print("Check whether extensions are available to add to IECore.")
# exit(1)
# Load the IENetwork into the plugin
self.exec_network = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, input1,input2=None,input3=None):
### TODO: Start an asynchronous request ###
inputiter = iter(self.network.inputs)
if(input2 is None and input3 is None):
self.exec_network.start_async(request_id=0,
inputs={self.input_blob: input1})
elif(input3 is None):
next(inputiter)#ignore first input
self.exec_network.start_async(request_id=0,
inputs={self.input_blob: input1, next(inputiter):input2})
else:
next(inputiter)#ignore first input
self.exec_network.start_async(request_id=0,
inputs={self.input_blob: input1, next(inputiter):input2, next(inputiter):input3})
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return
def wait(self):
### TODO: Wait for the request to be complete. ###
status = self.exec_network.requests[0].wait(-1)
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.exec_network.requests[0].outputs
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# Plugin initialization for specified device and load extensions library if specified
log.info("Creating Inference Engine")
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, "CPU")
# Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format
model = args.model
log.info(f"Loading network:\n\t{model}")
net = ie.read_network(model=model)
assert len(net.input_info.keys()
) == 1, "Sample supports only single input topologies"
assert len(
net.outputs) == 1, "Sample supports only single output topologies"
log.info("Preparing input blobs")
input_blob = next(iter(net.input_info))
out_blob = next(iter(net.outputs))
net.batch_size = len(args.input)
# Read and pre-process input images
n, c, h, w = net.input_info[input_blob].input_data.shape
images = np.ndarray(shape=(n, c, h, w))
for i in range(n):
image = cv2.imread(args.input[i])
if image.shape[:-1] != (h, w):
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = cv2.resize(image, (w, h))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
log.info("Batch size is {}".format(n))
# Loading model to the plugin
log.info("Loading model to the plugin")
exec_net = ie.load_network(network=net, device_name=args.device)
# Start sync inference
log.info("Starting inference")
res = exec_net.infer(inputs={input_blob: images})
# Processing output blob
log.info("Processing output blob")
res = res[out_blob]
# Post process output
for batch, data in enumerate(res):
# Clip values to [0, 255] range
data = np.swapaxes(data, 0, 2)
data = np.swapaxes(data, 0, 1)
data = cv2.cvtColor(data, cv2.COLOR_BGR2RGB)
data[data < 0] = 0
data[data > 255] = 255
data = data[::] - (args.mean_val_r, args.mean_val_g, args.mean_val_b)
out_img = os.path.join(os.path.dirname(__file__),
"out_{}.bmp".format(batch))
cv2.imwrite(out_img, data)
log.info("Result image was saved to {}".format(out_img))
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n"
)
class FacialLandmardDetectionModel:
'''
Class for the Face LandMark detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
self.plugin = None
self.network = None
self.exec_network = None
self.input_name = None
self.output_name = None
self.input_shape = None
self.output_shape = None
self.infer_req = None
self.device = device
self.extensions = extensions
self.model_path = model_name
def load_model(self):
model_xml = self.model_path
model_bin = os.path.splitext(self.model_path)[0] + ".bin"
self.plugin = IECore()
# network model
self.network = self.plugin.read_network(model=model_xml,
weights=model_bin)
network_layers = self.network.layers.keys()
supported_layers = self.plugin.query_network(
network=self.network, device_name=self.device).keys()
ext_required = False
for layer in network_layers:
if layer in supported_layers:
pass
else:
ext_required = True
break
# cpu extension added
if self.extensions != None and "CPU" in self.device and ext_required:
self.plugin.add_extension(self.extensions, self.device)
for layer in network_layers:
if layer in supported_layers:
pass
else:
msg = "Layer extension doesn't support all layers"
log.error(msg)
raise Exception(msg)
self.exec_network = self.plugin.load_network(self.network, self.device)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_name = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_name].shape
return
def predict(self, image, preview=False):
input_img = self.preprocess_input(image)
input_dict = {self.input_name: input_img}
outputs = self.exec_network.infer(input_dict)[self.output_name]
coords = self.preprocess_outputs(outputs,
(image.shape[1], image.shape[0]))
return self.draw_outputs(coords, image, preview)
def draw_outputs(self, coords, image, preview):
left_eye_min = (coords[0] - 15, coords[1] - 15)
left_eye_max = (coords[0] + 15, coords[1] + 15)
right_eye_min = (coords[2] - 15, coords[3] - 15)
right_eye_max = (coords[2] + 15, coords[3] + 15)
left_eye = image[left_eye_min[1]:left_eye_max[1],
left_eye_min[0]:left_eye_max[0]]
right_eye = image[right_eye_min[1]:right_eye_max[1],
right_eye_min[0]:right_eye_max[0]]
if preview:
cv2.rectangle(image, left_eye_min, left_eye_max, (200, 10, 10), 2)
cv2.rectangle(image, right_eye_min, right_eye_max, (200, 10, 10),
2)
eye_coords = [[
left_eye_min[0], left_eye_min[1], left_eye_max[1], left_eye_max[1]
],
[
right_eye_min[0], right_eye_min[1], right_eye_max[0],
right_eye_max[1]
]]
return eye_coords, left_eye, right_eye, image
def preprocess_input(self, image):
preprocessed_frame = cv2.resize(
image, (self.input_shape[3], self.input_shape[2]))
preprocessed_frame = preprocessed_frame.transpose((2, 0, 1))
return preprocessed_frame.reshape(1, *preprocessed_frame.shape)
def preprocess_outputs(self, outputs, dim):
left_eye_x = int(outputs[0][0] * dim[0])
left_eye_y = int(outputs[0][1] * dim[1])
right_eye_x = int(outputs[0][2] * dim[0])
right_eye_y = int(outputs[0][3] * dim[1])
return (left_eye_x, left_eye_y, right_eye_x, right_eye_y)
class Model_Head_Pose_Estimation:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None, threshold=0.5):
'''
Setting Instance variables.
'''
self.model_name = model_name
self.device = device
self.extension = extensions
self.threshold = threshold
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.input_shape = None
def load_model(self):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
model_structure = self.model_name + ".xml"
model_weight = self.model_name + ".bin"
self.network = IENetwork(model=model_structure, weights=model_weight)
self.plugin = IECore()
if self.extension is not None and "CPU" in self.device:
self.plugin.add_extension(self.extension, self.device)
self.exec_network = self.plugin.load_network(self.network, self.device)
self.check_model()
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
self.input_shape = self.network.inputs[self.input_blob].shape
def predict(self, image):
'''
This method is meant for running predictions on the input image.
'''
pre_pro_img = self.preprocess_input(image)
inference_result=self.exec_network.infer(inputs={self.input_blob: pre_pro_img})
# if self.wait() == 0:
# inference_result = self.get_output_result()
type(inference_result)
head_pose = self.preprocess_output(inference_result)
return head_pose
def check_model(self):
if "CPU" in self.device:
supported_layers = self.plugin.query_network(self.network, "CPU")
none_supported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(none_supported_layers) > 0:
log.error("Following layers are not supported by the plugin for the specified device {}:\n {}".
format(self.device, ', '.join(none_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
n, c, h, w = self.input_shape
pre_pro_frame = cv2.resize(image, (w, h))
pre_pro_frame = pre_pro_frame.transpose((2, 0, 1))
pre_pro_frame = pre_pro_frame.reshape(n, c, h, w)
return pre_pro_frame
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
return [outputs["angle_y_fc"].tolist()[0][0], outputs["angle_p_fc"].tolist()[0][0], outputs["angle_r_fc"].tolist()[0][0]]
def wait(self):
inf_status = self.exec_network.requests[0].wait(-1)
return inf_status
def get_output_result(self):
return self.exec_network.requests[0].outputs[self.output_blob]
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device="CPU", CPU_EXTENSION=None):
"""
To load the specified model and check for supported layers.
:param model: path where model available in IR format
:param device: device to be used to load the model
:param CPU_EXTENSION: CPU extension to used
:return: None
"""
# Initialize the plugin
self.plugin = IECore()
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Read the IR as a IENetwork
self.network = IENetwork(model=model_xml, weights=model_bin)
# Check for supported layers
supported_layers = self.plugin.query_network(network=self.network,
device_name=device)
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
# Add necessary extensions
self.plugin.add_extension(CPU_EXTENSION, device)
# Load the network into the Inference Engine
self.exec_network = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
# sucessfully loaded inference plugin
return
def get_input_shape(self):
"""
Provides the shape of input to the network.
:return: Return the shape of the input layer
"""
# return self.network.inputs[self.input_blob].shape
input_shapes = {}
for input in self.network.inputs:
input_shapes[input] = (self.network.inputs[input].shape)
return input_shapes
def exec_net(self, request_id, net_input):
"""
Perform the inference request.
:param requestId: inference requested ID
:param net_input: input to the model for inference
:return: None
"""
#self.infer_request = self.exec_network.start_async(request_id=request_id,inputs={self.input_blob: net_input})
# Start an asynchronous request
self.infer_request = self.exec_network.start_async(request_id,
inputs=net_input)
return
def wait(self, request_id):
"""
Wait for the request to be complete.
:param requestId: inference requested ID
:return: status of the inference request
"""
status = self.exec_network.requests[request_id].wait(-1)
return status
def get_output(self):
"""
To extract and return the output results.
:return: inference output results
"""
# Extract and return the output results
return self.infer_request.outputs[self.output_blob]
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.core = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
def get_unsupported_layers(self, device):
#get a list of the supported layers
supported_layers = self.core.query_network(self.network,
device_name=device)
#get the required layers
required_layers = list(self.network.layers.keys())
#check if there are unsupported layers
unsupported_layers = []
for layer in required_layers:
if layer not in supported_layers:
unsupported_layers.append(layer)
return unsupported_layers
def load_model(self, device, model_xml, cpu_extension):
### TODO: Load the model ###
### TODO: Check for supported layers ###
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
#check if we have provided a valid file for the xml
if (model_xml.endswith('.xml')) and (os.path.exists(model_xml)):
model_bin = model_xml.replace('.xml', '.bin') #get the model bin
if DEBUG:
print("model found")
print("model_xml: ", model_xml)
print("model_bin: ", model_bin)
print("device: ", device)
else:
print(
"There was a problem reading the xml file provided, exiting..."
)
exit(1)
#initialize the Inference Engine (NOTE using the 2019R3 version of OPENVino, as stated in the exercise)
self.core = IECore()
#initialize the Network (NOTE in the 2020R1 version of OPENVino, can be used from the IECore)
self.network = IENetwork(model=model_xml, weights=model_bin)
#check if there are any unsupported layers
unsupported_layers = self.get_unsupported_layers(device)
#if there are any unsupported layers, add CPU extension, if avaiable
if (len(unsupported_layers) > 0) and (device == 'CPU'):
print(
"There are unsupported layers found, will try to add CPU extension..."
)
self.core.add_extension(extension_path=cpu_extension,
device=device)
#add, if provided, a cpu extension
if (cpu_extension):
self.core.add_extension(cpu_extension)
#recheck for unsupported layers, and exit if there are any
unsupported_layers = self.get_unsupported_layers(device)
if (len(unsupported_layers) > 0):
print(
"After adding CPU extension, there are still unsupported layers, exiting..."
)
exit(1)
#load to network to get the executable network
self.exec_network = self.core.load_network(self.network, device)
#get the input and output blobs
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
return self.exec_network
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, image, request_id=0):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
#requiest_id is set to zero, if not specified, for pseudo async mode
self.exec_network.start_async(request_id=request_id,
inputs={self.input_blob: image})
return
def wait(self, request_id=0):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
infer_status = self.exec_network.requests[request_id].wait(-1)
return infer_status
def get_output(self, request_id=0):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
result = self.exec_network.requests[request_id].outputs[
self.output_blob]
#print('---------------------------------------- latency:', self.exec_network.requests[request_id].latency)
return result
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device and load extensions library if specified
log.info("Creating Inference Engine")
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, "CPU")
# Read IR
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = ie.read_network(model=model_xml, weights=model_bin)
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(net.input_info.keys()
) == 1, "Sample supports only single input topologies"
assert len(
net.outputs) == 1, "Sample supports only single output topologies"
log.info("Preparing input blobs")
input_blob = next(iter(net.input_info))
out_blob = next(iter(net.outputs))
net.batch_size = len(args.input)
# Read and pre-process input images
n, c, h, w = net.input_info[input_blob].input_data.shape
images = np.ndarray(shape=(n, c, h, w))
for i in range(n):
image = cv2.imread(args.input[i])
if image.shape[:-1] != (h, w):
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = cv2.resize(image, (w, h))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
log.info("Batch size is {}".format(n))
# Loading model to the plugin
log.info("Loading model to the plugin")
exec_net = ie.load_network(network=net, device_name=args.device)
# Start sync inference
log.info("Starting inference")
res = exec_net.infer(inputs={input_blob: images})
# Processing output blob
log.info("Processing output blob")
res = res[out_blob]
# Post process output
for batch, data in enumerate(res):
# Clip values to [0, 255] range
data = np.swapaxes(data, 0, 2)
data = np.swapaxes(data, 0, 1)
data = cv2.cvtColor(data, cv2.COLOR_BGR2RGB)
data[data < 0] = 0
data[data > 255] = 255
data = data[::] - (args.mean_val_r, args.mean_val_g, args.mean_val_b)
out_img = os.path.join(os.path.dirname(__file__),
"out_{}.bmp".format(batch))
cv2.imwrite(out_img, data)
log.info("Result image was saved to {}".format(out_img))
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n"
)
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self._core = None
self._network = None
self._exec_network = None
self._input_blob = None
self._output_blob = None
self._img_info_input_blob = None
self._feed_dict = {}
def load_model(self, model, device, cpu_extension=None):
### Load the model ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self._core = IECore()
### Add any necessary extensions ###
if cpu_extension and "CPU" in device:
self._core.add_extension(cpu_extension, device)
self._network = IENetwork(model=model_xml, weights=model_bin)
### Check for supported layers ###
unsupported = self.check_unsupported_layers(self._network, device)
if len(unsupported) != 0:
print("There are unsupported layers in the current model.\n"
"Try to load a CPU extension to solve this problem.\n"
"Layer types: " + str(unsupported) + "\n"
"Cannot continue. Exiting.",
file=sys.stderr)
exit(1)
try:
self._exec_network = self._core.load_network(self._network, device)
except Exception as e:
if "unsupported layer" in str(e):
# OpenVINO throws a RuntimeException on unsupported layer,
# not an specific type of exception
print("Cannot run the model, unsupported layer: ",
e,
file=sys.stderr)
print(
"You can try to pass a CPU Extension with the argument --cpu_extension",
file=sys.stderr)
else:
print(e, file=sys.stderr)
exit(1)
#self._input_blob = next(iter(self._network.inputs)) #does not work with Faster RCNN (2 inputs)
self._handle_inputs()
assert len(self._network.outputs
) == 1, "Demo supports only single output topologies"
self._output_blob = next(iter(self._network.outputs))
### Note: You may need to update the function parameters. ###
return self._exec_network ########hay que ver si no es un "leak" de implementación
def _handle_inputs(self):
#code copied from object_detection_demo_ssd_async.py.
# Before this, Faster RCNN did not work because it have 2 inputs
for blob_name in self._network.inputs:
if len(self._network.inputs[blob_name].shape) == 4:
self._input_blob = blob_name
elif len(self._network.inputs[blob_name].shape) == 2:
self._img_info_input_blob = blob_name
else:
raise RuntimeError(
"Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported"
.format(len(self._network.inputs[blob_name].shape),
blob_name))
#
n, c, h, w = self._network.inputs[self._input_blob].shape
if self._img_info_input_blob:
self._feed_dict[self._img_info_input_blob] = [h, w, 1]
def get_input_shape(self):
return self._network.inputs[self._input_blob].shape
def exec_net(self,
image,
request_id=0): #### #Renombrar a async_inference?
### TODO: Start an asynchronous request ###
self._feed_dict[self._input_blob] = image
self._exec_network.start_async(
request_id=request_id,
inputs=self._feed_dict) # inputs={self._input_blob: image})
## ???
# Implement exec_net() by setting a self.infer_request_handle variable
# to an instance of self.net_plugin.start_async
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return
def wait(self, request_id=0, timeout_ms=-1):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self._exec_network.requests[request_id].wait(timeout_ms) #
def get_output(self, request_id=0):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self._exec_network.requests[request_id].outputs[
self._output_blob]
def check_unsupported_layers(self, net, device):
"""
Checks for unsupported layers on selected device.
Returns a set of unsupported layers (empty if there are none)
"""
supported_layers = self._core.query_network(network=net,
device_name=device)
unsupported = set()
for layer, obj in net.layers.items():
if not layer in supported_layers and obj.type != 'Input':
log.debug(f"Unsupported layer: {layer}, Type: {obj.type}")
unsupported.add(obj.type)
return unsupported
class GazeEstimation:
"""
Model Inference Network
"""
def __init__(self):
"""
Initializing data fields.
"""
self.ie = IECore()
self.network = None
self.input_blob = None
self.output_blob = None
self.inference_plugin = None
self.inference_handler = None
self.device = None
def load_model(self, model, device, num_requests, cpu_extension=None):
"""
Loading the necessary .xml, .bin files and adding cpu extensions
while dealing with unsupported layers and customer layer
implementations.
:param model: location of the model's .xml file
:param device: inference device to use
:param num_requests: Request ID for the inference request
:param cpu_extension: Location of the CPU extension to deal with
unsupported layers, mostly used with MKLDNN plugin.
:return: input shape of the model in [n, c, h, w ] shape.
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + '.bin'
self.network = self.ie.read_network(model_xml, model_bin)
if 'CPU' in device and cpu_extension:
self.ie.add_extension(cpu_extension, 'CPU')
if 'CPU' in device:
supported_layers = self.ie.query_network(self.network, 'CPU')
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(unsupported_layers) != 0:
log.error("One or More Unsupported Layers cannot be interpreted. Use MKLDNN Extension if not already "
"done")
log.error(unsupported_layers)
sys.exit(1)
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
if num_requests == 0:
self.inference_plugin = self.ie.load_network(self.network, device)
else:
self.inference_plugin = self.ie.load_network(network=self.network, device_name=device, num_requests=0)
return self.get_input_shape()
def get_input_shape(self):
"""
Obtaining the Shape of the network's input.
:return: Shape of the network in [n, c, h, w] format.
"""
return self.network.inputs[self.input_blob].shape
def wait(self, request_id):
"""
To Deal with the [REQUEST BUSY] error.
:return: (int) waiting semaphore
"""
wait_for_inference = self.inference_plugin.requests[request_id].wait(-1)
return wait_for_inference
def get_output(self, request_id, prev_output=None):
"""
Getting the output of the inference using the Asynchronous Inference
Request.
:param request_id: Request ID for the inference request
:param prev_output:
:return: result of the inference.
"""
if prev_output:
res = self.inference_handler.outputs[prev_output]
else:
res = self.inference_plugin.requests[request_id].outputs[self.output_blob]
return res
def get_performance_counts(self, request_id):
"""
Get Layer wise Performance Stats.
:param request_id:
:return: Performance stats in json.
"""
performance_count = self.inference_plugin.requests[request_id].get_perf_counts()
return performance_count
def multi_in_infer(self, request_id, le_img, re_img, hp_ang):
inp_dict = {'left_eye_image': le_img, 'right_eye_image': re_img, 'head_pose_angles': hp_ang}
multi_input_dict = {self.input_blob: inp_dict}
self.inference_handler = self.inference_plugin.start_async(request_id=0, inputs=inp_dict)
return self.inference_plugin
def process_output(self, gaze_vector, angle_r_fc):
gaze_vector = gaze_vector / np.linalg.norm(gaze_vector)
gaze_x = gaze_vector[0]
gaze_y = gaze_vector[1]
gaze_z = gaze_vector[2]
gaze_yaw = np.arctan(gaze_y / gaze_x)
sin = np.sin(angle_r_fc * math.pi / 180.)
cos = np.cos(angle_r_fc * math.pi / 180.)
move_x = gaze_x * cos + gaze_y * sin
move_y = - gaze_x * sin + gaze_y * cos
gaze_yaw = gaze_yaw * np.pi / 180.0
gaze_x_angle = np.arctan2(gaze_x, -gaze_z)
gaze_y_angle = np.arctan2(-gaze_y, -gaze_z)
return move_x, move_y, gaze_x_angle, gaze_y_angle, gaze_yaw
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Creating Inference Engine...")
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, "CPU")
# Read IR
log.info("Loading network")
# Path to your model here
args.model = 'gloves_model\\frozen_inference_graph.xml'
net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin")
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
img_info_input_blob = None
feed_dict = {}
for blob_name in net.input_info:
if len(net.input_info[blob_name].input_data.shape) == 4:
input_blob = blob_name
elif len(net.input_info[blob_name].input_data.shape) == 2:
img_info_input_blob = blob_name
else:
raise RuntimeError(
"Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported"
.format(len(net.input_info[blob_name].input_data.shape),
blob_name))
output_postprocessor = get_output_postprocessor(net)
log.info("Loading IR to the plugin...")
exec_net = ie.load_network(network=net,
num_requests=2,
device_name=args.device)
# Read and pre-process input image
n, c, h, w = net.input_info[input_blob].input_data.shape
if img_info_input_blob:
feed_dict[img_info_input_blob] = [h, w, 1]
# Input stream-------------------------------------------------------------
# Labels-----------------------------------------------------------
args.labels = 'gloves_model\\labels.pbtxt'
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
def UploadAction(event=None):
global chosenInput
chosenInput = True
filename = filedialog.askopenfilename()
input_stream = filename
global cap
cap = cv2.VideoCapture(input_stream)
def CameraAction():
global chosenInput
chosenInput = True
global cap
cap = cv2.VideoCapture(0)
cur_request_id = 0
next_request_id = 1
log.info("Starting inference in async mode...")
is_async_mode = True
render_time = 0
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
print(
"To switch between sync/async modes, press TAB key in the output window"
)
#======================================================================================================
#Set up GUI
window = tk.Tk() #Makes main window
window.wm_title("FarAhead")
window.config(background="#FFFFFF")
inputFrame = tk.Frame(window, width=300, height=600)
inputFrame.grid(row=0, column=0, padx=10, pady=2)
fileButton = tk.Button(inputFrame,
width=20,
text='Choose a file',
border=5,
command=UploadAction)
fileButton.grid(row=0, column=1, columnspan=3)
cameraButton = tk.Button(inputFrame,
width=20,
text='Use Camera',
border=5,
command=CameraAction)
cameraButton.grid(row=1, column=1, columnspan=3)
#Graphics window
imageFrame = tk.Frame(window, width=800, height=600)
imageFrame.grid(row=0, column=1, padx=10, pady=2)
#Capture video frames
# this is just a label inside the frame
mainBg = cv2.imread('mainBG.png')
b, g, r = cv2.split(mainBg)
mainBg = cv2.merge((r, g, b))
mainBg = Image.fromarray(mainBg)
mainBg = ImageTk.PhotoImage(image=mainBg)
lmain = tk.Label(imageFrame)
lmain.grid(row=0, column=0)
lmain.configure(image=mainBg)
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<#
def no_input():
lmain.configure(image=mainBg)
def show_frame():
# reading the frame
cv2image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
#changing opencv image to pil image
img = Image.fromarray(cv2image)
imgtk = ImageTk.PhotoImage(image=img)
lmain.imgtk = imgtk
lmain.configure(image=imgtk)
while True:
global chosenInput
if not chosenInput:
no_input()
elif chosenInput:
if not cap.isOpened():
chosenInput = False
continue
if is_async_mode:
ret, frame = cap.read()
if not ret:
chosenInput = False
continue # abandons the last frame in case of async_mode
frame_h, frame_w = frame.shape[:2]
if cap.isOpened():
if is_async_mode:
ret, next_frame = cap.read()
else:
ret, frame = cap.read()
if ret:
frame_h, frame_w = frame.shape[:2]
if not ret:
chosenInput = False
continue # abandons the last frame in case of async_mode
# Main sync point:
# in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
# in the regular mode we start the CURRENT request and immediately wait for it's completion
inf_start = time.time()
if is_async_mode:
in_frame = cv2.resize(next_frame, (w, h))
in_frame = in_frame.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
in_frame = in_frame.reshape((n, c, h, w))
feed_dict[input_blob] = in_frame
exec_net.start_async(request_id=next_request_id,
inputs=feed_dict)
else:
in_frame = cv2.resize(frame, (w, h))
in_frame = in_frame.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
in_frame = in_frame.reshape((n, c, h, w))
feed_dict[input_blob] = in_frame
exec_net.start_async(request_id=cur_request_id,
inputs=feed_dict)
if exec_net.requests[cur_request_id].wait(-1) == 0:
inf_end = time.time()
det_time = inf_end - inf_start
# Parse detection results of the current request
for obj in output_postprocessor(
exec_net.requests[cur_request_id].output_blobs):
# Draw only objects when probability more than specified threshold
if obj[2] > args.prob_threshold:
xmin = int(obj[3] * frame_w)
ymin = int(obj[4] * frame_h)
xmax = int(obj[5] * frame_w)
ymax = int(obj[6] * frame_h)
class_id = int(obj[1])
# Draw box and label\class_id
if class_id == 1:
color = (50, 205, 50)
else:
color = (255, 0, 0)
#color = (min(class_id * 12.5, 255), min(class_id * 7, 255), min(class_id * 5, 255))
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
color, 5)
det_label = labels_map[
class_id] if labels_map else str(class_id)
cv2.putText(
frame, det_label + ' ' +
str(round(obj[2] * 100, 1)) + ' %',
(xmin, ymin - 7), cv2.FONT_HERSHEY_DUPLEX, 0.8,
(225, 225, 225), 1)
# Draw performance stats
inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
"Inference time: {:.3f} ms".format(det_time * 1000)
render_time_message = "OpenCV rendering time: {:.3f} ms".format(
render_time * 1000)
async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
"Async mode is off. Processing request {}".format(cur_request_id)
#cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
#cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
# cv2.putText(frame, async_mode_message, (10, int(frame_h - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5,
# (10, 10, 200), 1)
cv2image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
#changing opencv image to pil image
# img = Image.fromarray(cv2image)
# imgtk = ImageTk.PhotoImage(image=img)
# lmain.imgtk = imgtk
# lmain.configure(image=imgtk)
# presenter.drawGraphs(frame)
render_start = time.time()
if not args.no_show:
show_frame()
# cv2.imshow('det',frame)
render_end = time.time()
render_time = render_end - render_start
if is_async_mode:
cur_request_id, next_request_id = next_request_id, cur_request_id
frame = next_frame
frame_h, frame_w = frame.shape[:2]
if not args.no_show:
key = cv2.waitKey(1)
if key == 27:
break
if (9 == key):
is_async_mode = not is_async_mode
log.info("Switched to {} mode".format(
"async" if is_async_mode else "sync"))
window.update_idletasks()
window.update()
class FaceDetectionClass:
def __init__(self, model_name, device, threshold, extensions=None):
self.model_weights = model_name + '.bin'
self.model_structure = model_name + ".xml"
self.device = device
self.threshold = threshold
self.extension = extensions
self.network = None
self.plugin = None
self.exec_net = None
def load_model(self):
self.plugin = IECore()
self.check_model(self.model_structure, self.model_weights)
if not all_layers_supported(self.plugin, self.network):
self.plugin.add_extension(self.extension, self.device)
# Load model
self.exec_net = self.plugin.load_network(network=self.network,
device_name=self.device,
num_requests=1)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_name = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_name].shape
def predict(self, image):
p_image = self.preprocess_input(image)
results = self.exec_net.infer({self.input_name: p_image})
coordinates = self.preprocess_output(results, image)
if len(coordinates) == 0:
return 0, 0
cropped_face = image[coordinates[1]:coordinates[3],
coordinates[0]:coordinates[2]]
return coordinates, cropped_face
def check_model(self, model_structure, model_weights):
try:
self.network = IENetwork(model_structure, model_weights)
except Exception as e:
raise ValueError("Could not Initialize Network")
def preprocess_input(self, image):
p_frame = cv2.resize(image, (self.input_shape[3], self.input_shape[2]))
p_frame = p_frame.transpose((2, 0, 1))
p_frame = p_frame.reshape(1, *p_frame.shape)
return p_frame
def preprocess_output(self, outputs, image):
coordinates = []
outputs = outputs[self.output_name][0][0]
w, h = image.shape[1], image.shape[0]
for output in outputs:
confidence = output[2]
if confidence >= self.threshold:
xmin = int(output[3] * w)
ymin = int(output[4] * h)
xmax = int(output[5] * w)
ymax = int(output[6] * h)
coordinates.append([xmin, ymin, xmax, ymax])
return list(map(int, coordinates[0]))
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, CPU_EXTENSION, DEVICE, console_output=False):
### TODO: Load the model ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.plugin = IECore()
self.network = IENetwork(model=model_xml, weights=model_bin)
### TODO: Check for supported layers ###
#if not all_layers_supported(self.plugin, self.network, console_output=console_output):
supported_layers = self.plugin.query_network(self.network,
device_name="CPU")
for layer in self.network.layers.keys():
if layer not in supported_layers:
print(
"Check whether extensions are available to add to IECore.")
self.plugin.add_extension(CPU_EXTENSION, DEVICE)
self.exec_network = self.plugin.load_network(self.network, DEVICE)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
all_shapes = {}
for shape in self.network.inputs:
all_shapes[shape] = (self.network.inputs[shape].shape)
return all_shapes
def exec_net(self, net_input, request_id):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
self.infer_request_handle = self.exec_network.start_async(
request_id, inputs=net_input)
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.infer_request_handle.wait()
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.infer_request_handle.outputs[self.output_blob]
class GazeEstimation:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model_name = model_name
self.model_structure = self.model_name
self.model_weights = self.model_name.split('.')[0] + '.bin'
self.device = device
self.extensions = extensions
self.core = None
self.model = None
self.net = None
self.input_name = None
self.input_shape = None
self.output_names = None
self.output_shape = None
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.core = IECore()
self.model = IENetwork(self.model_structure, self.model_weights)
supported_layers = self.core.query_network(self.model,
device_name=self.device)
unsupported_layers = [
l for l in self.model.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0 and self.device == 'CPU':
print("Unsupported layers found: {}".format(unsupported_layers))
if not self.extensions == None:
self.core.add_extension(self.extensions, self.device)
supported_layers = self.core.query_network(
network=self.model, device_name=self.device)
unsupported_layers = [
l for l in self.model.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
print(
"Even adding the extension there are unsupported layers found"
)
exit(1)
else:
print(
"Check whether extensions are available to add to IECore and provide the path."
)
exit(1)
self.net = self.core.load_network(network=self.model,
device_name=self.device,
num_requests=1)
self.input_name = [i for i in self.model.inputs.keys()]
self.input_shape = self.model.inputs[self.input_name[1]].shape
self.output_names = [i for i in self.model.outputs.keys()]
def predict(self, left_eye_img, right_eye_img, head_pose_angle):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
left_eye_img_proc, right_eye_img_proc = self.preprocess_input(
left_eye_img, right_eye_img)
results = self.net.infer({
'head_pose_angles': head_pose_angle,
'left_eye_image': left_eye_img_proc,
'right_eye_image': right_eye_img_proc
})
new_mouse_coord, gaze_vector = self.preprocess_output(
results, head_pose_angle)
return new_mouse_coord, gaze_vector
def check_model(self):
pass
def preprocess_input(self, left_eye_img, right_eye_img):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
left_eye_img_proc = cv2.resize(
left_eye_img, (self.input_shape[3], self.input_shape[2]))
left_eye_img_proc = np.transpose(
np.expand_dims(left_eye_img_proc, axis=0), (0, 3, 1, 2))
right_eye_img_proc = cv2.resize(
right_eye_img, (self.input_shape[3], self.input_shape[2]))
right_eye_img_proc = np.transpose(
np.expand_dims(right_eye_img_proc, axis=0), (0, 3, 1, 2))
return left_eye_img_proc, right_eye_img_proc
def preprocess_output(self, outputs, head_pose_estimation):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
gaze_vector = outputs[self.output_names[0]].tolist()[0]
roll_angle = head_pose_estimation[2]
cos_angle = math.cos(roll_angle * math.pi / 180.0)
sin_angle = math.sin(roll_angle * math.pi / 180.0)
x_coord = gaze_vector[0] * cos_angle + gaze_vector[1] * sin_angle
y_coord = gaze_vector[1] * cos_angle - gaze_vector[0] * sin_angle
return (x_coord, y_coord), gaze_vector
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = IECore()
self.exec_network = None
self.net = None
self.input_layer = None
def load_model(self, xml_path, num_requests, device_name, cpu_extension):
### TODO: Load the model ###
self.net = IENetwork(model=xml_path,
weights=xml_path.split('.')[0] + '.bin')
### TODO: Check for supported layers ###
supported_layers = self.plugin.query_network(network=self.net,
device_name=device_name)
unsupported_layers = [
l for l in self.net.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
#print("unsupported layers found:{}".format(unsupported_layers))
if not cpu_extension == None:
#print("Adding cpu_extension")
self.plugin.add_extension(cpu_extension, device_name)
supported_layers = self.plugin.query_network(
network=self.net, device_name=device_name)
unsupported_layers = [
l for l in self.net.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
#print("After adding the extension still unsupported layers found")
exit(1)
#print("After adding the extension the issue is resolved")
else:
#print("Give the path of cpu extension")
exit(1)
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
self.exec_network = self.plugin.load_network(network=self.net,
num_requests=num_requests,
device_name=device_name)
### Note: You may need to update the function parameters. ###
self.input_layer = next(iter(self.net.inputs))
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.net.inputs[self.input_layer].shape
def exec_net(self, frame, req_id):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.exec_network.start_async(request_id=req_id,
inputs={self.input_layer: frame})
return
def wait(self, req_id):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.exec_network.requests[req_id].wait(-1)
def get_output(self, req_id):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
out = self.exec_network.requests[req_id].outputs
return [out[i] for i in out.keys()]