class Model_FD:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device, extensions):
self.plugin = None
self.net = None
self.input_blob = None
self.output_blob = None
self.exec_net = None
self.infer_request = None
self.input_shape = None
self.output_shape = None
self.input_name = None
self.device = device
self.extension = extensions
self.model = model_name
def load_model(self):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
model_xml = self.model + ".xml"
model_weights = self.model + ".bin"
self.plugin = IECore()
self.net = IENetwork(model_xml, model_weights)
self.exec_net = self.plugin.load_network(network=self.net,
device_name=self.device,
num_requests=1)
if self.extension and 'CPU' in self.device:
self.plugin.add_cpu_extension(self.extension)
self.check_model()
self.input_blob = next(iter(self.net.inputs))
self.output_blob = next(iter(self.net.outputs))
self.input_shape = self.net.inputs[self.input_blob].shape
self.output_shape = self.net.outputs[self.output_blob].shape
def predict(self, image, benchmark_timing):
'''
This method is meant for running predictions on the input image.
'''
global outputs
self.exec_net.start_async(request_id=0,
inputs={self.input_blob: image})
if self.exec_net.requests[0].wait(-1) == 0:
outputs = self.exec_net.requests[0].outputs[self.output_blob]
if benchmark_timing:
pp = PrettyPrinter(indent=4)
print('Benchmark Timing for Face_Detection')
pp.pprint(self.exec_net.requests[0].get_perf_counts())
# Write get_perf_counts() data to a text file
data = (self.exec_net.requests[0].get_perf_counts())
self.write_benchmark('Benchmark Timing for Face_Detection',
data)
return outputs
def check_model(self):
supported_layers = self.plugin.query_network(network=self.net,
device_name=self.device)
unsupported_layers = [
l for l in self.net.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error(
"Unsupported layers found: {}".format(unsupported_layers))
log.error(
"Check whether extensions are available to add to IECore.")
exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
global pimage
try:
pimage = cv2.resize(image,
(self.input_shape[3], self.input_shape[2]),
interpolation=cv2.INTER_AREA)
pimage = pimage.transpose((2, 0, 1))
pimage = pimage.reshape(image, (1, *image.shape))
except Exception as e:
log.error(str(e))
return pimage
def preprocess_output(self, image, outputs, threshold, display):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
global h, w
try:
h = image.shape[0]
w = image.shape[1]
except Exception as e:
log.error(str(e))
facebox = []
# Drawing the box/boxes
for obj in outputs[0][0]:
if obj[2] > threshold:
if obj[3] < 0:
obj[3] = -obj[3]
if obj[4] < 0:
obj[4] = -obj[4]
xmin = int(obj[3] * w)
ymin = int(obj[4] * h)
xmax = int(obj[5] * w)
ymax = int(obj[6] * h)
# Drawing the box in the image
if display:
cv2.rectangle(image, (xmin, ymin), (xmax, ymax),
(255, 0, 0), 1)
facebox.append([xmin, ymin, xmax, ymax])
return facebox
def get_model_name(self):
return self.model
def write_benchmark(self, title, data):
with open("FaceDetectione_benchmark_timing.txt", "a") as f:
f.write(str(title) + "\n")
f.write(str(data) + '\n')
f.close()
def main(args=None):
try:
# ------------------------------ 1. Parsing and validating input arguments -------------------------------------
next_step()
if not args:
args = parse_args()
# ------------------------------ 2. Loading Inference Engine ---------------------------------------------------
next_step()
device_name = args.target_device.upper()
ie = IECore()
if CPU_DEVICE_NAME in device_name:
if args.path_to_extension:
ie.add_cpu_extension(extension_path=args.path_to_extension,
device_name=CPU_DEVICE_NAME)
if GPU_DEVICE_NAME in device_name:
if args.path_to_cldnn_config:
ie.set_config({'CONFIG_FILE': args.path_to_cldnn_config},
GPU_DEVICE_NAME)
logger.info("GPU extensions is loaded {}".format(
args.path_to_cldnn_config))
logger.info("InferenceEngine:\n{: <9}{}".format("", get_version()))
version_string = "Device is {}\n".format(device_name)
for device, version in ie.get_versions(device_name).items():
version_string += "{: <9}{}\n".format("", device)
version_string += "{: <9}{:.<24}{} {}.{}\n".format(
"", version.description, " version", version.major,
version.minor)
version_string += "{: <9}{:.<24} {}\n".format(
"", "Build", version.build_number)
logger.info(version_string)
# --------------------- 3. Read the Intermediate Representation of the network ---------------------------------
next_step()
xml_filename = os.path.abspath(args.path_to_model)
head, tail = os.path.splitext(xml_filename)
bin_filename = os.path.abspath(head + BIN_EXTENSION)
ie_network = IENetwork(xml_filename, bin_filename)
input_info = ie_network.inputs
if len(input_info) == 0:
raise AttributeError('No inputs info is provided')
# --------------------- 4. Resizing network to match image sizes and given batch -------------------------------
next_step()
batch_size = ie_network.batch_size
precision = ie_network.precision
if args.batch_size and args.batch_size != ie_network.batch_size:
new_shapes = {}
for key in input_info.keys():
shape = input_info[key].shape
layout = input_info[key].layout
batchIndex = -1
if ((layout == 'NCHW') or (layout == 'NCDHW')
or (layout == 'NHWC') or (layout == 'NDHWC')
or (layout == 'NC')):
batchIndex = 0
elif (layout == 'CN'):
batchIndex = 1
if ((batchIndex != -1)
and (shape[batchIndex] != args.batch_size)):
shape[batchIndex] = args.batch_size
new_shapes[key] = shape
if (len(new_shapes) > 0):
logger.info("Resizing network to batch = {}".format(
args.batch_size))
ie_network.reshape(new_shapes)
batch_size = args.batch_size
logger.info("Network batch size: {}, precision {}".format(
batch_size, precision))
# --------------------- 5. Configuring input of the model ------------------------------------------------------
next_step()
for key in input_info.keys():
if (isImage(input_info[key])):
# Set the precision of input data provided by the user
# Should be called before load of the network to the plugin
input_info[key].precision = 'U8'
# --------------------- 6. Setting device configuration --------------------------------------------------------
next_step()
devices = parseDevices(device_name)
device_nstreams = parseValuePerDevice(devices, args.number_streams)
for device in devices:
if device == CPU_DEVICE_NAME: ## CPU supports few special performance-oriented keys
## limit threading for CPU portion of inference
if args.number_threads:
ie.set_config(
{'CPU_THREADS_NUM': str(args.number_threads)}, device)
if MULTI_DEVICE_NAME in device_name and GPU_DEVICE_NAME in device_name:
ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME)
else:
# pin threads for CPU portion of inference
ie.set_config(
{'CPU_BIND_THREAD': args.infer_threads_pinning},
device)
## for CPU execution, more throughput-oriented execution via streams
# for pure CPU execution, more throughput-oriented execution via streams
if args.api_type == 'async':
ie.set_config(
{
'CPU_THROUGHPUT_STREAMS':
str(device_nstreams.get(device))
if device in device_nstreams.keys() else
'CPU_THROUGHPUT_AUTO'
}, device)
device_nstreams[device] = int(
ie.get_config(device, 'CPU_THROUGHPUT_STREAMS'))
elif device == GPU_DEVICE_NAME:
if args.api_type == 'async':
ie.set_config(
{
'GPU_THROUGHPUT_STREAMS':
str(device_nstreams.get(device))
if device in device_nstreams.keys() else
'GPU_THROUGHPUT_AUTO'
}, device)
device_nstreams[device] = int(
ie.get_config(device, 'GPU_THROUGHPUT_STREAMS'))
if MULTI_DEVICE_NAME in device_name and CPU_DEVICE_NAME in device_name:
## multi-device execution with the CPU+GPU performs best with GPU trottling hint,
## which releases another CPU thread (that is otherwise used by the GPU driver for active polling)
ie.set_config({'CLDNN_PLUGIN_THROTTLE': str(1)}, device)
elif device == MYRIAD_DEVICE_NAME:
ie.set_config(
{
'LOG_LEVEL': 'LOG_INFO',
'VPU_LOG_LEVEL': 'LOG_WARNING'
}, MYRIAD_DEVICE_NAME)
# --------------------- 7. Loading the model to the device -----------------------------------------------------
next_step()
config = {'PERF_COUNT': ('YES' if args.perf_counts else 'NO')}
exe_network = ie.load_network(ie_network,
device_name,
config=config,
num_requests=args.number_infer_requests
if args.number_infer_requests else 0)
# --------------------- 8. Setting optimal runtime parameters --------------------------------------------------
next_step()
## Number of requests
infer_requests = exe_network.requests
nireq = len(infer_requests)
## Iteration limit
niter = args.number_iterations
if niter and args.api_type == 'async':
niter = (int)((niter + nireq - 1) / nireq) * nireq
if (args.number_iterations != niter):
logger.warn(
"Number of iterations was aligned by request number "
"from {} to {} using number of requests {}".format(
args.number_iterations, niter, nireq))
## Time limit
duration_seconds = 0
if args.time:
## time limit
duration_seconds = args.time
elif not args.number_iterations:
## default time limit
duration_seconds = get_duration_in_secs(device)
# ------------------------------------ 8. Creating infer requests and filling input blobs ----------------------
next_step()
request_queue = InferRequestsQueue(infer_requests)
path_to_input = os.path.abspath(
args.path_to_input) if args.path_to_input else None
requests_input_data = getInputs(path_to_input, batch_size,
ie_network.inputs, infer_requests)
# ------------------------------------ 9. Measuring performance ------------------------------------------------
progress_count = 0
progress_bar_total_count = 10000
output_string = "Start inference {}ronously".format(args.api_type)
if (args.api_type == "async"):
if output_string != "":
output_string += ", "
output_string += str(nireq) + " inference requests"
device_ss = ''
for device, nstreams in device_nstreams.items():
if device_ss != '':
device_ss += ', '
device_ss += "{} streams for {}".format(str(nstreams), device)
if device_ss != '':
output_string += " using " + device_ss
output_string += ", limits: "
if niter:
if not duration_seconds:
progress_bar_total_count = niter
output_string += str(niter) + " iterations"
if duration_seconds:
if niter:
output_string += ", "
output_string += str(
getDurationInMilliseconds(duration_seconds)) + " ms duration"
next_step(output_string)
## warming up - out of scope
infer_request = request_queue.getIdleRequest()
if not infer_request:
raise Exception("No idle Infer Requests!")
if (args.api_type == 'sync'):
infer_request.infer(requests_input_data[infer_request.id])
else:
infer_request.startAsync(requests_input_data[infer_request.id])
request_queue.waitAll()
request_queue.resetTimes()
start_time = datetime.now()
exec_time = (datetime.now() - start_time).total_seconds()
iteration = 0
progress_bar = ProgressBar(progress_bar_total_count,
args.stream_output, args.progress)
## Start inference & calculate performance
## to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
while ((niter and iteration < niter)
or (duration_seconds and exec_time < duration_seconds)
or (args.api_type == "async" and iteration % nireq != 0)):
infer_request = request_queue.getIdleRequest()
if not infer_request:
raise Exception("No idle Infer Requests!")
if (args.api_type == 'sync'):
infer_request.infer(requests_input_data[infer_request.id])
else:
infer_request.startAsync(requests_input_data[infer_request.id])
iteration += 1
exec_time = (datetime.now() - start_time).total_seconds()
if niter:
progress_bar.add_progress(1)
else:
## calculate how many progress intervals are covered by current iteration.
## depends on the current iteration time and time of each progress interval.
## Previously covered progress intervals must be skipped.
progress_interval_time = duration_seconds / progress_bar_total_count
new_progress = (int)(exec_time / progress_interval_time -
progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
## wait the latest inference executions
request_queue.waitAll()
total_duration_sec = request_queue.getDurationInSeconds()
times = request_queue.times
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms if args.api_type == 'sync' else batch_size * iteration / total_duration_sec
progress_bar.finish()
# ------------------------------------ 10. Dumping statistics report -------------------------------------------
next_step()
if args.exec_graph_path:
try:
exec_graph_info = exe_network.get_exec_graph_info()
exec_graph_info.serialize(args.exec_graph_path)
logger.info("Executable graph is stored to {}".format(
args.exec_graph_path))
del exec_graph_info
except Exception as e:
logging.exception(e)
if args.perf_counts:
for ni in range(int(nireq)):
perf_counts = exe_network.requests[ni].get_perf_counts()
logger.info(
"Pefrormance counts for {}-th infer request".format(ni))
for layer, stats in perf_counts.items():
max_layer_name = 30
print("{:<30}{:<15}{:<30}{:<20}{:<20}{:<20}".format(
layer[:max_layer_name - 4] + '...' if
(len(layer) >= max_layer_name) else layer,
stats['status'],
'layerType: ' + str(stats['layer_type']),
'realTime: ' + str(stats['real_time']),
'cpu: ' + str(stats['cpu_time']),
'execType: ' + str(stats['exec_type'])))
print("Count: {} iterations".format(iteration))
print("Duration: {:.2f} ms".format(
getDurationInMilliseconds(total_duration_sec)))
if not MULTI_DEVICE_NAME in device_name:
print("Latency: {:.4f} ms".format(latency_ms))
print("Throughput: {:.2f} FPS".format(fps))
del exe_network
del ie
next_step.step_id = 0
except Exception as e:
logging.exception(e)
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
"""
Initialize any class variables desired
"""
self.net = None
self.plugin = None
self.input_blob = None
self.out_blob = None
self.net_plugin = None
self.infer_request_handle = None
def load_model(self,
model,
device,
input_size,
output_size,
num_requests,
cpu_extension=None,
plugin=None):
"""
Loads a network and an image to the Inference Engine plugin.
:param model: .xml file of pre trained model
:param cpu_extension: extension for the CPU device
:param device: Target device
:param input_size: Number of input layers
:param output_size: Number of output layers
:param num_requests: Index of Infer request value. Limited to device capabilities.
:param plugin: Plugin for specified device
:return: Shape of input layer
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device
# and load extensions library if specified
if not plugin:
log.info("Initializing plugin for {} device...".format(device))
self.plugin = IEPlugin(device=device)
else:
self.plugin = plugin
if cpu_extension and 'CPU' in device:
self.plugin.add_cpu_extension(cpu_extension)
# Read IR
log.info("Reading IR...")
self.net = IENetwork(model=model_xml, weights=model_bin)
log.info("Loading IR to the plugin...")
if self.plugin.device == "CPU":
supported_layers = self.plugin.get_supported_layers(self.net)
not_supported_layers = \
[l for l in self.net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by "
"the plugin for specified device {}:\n {}".format(
self.plugin.device,
', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path"
" in command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
if num_requests == 0:
# Loads network read from IR to the plugin
self.net_plugin = self.plugin.load(network=self.net)
else:
self.net_plugin = self.plugin.load(network=self.net,
num_requests=num_requests)
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
assert len(self.net.inputs.keys()) == input_size, \
"Supports only {} input topologies".format(len(self.net.inputs))
assert len(self.net.outputs) == output_size, \
"Supports only {} output topologies".format(len(self.net.outputs))
return self.plugin, self.get_input_shape()
def load_model_2(self,
model,
device,
input_size,
output_size,
num_requests,
cpu_extension=None,
plugin=None):
"""
Load the model
Check for supported layers
Add any necessary extensions
Return the loaded inference plugin
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device
# and load extensions library if specified
if not plugin:
log.info("Initializing plugin for {} device...".format(device))
self.plugin = IECore()
else:
self.plugin = plugin
if cpu_extension and 'CPU' in device:
self.plugin.add_extension(cpu_extension, "CPU")
# Read IR
log.info("Reading IR...")
self.net = IECore.load_network(model=model_xml, weights=model_bin)
log.info("Loading IR to the plugin...")
if "CPU" in device:
supported_layers = self.plugin.query_network(self.net, "CPU")
not_supported_layers = \
[l for l in self.net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by "
"the plugin for specified device {}:\n {}".format(
device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path"
" in command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
if num_requests == 0:
# Loads network read from IR to the plugin
self.net_plugin = self.plugin.load_network(network=self.net,
device_name=device)
else:
self.net_plugin = self.plugin.load_network(
network=self.net,
num_requests=num_requests,
device_name=device)
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
assert len(self.net.inputs.keys()) == input_size, \
"Supports only {} input topologies".format(len(self.net.inputs))
assert len(self.net.outputs) == output_size, \
"Supports only {} output topologies".format(len(self.net.outputs))
return self.plugin, self.get_input_shape()
def get_input_shape(self):
"""
Return the shape of the input layer
"""
return self.net.inputs[self.input_blob].shape
def exec_net(self, request_id, frame):
"""
Start an asynchronous request
Return any necessary information
Note: You may need to update the function parameters.
"""
self.infer_request_handle = self.net_plugin.start_async(
request_id=request_id, inputs={self.input_blob: frame})
return self.net_plugin
def wait(self, request_id):
"""
Wait for the request to be complete.
Return any necessary information
Note: You may need to update the function parameters.
"""
request_waiting = self.net_plugin.requests[request_id].wait(-1)
return request_waiting
def get_output(self, request_id, output=None):
"""
Extract and return the output results
Note: You may need to update the function parameters.
"""
if output:
return self.infer_request_handle.outputs[output]
else:
return self.net_plugin.requests[request_id].outputs[self.out_blob]
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.net = None
self.plugin = None
self.input_blob = None
self.output_blob = None
self.net_plugin = None
self.infer_request = None
def load_model(self,
model,
device,
num_requests,
cpu_extension=None,
plugin=None):
### TODO: Load the model ###
load_model_xml = model
load_model_bin = os.path.splitext(load_model_xml)[0] + ".bin"
while not plugin:
log.info("Please wait. Starting plugin for {} device... ".format(
device))
self.plugin = IECore()
else:
self.plugin = plugin
if cpu_extension and CPU in device:
self.plugin.add_cpu_extension(cpu_extension)
log.info('Reading IR, Please wait.')
self.net = IENetwork(model=load_model_xml, weights=load_model_bin)
log.info(
'Completed. Loading IR to the plugin. This may take some time')
### TODO: Check for supported layers ###
if self.plugin.device == "CPU":
supported_layers = self.plugin.get_supported_layers(self.net)
unsupported_layers = [
l for l in self.net.layers.key() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error(
'There are a number of unsupported layers found: '.format(
unsupported_layers))
sys.exit(1)
if num_request == 0:
self.net_plugin = self.plugin.load(network=self.net)
else:
self.net_plugin = self.plugin.load(network=self.net,
num_requests=num_requests)
self.input_blob = next(iter(self.net.input))
self.output_blob = next(iter(self.net.output))
if len(self.net.inputs.key()) == input_size:
log.error(
'Sorry, this app supports {} input topologies. Please make the necessary changes and try again'
.format(len(self.net.inputs)))
sys.exit(1)
if len(self.net.outputs) == output_size:
log.error(
'Sorry, this app supports {} output topologies. Please make the necessary changes and try again'
.format(len(self.net.inputs)))
sys.exit(1)
return self.plugin, self.get_input_shape
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.net.inputs[self.input.blob].shape
def exec_net(self, request_id, frame):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.infer_request = self.net_plugin.start_async(
request_id=request_id, inputs={self.input_blob: frame})
return self.net_plugin
def wait(self, request_id):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
wait_status = self.net_plugin.requests[request_id].wait(-1)
return wait_status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
if output:
result = self.infer_request.outputs[output]
else:
result = self.net_plugin.requests[request_id].outputs[
self.output_blob]
return result
def delete_instances(self):
del self.net_plugin
del self_plugin
del self.net
class Model_FLD:
'''
Class for the Face Landmarks Detection Model.
'''
def __init__(self, model_name, device, extensions):
self.plugin = None
self.net = None
self.input_blob = None
self.output_blob = None
self.exec_net = None
self.infer_request = None
self.input_shape = None
self.output_shape = None
self.device = device
self.extension = extensions
self.model = model_name
self.outputs = None
self.eyes_coords_dict = None
def load_model(self):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
model_xml = self.model + ".xml"
model_weights = self.model + ".bin"
self.plugin = IECore()
self.net = IENetwork(model_xml, model_weights)
self.exec_net = self.plugin.load_network(network=self.net,
device_name=self.device,
num_requests=1)
if self.extension and 'CPU' in self.device:
self.plugin.add_cpu_extension(self.extension)
self.check_model()
self.input_blob = next(iter(self.net.inputs))
self.output_blob = next(iter(self.net.outputs))
self.input_shape = self.net.inputs[self.input_blob].shape
self.output_shape = self.net.outputs[self.output_blob].shape
def predict(self, image, benchmark_timing):
'''
This method is meant for running predictions on the input image.
'''
self.exec_net.start_async(request_id=0,
inputs={self.input_blob: image})
if self.exec_net.requests[0].wait(-1) == 0:
self.outputs = self.exec_net.requests[0].outputs[self.output_blob]
if benchmark_timing:
pp = PrettyPrinter(indent=4)
print('Benchmark Timing for Facial_Landmark_Detection')
pp.pprint(self.exec_net.requests[0].get_perf_counts())
# Write get_perf_counts() data to a text file
data = (self.exec_net.requests[0].get_perf_counts())
self.write_benchmark(
'Benchmark Timing for Facial_Landmark_Detection', data)
return self.outputs
def check_model(self):
supported_layers = self.plugin.query_network(network=self.net,
device_name=self.device)
unsupported_layers = [
l for l in self.net.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error('Unsupported layers found:{}'.format(unsupported_layers))
log.error(
'Check for any extensions for these unsupported layers available for adding to IECore'
)
exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
global pimage
try:
pimage = cv2.resize(image,
(self.input_shape[3], self.input_shape[2]))
pimage = pimage.transpose((2, 0, 1))
pimage = pimage.reshape(pimage, (1, *pimage.shape))
except Exception as e:
log.error(str(e))
return pimage
def preprocess_output(self, image, outputs, facebox, display):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
The net outputs a blob with the shape: [1, 10], containing a row-vector of 10 floating point values
for five landmarks coordinates in the form (x0, y0, x1, y1, ..., x5, y5).
All the coordinates are normalized to be in range [0,1]
'''
"""
# Output layer names in Inference Engine format:
# landmarks-regression-retail-0009:
# "95", [1, 10, 1, 1], containing a row-vector of 10 floating point values for five landmarks
# coordinates in the form (x0, y0, x1, y1, ..., x5, y5).
# All the coordinates are normalized to be in range [0,1]
"""
normed_landmarks = outputs.reshape(1, 10)[0]
height = facebox[3] - facebox[1]
width = facebox[2] - facebox[0]
if display:
# Drawing the box/boxes
for i in range(2):
x = int(normed_landmarks[i * 2] * width)
y = int(normed_landmarks[i * 2 + 1] * height)
cv2.circle(image, (facebox[0] + x, facebox[1] + y), 30,
(0, 255, i * 255), 1)
left_eye_point = [
normed_landmarks[0] * width, normed_landmarks[1] * height
]
right_eye_point = [
normed_landmarks[2] * width, normed_landmarks[3] * height
]
return left_eye_point, right_eye_point
def get_model_name(self):
return self.model
def write_benchmark(self, title, data):
with open("FacialLandmarks_benchmark_timing.txt", "a") as f:
f.write(str(title) + "\n")
f.write(str(data) + '\n')
f.close()
class Model_GE:
'''
Class for the gaze estimation Model.
'''
def __init__(self, model_name, device, extensions):
self.plugin = None
self.net = None
self.exec_net = None
self.infer_request = None
self.output_shape = None
self.device = device
self.extension = extensions
self.outputs = None
self.head_pose_angles = None
self.left_eye_image = None
self.right_eye_image = None
self.model = model_name
self.head_pose_angles_shape = None
self.left_eye_shape = None
self.right_eye_shape = None
def load_model(self,device):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
model_xml = self.model + ".xml"
model_weights = self.model + ".bin"
self.plugin = IECore()
self.net = IENetwork(model_xml, model_weights)
self.exec_net = self.plugin.load_network(network=self.net, device_name=device, num_requests=1)
if self.extension and 'CPU' in self.device:
self.plugin.add_cpu_extension(self.extension)
self.check_model()
self.input_blob = next(iter(self.net.inputs))
self.input_blob2 = next(iter(self.net.inputs))
self.input_shape = self.net.inputs[self.input_blob].shape
self.output_blob = next(iter(self.net.outputs))
self.output_shape = self.net.outputs[self.output_blob].shape
def predict(self, processed_left_eye, processed_right_eye, head_position, benchmark_timing):
input_dict = {'left_eye_image': processed_left_eye,
'right_eye_image': processed_right_eye,
'head_pose_angles': head_position}
infer_request_handle = self.exec_net.start_async(request_id=0, inputs=input_dict)
infer_status = infer_request_handle.wait()
if infer_status == 0:
self.outputs = self.exec_net.requests[0].outputs[self.output_blob][0]
if benchmark_timing:
pp = PrettyPrinter(indent=4)
print('Benchmark Timing for Gaze_detection')
pp.pprint(self.exec_net.requests[0].get_perf_counts())
# Write get_perf_counts() data to a text file
data = pp.pprint(self.exec_net.requests[0].get_perf_counts())
self.write_benchmark('Benchmark Timing for Gaze_Estimation', data)
return self.outputs
def check_model(self):
supported_layers = self.plugin.query_network(network=self.net, device_name=self.device)
unsupported_layers = [l for l in self.net.layers.keys() if l not in supported_layers]
if len(unsupported_layers) != 0:
log.error('Unsupported layers found:{}'.format(unsupported_layers))
log.error('Check for any extensions for these unsupported layers available for adding to IECore')
exit(1)
def eye_corp(self, face, eyepoint):
Eye_shape = [1, 3, 60, 60]
# cropping the eye
x_center = eyepoint[0]
y_center = eyepoint[1]
width = Eye_shape[3]
height = Eye_shape[2]
# ymin:ymax, xmin:xmax
face_width = face.shape[0]
face_height = face.shape[1]
face_array = np.array(face)
ymin = int(y_center - height // 2) if int(y_center - height // 2) >= 0 else 0
ymax = int(y_center + height // 2) if int(y_center + height // 2) <= face_height else face_height
xmin = int(x_center - width // 2) if int(x_center - width // 2) >= 0 else 0
xmax = int(x_center + width // 2) if int(x_center + width // 2) <= face_width else face_width
eye_image = face_array[ymin:ymax, xmin:xmax]
return eye_image
def resize_frame(self, frame):
global p_frame
try:
p_frame = cv2.resize(frame, (60, 60))
p_frame = p_frame.transpose((2, 0, 1))
p_frame = p_frame.reshape(1, *p_frame.shape)
except Exception as e:
log.error(str(e))
return p_frame
def preprocess_input(self, face, left_eye_point, right_eye_point):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
Blob in the format [BxCxHxW] where:
B - batch size
C - number of channels
H - image height
W - image width
with the name left_eye_image and the shape [1x3x60x60].
Blob in the format [BxCxHxW] where:
B - batch size
C - number of channels
H - image height
W - image width
with the name right_eye_image and the shape [1x3x60x60].
Blob in the format [BxC] where:
B - batch size
C - number of channels
with the name head_pose_angles and the shape [1x3].
'''
left_eye_image = self.eye_corp(face, left_eye_point)
right_eye_image = self.eye_corp(face, right_eye_point)
# eye shape [1x3x60x60]
p_frame_left = self.resize_frame(left_eye_image)
p_frame_right = self.resize_frame(right_eye_image)
return p_frame_left, p_frame_right
def preprocess_output(self, outputs, image, face, facebox, left_eye_point, right_eye_point, display):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
output = np.squeeze(outputs)
x = output[0]
y = output[1]
z = output[2]
# left eye
xmin, ymin, _, _ = facebox
l_x_center = left_eye_point[0]
l_y_center = left_eye_point[1]
left_eye_center_x = int(xmin + l_x_center)
left_eye_center_y = int(ymin + l_y_center)
# right eye
r_x_center = right_eye_point[0]
r_y_center = right_eye_point[1]
right_eye_center_x = int(xmin + r_x_center)
right_eye_center_y = int(ymin + r_y_center)
if display:
cv2.arrowedLine(image, (left_eye_center_x, left_eye_center_y),
(left_eye_center_x + int(x * 100), left_eye_center_y + int(-y * 100)),
(255, 0, 0), 3)
cv2.arrowedLine(image, (right_eye_center_x, right_eye_center_y),
(right_eye_center_x + int(x * 100), right_eye_center_y + int(-y * 100)),
(0, 255, 0), 3)
return [x, y, z]
def get_model_name(self):
return self.model
def write_benchmark(self, title, data):
with open("Gaze_benchmark_timing.txt", "a") as f:
f.write(str(title) + "\n")
f.write(str(data) + '\n')
f.close()
class Model_HPE:
'''
Class for the Head Pose Estimation Model.
'''
def __init__(self, model_name, device, extensions):
self.plugin = None
self.net = None
self.input_blob = None
self.output_blob = None
self.exec_net = None
self.infer_request = None
self.input_shape = None
self.output_shape = None
self.input_name = None
self.device = device
self.extension = extensions
self.model = model_name
self.output = None
def load_model(self):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
model_xml = self.model + ".xml"
model_weights = self.model + ".bin"
self.plugin = IECore()
self.net = IENetwork(model_xml, model_weights)
self.exec_net = self.plugin.load_network(network=self.net, device_name=self.device,
num_requests=1)
if self.extension and 'CPU' in self.device:
self.plugin.add_cpu_extension(self.extension)
self.check_model()
self.input_blob = next(iter(self.net.inputs))
self.input_shape = self.net.inputs[self.input_blob].shape
self.output_blob = next(iter(self.net.outputs))
self.output_shape = self.net.outputs[self.output_blob].shape
def predict(self, image, benchmark_timing):
self.exec_net.start_async(request_id=0, inputs={self.input_blob: image})
if self.exec_net.requests[0].wait(-1) == 0:
self.result = self.exec_net.requests[0].outputs
if benchmark_timing:
pp = PrettyPrinter(indent=4)
print('Benchmark Timing for Head_Pose_Estimation')
pp.pprint(self.exec_net.requests[0].get_perf_counts())
# Write get_perf_counts() data to a text file
data = (self.exec_net.requests[0].get_perf_counts())
self.write_benchmark('Benchmark Timing for Head_Pose_Estimation', data)
return self.result
def check_model(self):
supported_layers = self.plugin.query_network(network=self.net, device_name=self.device)
unsupported_layers = [l for l in self.net.layers.keys() if l not in supported_layers]
if len(unsupported_layers) != 0:
log.error("Unsupported layers found: {}".format(unsupported_layers))
log.error("Check whether extensions are available to add to IECore.")
exit(1)
def preprocess_input(self, image):
temp = image.copy()
temp = cv2.resize(temp, (self.input_shape[3], self.input_shape[2])) # n,c,h,w
temp = temp.transpose((2, 0, 1))
temp = temp.reshape(1, *temp.shape)
return temp
def preprocess_output(self, image, outputs, facebox, face, display):
output = []
output.append(outputs['angle_y_fc'].tolist()[0][0])
output.append(outputs['angle_p_fc'].tolist()[0][0])
output.append(outputs['angle_r_fc'].tolist()[0][0])
pitch = np.squeeze(outputs['angle_p_fc'])
roll = np.squeeze(outputs['angle_r_fc'])
yaw = np.squeeze(outputs['angle_y_fc'])
axes_op = np.array([pitch, roll, yaw])
if display:
xmin, ymin, _, _ = facebox
face_center = (xmin + face.shape[1] / 2, ymin + face.shape[0] / 2, 0)
self.draw_axes(image, face_center, yaw, pitch, roll)
return axes_op
# code source: https://knowledge.udacity.com/questions/171017
def draw_axes(self, frame, center_of_face, yaw, pitch, roll):
focal_length = 950.0
scale = 100
yaw *= np.pi / 180.0
pitch *= np.pi / 180.0
roll *= np.pi / 180.0
cx = int(center_of_face[0])
cy = int(center_of_face[1])
Rx = np.array([[1, 0, 0],
[0, math.cos(pitch), -math.sin(pitch)],
[0, math.sin(pitch), math.cos(pitch)]])
Ry = np.array([[math.cos(yaw), 0, -math.sin(yaw)],
[0, 1, 0],
[math.sin(yaw), 0, math.cos(yaw)]])
Rz = np.array([[math.cos(roll), -math.sin(roll), 0],
[math.sin(roll), math.cos(roll), 0],
[0, 0, 1]])
# ref: https://www.learnopencv.com/rotation-matrix-to-euler-angles/
R = Rz @ Ry @ Rx
camera_matrix = self.build_camera_matrix(center_of_face, focal_length)
xaxis = np.array(([1 * scale, 0, 0]), dtype='float32').reshape(3, 1)
yaxis = np.array(([0, -1 * scale, 0]), dtype='float32').reshape(3, 1)
zaxis = np.array(([0, 0, -1 * scale]), dtype='float32').reshape(3, 1)
zaxis1 = np.array(([0, 0, 1 * scale]), dtype='float32').reshape(3, 1)
o = np.array(([0, 0, 0]), dtype='float32').reshape(3, 1)
o[2] = camera_matrix[0][0]
xaxis = np.dot(R, xaxis) + o
yaxis = np.dot(R, yaxis) + o
zaxis = np.dot(R, zaxis) + o
zaxis1 = np.dot(R, zaxis1) + o
xp2 = (xaxis[0] / xaxis[2] * camera_matrix[0][0]) + cx
yp2 = (xaxis[1] / xaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(frame, (cx, cy), p2, (0, 0, 255), 2)
xp2 = (yaxis[0] / yaxis[2] * camera_matrix[0][0]) + cx
yp2 = (yaxis[1] / yaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(frame, (cx, cy), p2, (0, 255, 0), 2)
xp1 = (zaxis1[0] / zaxis1[2] * camera_matrix[0][0]) + cx
yp1 = (zaxis1[1] / zaxis1[2] * camera_matrix[1][1]) + cy
p1 = (int(xp1), int(yp1))
xp2 = (zaxis[0] / zaxis[2] * camera_matrix[0][0]) + cx
yp2 = (zaxis[1] / zaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(frame, p1, p2, (255, 0, 0), 2)
cv2.circle(frame, p2, 3, (255, 0, 0), 2)
return frame
def build_camera_matrix(self, center_of_face, focal_length):
cx = int(center_of_face[0])
cy = int(center_of_face[1])
camera_matrix = np.zeros((3, 3), dtype='float32')
camera_matrix[0][0] = focal_length
camera_matrix[0][2] = cx
camera_matrix[1][1] = focal_length
camera_matrix[1][2] = cy
camera_matrix[2][2] = 1
return camera_matrix
def get_model_name(self):
return self.model
def write_benchmark(self, title, data):
with open("headpose_benchmark_timing.txt", "a") as f:
f.write(str(title) + "\n")
f.write(str(data) + '\n')
f.close()
