def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device and load extensions library if specified
log.info("Initializing plugin for {} device...".format(args.device))
ie = IECore()
#plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir)
if args.cpu_extension and 'CPU' in args.device:
log.info("Loading plugins for {} device...".format(args.device))
#ie.add_extension(args.cpu_extension, args.device) // openvino2019-R1
ie.add_extension(args.cpu_extension, "CPU")
# Read IR
log.info("Reading IR...")
net = IENetwork(model=model_xml, weights=model_bin)
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(
net.inputs.keys()) == 1, "Sample supports only single input topologies"
assert len(
net.outputs) == 1, "Sample supports only single output topologies"
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
if args.input == 'cam':
input_stream = 0
out_file_name = 'cam'
else:
input_stream = args.input
assert os.path.isfile(args.input), "Specified input file doesn't exist"
out_file_name = os.path.splitext(os.path.basename(args.input))[0]
log.info("Loading IR to the plugin...")
exec_net = ie.load_network(network=net,
num_requests=args.number_infer_requests,
device_name=args.device)
log.info(
"Starting inference in async mode, {} requests in parallel...".format(
args.number_infer_requests))
job_id = str(os.environ['PBS_JOBID'])
result_file = open(
os.path.join(args.output_dir, 'output_' + job_id + '.txt'), "w")
pre_infer_file = os.path.join(args.output_dir,
'pre_progress_' + job_id + '.txt')
infer_file = os.path.join(args.output_dir, 'i_progress_' + job_id + '.txt')
processed_vid = '/tmp/processed_vid.bin'
# Read and pre-process input image
if isinstance(net.inputs[input_blob], list):
n, c, h, w = net.inputs[input_blob]
else:
n, c, h, w = net.inputs[input_blob].shape
del net
cap = cv2.VideoCapture(input_stream)
video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
if video_len < args.number_infer_requests:
args.number_infer_requests = video_len
#Pre inference processing, read mp4 frame by frame, process using openCV and write to binary file
width = int(cap.get(3))
height = int(cap.get(4))
CHUNKSIZE = n * c * w * h
id_ = 0
with open(processed_vid, 'w+b') as f:
time_start = time.time()
while cap.isOpened():
ret, next_frame = cap.read()
if not ret:
break
in_frame = cv2.resize(next_frame, (w, h))
in_frame = in_frame.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
in_frame = in_frame.reshape((n, c, h, w))
bin_frame = bytearray(in_frame)
f.write(bin_frame)
id_ += 1
if id_ % 10 == 0:
progressUpdate(pre_infer_file,
time.time() - time_start, id_, video_len)
cap.release()
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
current_inference = 0
previous_inference = 1 - args.number_infer_requests
infer_requests = exec_net.requests
frame_count = 0
try:
infer_time_start = time.time()
with open(processed_vid, "rb") as data:
while frame_count < video_len:
# Read next frame from input stream if available and submit it for inference
byte = data.read(CHUNKSIZE)
if not byte == b"":
deserialized_bytes = np.frombuffer(byte, dtype=np.uint8)
in_frame = np.reshape(deserialized_bytes,
newshape=(n, c, h, w))
exec_net.start_async(request_id=current_inference,
inputs={input_blob: in_frame})
# Retrieve the output of an earlier inference request
if previous_inference >= 0:
status = infer_requests[previous_inference].wait()
if status is not 0:
raise Exception(
"Infer request not completed successfully")
# Parse inference results
res = infer_requests[previous_inference].outputs[out_blob]
processBoxes(frame_count, res, labels_map,
args.prob_threshold, width, height,
result_file)
frame_count += 1
# Write data to progress tracker
if frame_count % 10 == 0:
progressUpdate(infer_file,
time.time() - infer_time_start,
frame_count + 1, video_len + 1)
# Increment counter for the inference queue and roll them over if necessary
current_inference += 1
if current_inference >= args.number_infer_requests:
current_inference = 0
previous_inference += 1
if previous_inference >= args.number_infer_requests:
previous_inference = 0
# End while loop
total_time = time.time() - infer_time_start
with open(os.path.join(args.output_dir, 'stats_{}.txt'.format(job_id)),
'w') as f:
f.write('{:.3g} \n'.format(total_time))
f.write('{} \n'.format(frame_count))
result_file.close()
finally:
log.info("Processing done...")
del exec_net
class facelandmarksdetection:
def __init__(self, model_name, device='CPU', extensions=None):
self.model_name = model_name
self.device = device
self.extensions = extensions
self.model_structure = self.model_name
self.model_weights = self.model_name.split('.')[0]+'.bin'
self.plugin = None
self.network = None
self.exec_net = None
self.input_name = None
self.input_shape = None
self.output_names = None
self.output_shape = None
try:
self.model=IENetwork(self.model_structure, self.model_weights)
except Exception as e:
raise ValueError("Could not Initialise the network. Have you enterred the correct model path?")
self.input_name=next(iter(self.model.inputs))
self.input_shape=self.model.inputs[self.input_name].shape
self.output_name=next(iter(self.model.outputs))
self.output_shape=self.model.outputs[self.output_name].shape
def load_model(self):
self.plugin=IECore()
supported_layers = self.plugin.query_network(network=self.model, device_name=self.device)
unsupported_layers = [l for l in self.model.layers.keys() if l not in supported_layers]
if len(unsupported_layers)!=0:
print("unsupported layers found")
exit(1)
self.exec_net=self.plugin.load_network(network=self.model,device_name=self.device,num_requests=1)
def predict(self, image):
self.processed_image=self.preprocess_input(image)
outputs = self.exec_net.infer({self.input_name:self.processed_image})
coords = self.preprocess_output(outputs)
h=image.shape[0]
w=image.shape[1]
coords = coords* np.array([w, h, w, h])
coords = coords.astype(np.int32)
l_xmin=coords[0]-10
l_xmax=coords[0]+10
l_ymin=coords[1]-10
l_ymax=coords[1]+10
r_xmin=coords[2]-10
r_xmax=coords[2]+10
r_ymin=coords[3]-10
r_ymax=coords[3]+10
left_eye = image[l_ymin:l_ymax, l_xmin:l_xmax]
right_eye = image[r_ymin:r_ymax, r_xmin:r_xmax]
coords = [[l_xmin,l_ymin,l_xmax,l_ymax], [r_xmin,r_ymin,r_xmax,r_ymax]]
return left_eye, right_eye, coords
def check_model(self):
raise NotImplementedError
def preprocess_input(self, image):
image_ct = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
self.image=cv2.resize(image_ct,(self.input_shape[3],self.input_shape[2])) ## cv2.resize(frame, (w, h))
self.image=self.image.transpose((2, 0, 1))
self.image=self.image.reshape(1, *self.image.shape)
return self.image
def preprocess_output(self, outputs):
res=outputs[self.output_name][0]
lx = res[0].tolist()[0][0]
ly = res[1].tolist()[0][0]
rx = res[2].tolist()[0][0]
ry = res[3].tolist()[0][0]
return(lx,ly,rx,ry)
def main():
# arguments
parser = ArgumentParser()
parser.add_argument(
"-m",
"--model",
help="Required. Path to an .xml file with a trained model",
required=True,
type=str)
parser.add_argument("-i",
"--input",
help="Required. Path to a input image file",
required=True,
type=str)
parser.add_argument(
"-l",
"--cpu_extension",
help=
"Optional. Required for CPU custom layers. Absolute MKLDNN (CPU)-targeted custom layers. "
"Absolute path to a shared library with the kernels implementations",
type=str,
default=None)
parser.add_argument(
"-d",
"--device",
help=
"Optional. Specify the target device to infer on; CPU, GPU, FPGA, HDDL or MYRIAD is acceptable. "
"Sample will look for a suitable plugin for device specified. Default value is CPU",
default="CPU",
type=str)
args = parser.parse_args()
# logging
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
log.info("creating inference engine")
ie = IECore()
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("Loading network")
net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin")
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify a CPU extensions library path in sample's command line parameters "
"using -l or --cpu_extension command line argument")
sys.exit(1)
assert len(
net.input_info) == 1, "Sample supports only single input topologies"
assert len(
net.outputs) == 1, "Sample supports only single output topologies"
log.info("preparing input blobs")
input_blob = next(iter(net.input_info))
out_blob = next(iter(net.outputs))
net.batch_size = 1
# read and pre-process input image
_, _, height, width = net.input_info[input_blob].input_data.shape
image = cv2.imread(args.input, cv2.IMREAD_COLOR)
(input_height, input_width) = image.shape[:-1]
# resize
if (input_height, input_width) != (height, width):
log.info("Image is resized from {} to {}".format(
image.shape[:-1], (height, width)))
image = cv2.resize(image, (width, height), cv2.INTER_CUBIC)
# prepare input
image = image.astype(np.float32)
image = image.transpose((2, 0, 1))
image_input = np.expand_dims(image, 0)
# loading model to the plugin
log.info("loading model to the plugin")
exec_net = ie.load_network(network=net, device_name=args.device)
# start sync inference
log.info("starting inference")
res = exec_net.infer(inputs={input_blob: image_input})
# processing output blob
log.info("processing output blob")
disp = res[out_blob][0]
# resize disp to input resolution
disp = cv2.resize(disp, (input_width, input_height), cv2.INTER_CUBIC)
# rescale disp
disp_min = disp.min()
disp_max = disp.max()
if disp_max - disp_min > 1e-6:
disp = (disp - disp_min) / (disp_max - disp_min)
else:
disp.fill(0.5)
# pfm
out = os.path.join(os.path.dirname(__file__), 'disp.pfm')
cv2.imwrite(out, disp)
log.info("Disparity map was saved to {}".format(out))
# png
out = os.path.join(os.path.dirname(__file__), 'disp.png')
plt.imsave(out, disp, vmin=0, vmax=1, cmap='inferno')
log.info("Color-coded disparity image was saved to {}".format(out))
log.info(
"This demo is an API example, for any performance measurements please use "
"the dedicated benchmark_app tool from the openVINO toolkit\n")
class face_detection:
'''
Class for the Face Detection Model.
'''
def __init__(self, model, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model = model
self.device = device
self.extension = extensions
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
self.device_list = []
self.print_once = True
self.network_input_shape = None
self.filtered_result = [[]]
# ====== Load model files, verify model, and get the input shap parameters at start ======
log.info("<---------- class Facedetection model --------->")
self.load_model()
self.check_model()
self.network_input_shape = self.get_input_shape()
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.plugin = IECore()
log.info("------device avaibility--------")
for available_devices in self.plugin.available_devices: #Dont use device variable, conflicts.
self.device_list.append(available_devices)
log.info("Available device: " +
str(self.device_list)) #get name of available devices
log.info("---------Plugin version--------")
ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info
log.info("descr: maj.min.num" + str(ver.description) + "." +
str(ver.major) + "." + str(ver.minor) + "." +
str(ver.build_number))
### Load IR files into their related class
model_xml = self.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Check if path is not given or model name doesnt ends with .xml
if model_xml is not None and model_xml.find('.xml') != -1:
f, s = model_xml.rsplit(
".", 1
) #check from last "." and "r"split only one element from last
model_bin = f + ".bin"
else:
log.error(
"Error! Model files are not found or invalid, check paths")
log.error("Program stopped")
sys.exit() #exit program no further execution
log.info("-------------Model path----------")
log.info("XML: " + str(model_xml))
log.info("bin: " + str(model_bin))
self.network = IENetwork(model=model_xml, weights=model_bin)
log.info("ModelFiles are successfully loaded into IENetwork")
return
def check_model(self):
'''
This function will check the input model for compatibility with hardware,
It will check for supported and unsuported layers if any.
No return required.
'''
### Check for supported layers ###
log.info("Checking for supported Network layers...")
# Query network will return all the layer, required all the time if device changes.
supported_layers = self.plugin.query_network(network=self.network,
device_name="CPU")
log.info("------Status of default Network layers--------")
log.info("No. of Layers in network: " + str(len(self.network.layers)))
log.info("No. of supported layers:" + str(len(supported_layers)))
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.info("Unsupported layers found:" + str(unsupported_layers))
log.info("CPU extension required and adding...")
#exit(1)
### Adding any necessary extensions ###
if self.extension and "CPU" in self.device:
self.plugin.add_extension(self.extension, self.device)
log.info("Checking for CPU extension compatibility...")
# Again Query network will return fresh list of supported layers.
supported_layers = self.plugin.query_network(
network=self.network, device_name="CPU")
log.info(
"------Status of Network layers with CPU Extension--------"
)
log.info("No. of Layers in network:" +
str(len(self.network.layers)))
log.info("No. of supported layers:" +
str(len(supported_layers)))
log.info("CPU extension added sucessfully!")
unsupported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error("Unsupported layers found: " +
str((unsupported_layers)))
log.error("Error! Model not supported, Program stopped")
exit()
else:
log.error("Error! cpu extension not found")
log.error("Program stopped")
exit()
else:
log.info("All the layers are supported, No CPU extension required")
# This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference
self.exec_network = self.plugin.load_network(self.network, "CPU")
print("IR successfully loaded into Inference Engine")
return
def get_input_shape(self):
'''
Get dimensions of inputs
'''
### Get the input layer informations ###
log.info("-----Accessing input layer information-----")
log.info('Face detection model Network input layers = ' +
str(list(self.network.inputs)))
log.info('Face detection model Network input layers type: ' +
str(type(self.network.inputs)))
self.input_blob = next(iter(self.network.inputs)) #Origional
log.info("-------------------------------")
return self.network.inputs[self.input_blob].shape #Origional
def wait(self):
### Wait for the Async request to be complete. ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### Extract and return the output results
self.output_blob = next(iter(self.network.outputs))
# First return the name of blob as dictionary and second output of first blob as Nd array
return self.exec_network.requests[
0].outputs, self.exec_network.requests[0].outputs[self.output_blob]
def preprocess_input(self, input_frames_raw, network_input_shape_height,
network_input_shape_width):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
p_frame = cv2.resize(
input_frames_raw,
(network_input_shape_height,
network_input_shape_width)) #Resize as per network input spec.
p_frame = p_frame.transpose((2, 0, 1)) #swap channel cxhxw
p_frame = p_frame.reshape(
1,
*p_frame.shape) #add one axis 1 to make 4D shape for network input
#print(p_frame.shape) #Debug output
return p_frame
def predict(self, input_frames_raw, input_frame_raw_height,
input_frame_raw_width):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image or video frame.
input: RBG image in jpg format or video frame.
'''
# pre-process origional frame to match network inputs.
p_frame = self.preprocess_input(input_frames_raw,
self.network_input_shape[3],
self.network_input_shape[2])
# Run Async inference
self.exec_network.start_async(
request_id=0, #Origional
inputs={self.input_blob: p_frame}) # run inference
# wait until result available
if self.wait() == 0:
### the results of the inference request ###
blob, result = self.get_output()
# Print available blob infirmation
blob_list = []
if self.print_once: # Print only Once
self.print_once = False
for name, output_ in blob.items(
): #Find the possible BLOBS for name,
blob_list.append(name)
log.info(
"The name of available blob of face detection model is: " +
str(blob_list))
return result
# Load the model and create the inference engine
log.info(f'Loading model')
model_xml = args['model']
model_bin = os.path.splitext(model_xml)[0] + '.bin'
log.info(f'... model file {model_xml}')
log.info(f'... weights file {model_bin}')
log.info('Creating inference engine')
ie = IECore()
net = IENetwork(model=model_xml, weights=model_bin)
log.info('...Checking that the network can be run on the selected device')
supported_layers = ie.query_network(net, args['device'])
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
'...The following layers are not supported by the device.\n {}'.format(
', '.join(not_supported_layers)))
log.info('...Checking that the network has a single input and output')
assert len(net.inputs.keys()
) == 1, 'The application supports single input topologies.'
assert len(
net.outputs) == 1, 'The application supports single output topologies'
landmark_bin = 'models/landmarks-regression-retail-0009/landmarks-regression-retail-0009.bin'
head_pose_xml = 'models/head-pose-estimation-adas-0001/head-pose-estimation-adas-0001.xml'
head_pose_bin = 'models/head-pose-estimation-adas-0001/head-pose-estimation-adas-0001.bin'
gaze_est_xml = 'models/gaze-estimation-adas-0002/gaze-estimation-adas-0002.xml'
gaze_est_bin = 'models/gaze-estimation-adas-0002/gaze-estimation-adas-0002.bin'
print("INFO: Starting to load models.....")
core = IECore()
model = IENetwork(model_xml, model_bin)
net = core.load_network(model, device, num_requests=1)
fps = FPS().start()
layers_list = core.query_network(network=model, device_name=device)
print(layers_list)
model2 = IENetwork(landmark_xml, landmark_bin)
net2 = core.load_network(model2, device, num_requests=1)
model3 = IENetwork(head_pose_xml, head_pose_bin)
net3 = core.load_network(model3, device, num_requests=1)
model4 = IENetwork(gaze_est_xml, gaze_est_bin)
net4 = core.load_network(model4, device, num_requests=1)
print("INFO: All models loaded...")
input_name = next(iter(net.inputs))
input_shape = net.inputs[input_name].shape
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device="CPU", cpu_extension=None):
### TODO: Load the model ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Initialize the plugin
self.plugin = IECore()
# Add a CPU extension, if applicable
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
self.network = IENetwork(model=model_xml, weights=model_bin)
### TODO: Check for supported layers ###
supported_layers = self.plugin.query_network(network=self.network,
device_name=device)
# Unsupported layers
unsupported_layers = [
layer for layer in self.network.layers.keys()
if layer not in supported_layers
]
### TODO: Add any necessary extensions ###
if unsupported_layers:
log.warning(
"Unsupported layers found: {}".format(unsupported_layers))
self.plugin.add_extension(cpu_extension, device)
# Load the IENetwork into the plugin
self.exec_network = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, image):
### TODO: Start an asynchronous request ###
self.exec_network.start_async(request_id=0,
inputs={self.input_blob: image})
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return
def wait(self):
### TODO: Wait for the request to be complete. ###
status = self.exec_network.requests[0].wait(-1)
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.exec_network.requests[0].outputs[self.output_blob]
class FaceDetection:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
#From params
self.model_weights = model_name + '.bin'
self.model_structure = model_name + '.xml'
self.device = device
self.extensions = extensions
#For app
self.core = None
self.network = None
self.exec_net = None
self.unsupported_layers = None
self.prob_threshold = 0.6
self.image = None
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.core = IECore()
self.network = self.core.read_network(model=self.model_structure,
weights=self.model_weights)
supported_layers = self.core.query_network(network=self.network,
device_name=self.device)
self.unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
self.check_model()
logging.info("Checked face-dection model")
self.exec_net = self.core.load_network(network=self.network,
device_name=self.device,
num_requests=1)
self.input = next(iter(self.network.inputs))
self.output = next(iter(self.network.outputs))
def predict(self, image):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
img_processed = self.preprocess_input(image.copy())
self.image = image
self.exec_net.start_async(request_id=0,
inputs={self.input: img_processed})
while self.exec_net.requests[0].wait(-1) == 0:
result = self.exec_net.requests[0].outputs[self.output]
return self.preprocess_output(result)
def check_model(self):
if len(self.unsupported_layers) != 0:
self.core.add_extension(self.extensions, self.device)
supported_layers = self.core.query_network(network=self.network,
device_name=self.device)
self.unsupported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(self.unsupported_layers) != 0:
logging.error("Unsupported layers")
exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
net_input_shape = []
net_input_shape = self.network.inputs[self.input].shape
p_frame = None
p_frame = cv2.resize(image, (net_input_shape[3], net_input_shape[2]))
p_frame = p_frame.transpose(2, 0, 1)
p_frame = p_frame.reshape(1, *p_frame.shape)
return p_frame
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
face_coords = []
outs = outputs[0][0]
conf = 0
x_min = 0
x_max = 0
y_min = 0
y_max = 0
height = self.image.shape[0]
width = self.image.shape[1]
for out in outs:
conf = out[2]
if conf > self.prob_threshold:
x_min = out[3] * width
y_min = out[4] * height
x_max = out[5] * width
y_max = out[6] * height
face_coords.append([x_min, y_min, x_max, y_max])
if (len(face_coords) == 0):
return None
#Only need one person, so take the first box with good conf
first_face = face_coords[0]
first_face = [round(x) for x in first_face]
first_face_box = self.image[first_face[1]:first_face[3],
first_face[0]:first_face[2]]
return first_face_box, first_face
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model_xml, device="CPU"):
### TODO: Load the model ###
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.plugin = IECore()
self.network = IENetwork(model_xml, model_bin)
### TODO: Check for supported layers ###
supported_layers = self.plugin.query_network(self.network, device)
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if (len(unsupported_layers) > 0):
print("ERROR: Unsupported layers found!")
exit(1)
### DEPRECATED
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
self.exec_network = self.plugin.load_network(self.network, device)
# print("EXEC_NET INPUTS: " + str(self.network.outputs["detection_output"].shape))
self.input_blob = next(
iter(self.network.inputs
)) # "image_tensor" #next(iter(self.network.inputs))
self.output_blob = next(
iter(self.network.outputs
)) # "detection_output" #next(iter(self.network.outputs))
return self.plugin
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, image):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
infer_request = self.exec_network.start_async(0,
{self.input_blob: image})
return infer_request
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### TODO: Extract and return the output results
return self.exec_network.requests[0].outputs[self.output_blob]
class Model_Face_Detection:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, conf=0.5, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model_name = model_name
self.device = device
self.extensions = extensions
self.confidence = conf
self.ie = None
self.network = None
self.exec_network = None
self.infer_request = None
self.input_name = None
self.input_shape = None
self.output_name = None
self.output_shape = None
self.model_width = None
self.model_height = None
self.width = None
self.height = None
self.model_width = None
self.model_height = None
self.frame = None
def load_model(self):
'''
Load the model given IR files.
Defaults to CPU as device for use in the workspace.
Synchronous requests made within.
'''
log.info(f"##### Loading Model: {self.model_name}")
model_xml = self.model_name + ".xml"
model_bin = self.model_name + ".bin"
# Initialize the inference engine
self.ie = IECore()
# Add a CPU extension, if applicable
if self.extensions and "CPU" in self.device:
self.ie.add_extension(self.extensions, self.device)
# Read the IR as a IENetwork
self.network = self.ie.read_network(model=model_xml, weights=model_bin)
self.check_model()
# Load the IENetwork into the inference engine
self.exec_network = self.ie.load_network(self.network, self.device)
# Get the layer's info
self.input_name = next(iter(self.exec_network.inputs))
self.output_name = next(iter(self.exec_network.outputs))
self.input_shape = self.exec_network.inputs[self.input_name].shape
self.output_shape = self.exec_network.outputs[self.output_name].shape
log.info(f"Input shape: {self.input_shape}")
log.info(f"Output shape: {self.output_shape}")
self.model_width = self.input_shape[3]
self.model_height = self.input_shape[2]
log.info(
f'Input image will be resized to ( {self.model_width} x {self.model_height} ) for inference'
)
return self.exec_network
def get_input_shape(self):
'''
Gets the input shape of the network
'''
return self.input_shape
def get_output_shape(self):
'''
Gets the output shape of the network
'''
return self.output_shape
def predict(self, image):
'''
This method is meant for running predictions on the input image.
'''
self.frame = image
self.height = image.shape[0]
self.width = image.shape[1]
frame_inference = self.preprocess_input(image)
outputs = self.exec_network.infer(
inputs={self.input_name: frame_inference})
return self.preprocess_output(outputs[self.output_name])
def check_model(self):
### Check for any unsupported layers, and let the user
### know if anything is missing. Exit the program, if so.
supported_layers = self.ie.query_network(network=self.network,
device_name=self.device)
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error(
"Unsupported layers found: {}".format(unsupported_layers))
log.error(
"Check whether extensions are available to add to IECore.")
exit(1)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
frame_inference = cv2.resize(image,
(self.model_width, self.model_height))
# Transform the image from the original size to the (1, 3, 320, 544) input shape
frame_inference = frame_inference.transpose((2, 0, 1))
frame_inference = frame_inference.reshape(1, *frame_inference.shape)
return frame_inference
def extract_face(self, box):
margin_top = 0
margin = 0
width = self.width
height = self.height
x_min = int(max(box[3] - box[3] * margin, 0) * width)
y_min = int(max(box[4] - box[4] * margin_top, 0) * height)
x_max = int(min(box[5] + box[5] * margin, 1) * width)
y_max = int(min(box[6] + box[6] * margin, 1) * height)
return self.frame[y_min:y_max, x_min:x_max]
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
# Return only the first face
if outputs[0][0][
0] is not None and outputs[0][0][0][2] >= self.confidence:
return self.extract_face(outputs[0][0][0])
else:
return None
class Face_Detector:
'''
Class for the Face Detection Model.
'''
def __init__(self):
self.plugin = None
self.net = None
self.input_name = None
self.output_name = None
self.exec_net = None
self.request = None
def load_model(self, model_path, device='CPU', extensions=None):
start = time.time()
if not os.path.isfile(model_path):
log.error("Wrong model xml path specified" + model_path)
exit(1)
model_xml = model_path
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.plugin = IECore()
if extensions and "CPU" in device:
self.plugin.add_extension(extensions, device)
self.net = IENetwork(model=model_xml, weights=model_bin)
self.exec_net = self.plugin.load_network(self.net, device)
# iter_i = iter(self.net.inputs)
# print(next(iter_i))
# print(next(iter_i))
self.input_name = next(iter(self.net.inputs))
self.output_name = next(iter(self.net.outputs))
supported_layers = self.plugin.query_network(network=self.net,
device_name="CPU")
unsupported_layers = []
for l in self.net.layers.keys():
if l not in supported_layers:
unsupported_layers.append(l)
if len(unsupported_layers) != 0:
log.error(
"Unsupported layers found: {}".format(unsupported_layers))
log.error("Check your extensions")
exit(1)
end = time.time()
# print('Face detection model load time',start-end)
return
def preprocess(self, frame):
image = np.copy(frame)
shape = self.net.inputs[self.input_name].shape
image = cv2.resize(image, (shape[3], shape[2]))
image = image.transpose((2, 0, 1))
image = image.reshape(1, *image.shape)
return image
def predict(self, image, thres):
frame = self.preprocess(image)
start = time.time()
self.request = self.exec_net.start_async(
request_id=0, inputs={self.input_name: frame})
status = self.request.wait()
if (status == 0):
end = time.time()
# print('Face detection model inference time',start-end)
crop_face, face_count, points = self.preprocess_output(
image, thres)
return crop_face, face_count, points
def check_model(self):
raise NotImplementedError
def preprocess_output(self, frame, thres):
outputs = self.request.outputs[self.output_name]
h, w, c = frame.shape
faces = 0
crop_face = 0
points = []
for box in outputs[0][0]:
conf = box[2]
if conf >= thres and box[1] == 1:
if (faces > 1):
log.error("found more than one face this"
"may affect performance of the model")
return crop_face, faces
xmin = int(box[3] * w)
ymin = int(box[4] * h)
xmax = int(box[5] * w)
ymax = int(box[6] * h)
faces += 1
points.append(xmin)
points.append(ymin)
crop_face = frame[ymin:ymax, xmin:xmax]
self.draw_points(frame, (xmin, ymin), (xmax, ymax))
return crop_face, faces, points
def draw_points(self, frame, f, s):
cv2.rectangle(frame, f, s, (130, 20, 25), 4)
class Gaze_Estimation:
'''
Class for the Gaze Estimation Model.
'''
def __init__(self, model_name):
Model.__init__(self, model_name) # initialize the base class
# then create/override the variable
#self.model_name = "faceDetectionModel"
self.model_name = model_name
self.logger = logging.getLogger(__name__)
try:
self.model = IENetwork(self.model_structure, self.model_weights)
except Exception:
self.logger.exception("The Network could not be initialized. Check that you have the correct model path")
self.load_model()
Model.check_model(self, model_name)
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.core=IECore()
### TODO: Check for supported layers ###
supported_layers = self.core.query_network(network=self.model, device_name=self.device)
unsupported_layers = [l for l in self.model.layers.keys() if l not in supported_layers]
if len(unsupported_layers)!=0:
### TODO: Add any necessary extensions ###
if self.cpu_extension and "CPU" in self.device:
self.core.add_extension(self.cpu_extension, self.device)
else:
self.logger.debug("Add CPU extension and device type or run layer with original framework")
exit(1)
self.net=self.core.load_network(network=self.model,device_name=self.device,num_requests=1)
self.input_name=[i for i in self.model.inputs.keys()]
self.input_shape=self.model.inputs[self.input_name[1]].shape
self.output_name=next(iter(self.model.outputs))
self.output_shape=self.model.outputs[self.output_name].shape
def predict(self, leftEye, rightEye, headPose):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
self.logger.info("Commence prepocessing of inputs and start inferencing")
left_eye_image = self.preprocess_input(leftEye)
right_eye_image = self.preprocess_input(rightEye)
# start asynchronous inference for specified request
self.logger.info("Commence sync request")
self.net.infer({'head_pose_angles':headPose, 'left_eye_image': left_eye_image, 'right_eye_image': right_eye_image})
# wait for the result
if self.net.requests[0].wait(-1) == 0:
outputs=self.net.requests[0].outputs[self.output_name]
return self.preprocess_output(outputs)
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
image_size=(self.input_shape[3], self.input_shape[2])
image = cv2.resize(image,(image_size))
image = image.transpose((2,0,1))
image = image.reshape(1,*image.shape)
return image
#raise NotImplementedError
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
x = outputs[0][0]
y = outputs[0][1]
return x,y
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self.net = None
self.plugin = None
self.input_blob = None
self.out_blob = None
self.net_plugin = None
self.infer_request_handle = None
def load_model(self,
model,
device,
input_size,
output_size,
num_requests,
cpu_extension=None,
plugin=None):
"""
Loads a network and an image to the Inference Engine plugin.
:param model: .xml file of pre trained model
:param cpu_extension: extension for the CPU device
:param device: Target device
:param input_size: Number of input layers
:param output_size: Number of output layers
:param num_requests: Index of Infer request value. Limited to device capabilities.
:param plugin: Plugin for specified device
:return: Shape of input layer
"""
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Plugin initialization for specified device
# and load extensions library if specified
if not plugin:
log.info("Initializing plugin for {} device...".format(device))
self.plugin = IECore()
else:
self.plugin = plugin
if cpu_extension and 'CPU' in device:
self.plugin.add_extension(cpu_extension, "CPU")
# Read IR
log.info("Reading IR...")
self.net = IENetwork(model=model_xml, weights=model_bin)
# print("Network Layers : ")
# print(self.net.layers)
log.info("Loading IR to the plugin...")
if device == "CPU":
supported_layers = self.plugin.query_network(self.net, "CPU")
not_supported_layers = \
[l for l in self.net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by "
"the plugin for specified device {}:\n {}".format(
device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path"
" in command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
if num_requests == 0:
# Loads network read from IR to theplugin
self.net_plugin = self.plugin.load_network(network=self.net,
device_name=device)
else:
self.net_plugin = self.plugin.load_network(
network=self.net,
num_requests=num_requests,
device_name=device)
# log.error("num_requests != 0")
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
# assert len(self.net.inputs.keys()) == input_size, \
# "Supports only {} input topologies".format(len(self.net.inputs))
# assert len(self.net.outputs) == output_size, \
# "Supports only {} output topologies".format(len(self.net.outputs))
return self.plugin, self.get_input_shape()
def get_input_shape(self):
"""
Gives the shape of the input layer of the network.
:return: None
"""
return self.net.inputs[self.input_blob].shape
def performance_counter(self, request_id):
"""
Queries performance measures per layer to get feedback of what is the
most time consuming layer.
:param request_id: Index of Infer request value. Limited to device capabilities
:return: Performance of the layer
"""
perf_count = self.net_plugin.requests[request_id].get_perf_counts()
return perf_count
def exec_net(self, request_id, frame):
"""
Starts asynchronous inference for specified request.
:param request_id: Index of Infer request value. Limited to device capabilities.
:param frame: Input image
:return: Instance of Executable Network class
"""
self.infer_request_handle = self.net_plugin.start_async(
request_id=request_id, inputs={self.input_blob: frame})
return self.net_plugin
def wait(self, request_id):
"""
Waits for the result to become available.
:param request_id: Index of Infer request value. Limited to device capabilities.
:return: Timeout value
"""
wait_process = self.net_plugin.requests[request_id].wait(-1)
return wait_process
def get_output(self, request_id, output=None):
"""
Gives a list of results for the output layer of the network.
:param request_id: Index of Infer request value. Limited to device capabilities.
:param output: Name of the output layer
:return: Results for the specified request
"""
if output:
res = self.infer_request_handle.outputs[output]
else:
res = self.net_plugin.requests[request_id].outputs[self.out_blob]
# print("==" * 30)
# print(res[0][0])
# print("==" * 30)
return res
def clean(self):
"""
Deletes all the instances
:return: None
"""
del self.net_plugin
del self.plugin
del self.net
class GazeEstimation:
'''
Class for the Gaze Estimation Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
instantiating the necessary variables.
'''
self.model_weights = model_name + '.bin'
self.model_structure = model_name + '.xml'
self.device = device
self.extensions = extensions
self.exec_net = None
self.input_name = None
self.input_shape = None
self.output_name = None
self.output_shape = None
def load_model(self):
'''
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.core = IECore()
self.model = self.core.read_network(self.model_structure,
self.model_weights)
supported_layers = self.core.query_network(network=self.model,
device_name=self.device)
unsupported_layers = [
l for l in self.model.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0 and self.device == 'CPU':
log.error(
"Unsupported layers found: {}".format(unsupported_layers))
log.error("Check whether extensions are available to add")
self.core.add_extension(self.extension, self.device)
supported_layers = self.core.query_network(network=self.model,
device_name=self.device)
unsupported_layers = [
l for l in self.model.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error("ERROR: Unsupported layer issue not yet resolved")
exit(1)
self.exec_net = self.core.load_network(network=self.model,
device_name=self.device,
num_requests=1)
self.input_name = next(iter(self.model.inputs))
self.input_shape = self.model.inputs[self.input_name].shape
self.output_name = next(iter(self.model.outputs))
self.output_shape = self.model.outputs[self.output_name].shape
def predict(self, left_eye_image, right_eye_image, head_pose_output):
'''
This method is meant for running predictions on the input image.
'''
leye_image, reye_image = self.preprocess_input(left_eye_image.copy(),
right_eye_image.copy())
result = self.exec_net.infer({
'left_eye_image': leye_image,
'right_eye_image': reye_image,
'head_pose_angles': head_pose_output
})
mouse_coord, gaze_vector = self.preprocess_output(
result, head_pose_output)
return mouse_coord, gaze_vector
def check_model(self):
pass
def preprocess_input(self, left_eye_image, right_eye_image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
leye_p_image = cv2.resize(left_eye_image, (60, 60))
leye_p_image = leye_p_image.transpose((2, 0, 1))
leye_p_image = leye_p_image.reshape(1, *leye_p_image.shape)
reye_p_image = cv2.resize(right_eye_image, (60, 60))
reye_p_image = reye_p_image.transpose((2, 0, 1))
reye_p_image = reye_p_image.reshape(1, *reye_p_image.shape)
return leye_p_image, reye_p_image
def preprocess_output(self, outputs, head_pose_output):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
'''
gaze_output = outputs[self.output_name][0]
roll_value = head_pose_output[2]
cos_value = math.cos(roll_value * math.pi / 180.0)
sin_value = math.sin(roll_value * math.pi / 180.0)
x_gaze = gaze_output[0] * cos_value + gaze_output[1] * sin_value
y_gaze = gaze_output[1] * cos_value - gaze_output[0] * sin_value
return (x_gaze, y_gaze), gaze_output
def main():
total_timer = Timer(name='total')
total_timer.tic()
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1
), "Sample supports topologies only with 1 input"
input_name = next(iter(net.inputs.keys()))
input_info = net.inputs[input_name]
input_info.precision = 'FP32'
log.info('input shape: {}'.format(input_info.shape))
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
assert (len(net.outputs.keys()) == 2
), "Sample supports topologies only with 2 output"
loc_out_name = "797"
class_out_name = "741"
assert (loc_out_name in net.outputs.keys()) and (class_out_name
in net.outputs.keys())
loc_out_info = net.outputs[loc_out_name]
class_out_info = net.outputs[class_out_name]
loc_out_info.precision = "FP32"
class_out_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------------
log.info("Loading model to the device")
# cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}
ie.set_config({'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'},
args.device)
ie.set_config({'CPU_BIND_THREAD': 'YES'}, args.device)
exec_net = ie.load_network(network=net,
device_name=args.device,
num_requests=0)
infer_requests = exec_net.requests
request_queue = InferRequestsQueue(infer_requests)
log.info('nreqs: {}, nstream:{}'.format(
len(infer_requests),
ie.get_config(args.device, 'CPU_THROUGHPUT_STREAMS')))
# --------------------------- 3. Read and preprocess input --------------------------------------------
# -----------------------------------------------------------------------------------------------------
if not os.path.exists(args.result_dir):
os.makedirs(args.result_dir)
if args.voc_res_file and os.path.exists(args.voc_res_file):
os.remove(args.voc_res_file)
load_data_timer = Timer(name='load_data')
post_process_timer = Timer(name='post_process')
adapter = RetinaNetAdapter(input_shape=args.patch_size)
# --------------------------- Performing inference ----------------------------------------------------
result_all_images = defaultdict(list)
data_loader = DataLoader(args.image_dir, args.strides, args.patch_size)
while True:
load_data_timer.tic()
input_data = data_loader.next()
load_data_timer.toc()
if input_data == None:
break
infer_request = request_queue.get_idle_request()
if not infer_request:
raise Exception('No idle Infer Requests!')
if infer_request.cur_meta == None:
infer_request.start_async(input_name, input_data)
continue
# get result
post_process_timer.tic()
image_name = infer_request.cur_meta['image_name']
x = infer_request.cur_meta['x']
y = infer_request.cur_meta['y']
loc_out = infer_request.request.outputs[loc_out_name][0]
class_out = infer_request.request.outputs[class_out_name][0]
## start infer
infer_request.start_async(input_name, input_data)
## post-process
result = adapter.process(loc_out, class_out)
result, _ = nms(result, thresh=0.5, keep_top_k=100)
result[:, 0] += x
result[:, 1] += y
result[:, 2] += x
result[:, 3] += y
result_all_images[image_name].append(result)
post_process_timer.toc()
# wait the latest inference executions
request_queue.wait_all()
post_process_timer.tic()
for infer_request in request_queue.requests:
# get result
image_name = infer_request.cur_meta['image_name']
x = infer_request.cur_meta['x']
y = infer_request.cur_meta['y']
loc_out = infer_request.request.outputs[loc_out_name][0]
class_out = infer_request.request.outputs[class_out_name][0]
## post-process
result = adapter.process(loc_out, class_out)
result, _ = nms(result, thresh=0.5, keep_top_k=100)
result[:, 0] += x
result[:, 1] += y
result[:, 2] += x
result[:, 3] += y
result_all_images[image_name].append(result)
post_process_timer.toc()
post_process_timer.tic()
## process total image result
for image_name, result_per_image in result_all_images.items():
result_per_image = np.concatenate(result_per_image, axis=0)
nms_result, _ = nms(result_per_image, thresh=0.5)
voc_format = '{} {:.4f} {} {} {} {}'
pos_all = []
voc_all = []
for i in range(nms_result.shape[0]):
x = int(nms_result[i, 0])
y = int(nms_result[i, 1])
w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1)
h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1)
p = float(nms_result[i, 4])
pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p}
pos_all.append(pos)
if args.voc_res_file:
xmin = x
ymin = y
xmax = int(nms_result[i, 2])
ymax = int(nms_result[i, 3])
voc_str = voc_format.format(
os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax)
voc_all.append(voc_str)
file_name = os.path.splitext(image_name)[0] + '.json'
with open(os.path.join(args.result_dir, file_name), 'w') as f:
json.dump(pos_all, f)
if args.voc_res_file:
with open(args.voc_res_file, 'a') as f:
for voc_str in voc_all:
f.write(voc_str + '\n')
post_process_timer.toc()
total_timer.toc()
# -----------------------------------------------------------------------------------------------------
all_timers = []
# all_timers.extend([create_anchor_timer,
# read_img_timer,
# preprocess_timer,
# infer_timer,
# adapter_timer,
# patch_img_nms_timer,
# whole_img_nms_timer,
# add_offset_timer,
# write_result_timer,
# total_timer])
all_timers.extend([load_data_timer, post_process_timer, total_timer])
for timer in all_timers:
log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format(
timer.name, timer.avg * 1000, timer.total))
log.info('infer: {:2f}s'.format(request_queue.get_duration_in_seconds()))
log.info("Execution successful\n")
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device="CPU", cpu_extension=None):
'''
Load the model given IR files.
Defaults to CPU as device for use in the workspace.
Synchronous requests made within.
'''
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Initialize the plugin
self.plugin = IECore()
# Add a CPU extension, if applicable
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
network = IENetwork(model=model_xml, weights=model_bin)
# Get the supported layers of the network
supported_layers = self.plugin.query_network(network=network, device_name="CPU")
# Check for unsupported layers
unsupported_layers = [l for l in network.layers.keys() if l not in supported_layers]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(unsupported_layers))
print("Check whether extensions are available to add to IECore.")
exit(1)
# Load the IENetwork into the plugin
self.exec_network = self.plugin.load_network(network, device)
# Get the input layer
self.input_blob = next(iter(network.inputs))
self.output_blob = next(iter(network.outputs))
# Return the input shape (to determine preprocessing)
return network.inputs[self.input_blob].shape
def get_input_shape(self):
return self.network.inputs[self.input_blob].shape
def async_inference(self, image):
self.exec_network.start_async(request_id = 0,inputs = {self.input_blob:image})
return
def wait(self):
status = self.exec_network.requests[0].wait(-1)
return status
def extract_output(self): #get_output
return self.exec_network.requests[0].outputs[self.output_blob]
class HeadPoseEstimationModel:
'''
Class for defining the HeadPoseEstimation Model Attributes.
'''
def __init__(self,
model_name,
threshold,
device='CPU',
extensions=None,
async_mode=True,
plugin=None):
'''
TODO: Use this to set your instance variables.
'''
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.out_shape = None
self.exec_network = None
self.threshold = threshold
self.device = device
self.async_mode = async_mode
self.infer_request = None
self.net_plugin = None
self.net = None
self.extensions = extensions
self.model_xml = model_name
def load_model(self, model_xml, cpu_extension=None):
'''
TODO: load models
'''
self.model_xml = model_name
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.device = device
self.extensions = extensions
# Initializing the plugins
self.plugin = IECore()
# Add a CPU extension and any neccessary extension
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Reading the Intermediate Representation (IR) model as a IENetwork
# IENetwork deprecated in 2020 version
self.network = self.plugin.read_network(model=model_xml,
weights=model_bin)
self.check_plugin(self.plugin)
## Check for supported layers
supported_layers = self.plugin.query_network(network=self.network,
device_name=device)
## check for unsupported layers
unsupported_layers = [
l for l in self.network.layers.keys()
if l not in self.plugin.get_supported_layers(self.network)
]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(unsupported_layers))
print("Please check for supported extensions.")
exit(1)
# Loading the IENetwork into the plugin
self.exec_network = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
self.out_shape = self.network.outputs[self.output_blob].shape
logging.info("Model Head Pose Detection output shape printed : ",
self.out_shape)
return
def predict(self, image, width, height, threshold):
'''
TODO: You will need to complete this method.
To run predictions on the input image.
'''
# [1,3,60,60]
tally = 0
valuess = None
width = image.shape[1]
height = image.shape[0]
img_frame = self.preprocess_input(image)
if self.async_mode:
self.exec_network.requests[0].async_infer(
inputs={self.input_blob: img_frame})
else:
self.exec_network.requests[0].infer(
inputs={self.input_blob: img_frame})
if self.exec_network.requests[0].wait(-1) == 0:
outputs = self.exec_network.requests[0].outputs
person_in_frame, target_gaze = self.preprocess_output(
image, outputs)
return person_in_frame, target_gaze
def get_input_shape(self):
### Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def preprocess_input(self, image):
'''
TODO: You will need to complete this method.
preprocessing the input shape
'''
# [1,3,60,60]
(n, c, h, w) = self.network.inputs[self.input_blob].shape
img_frame = cv2.resize(image, (w, h))
img_frame = img_frame.transpose((2, 0, 1))
img_frame = img_frame.reshape((n, c, h, w))
return img_frame
def preprocess_output(self, image, outputs, width, height):
'''
TODO: You will need to complete this method.
Output layer names in Inference Engine format:
name: "angle_y_fc", shape: [1, 1] - Estimated yaw (in degrees).
name: "angle_p_fc", shape: [1, 1] - Estimated pitch (in degrees).
name: "angle_r_fc", shape: [1, 1] - Estimated roll (in degrees).
Each output contains one float value (yaw, pitсh, roll).
'''
# To Parse head pose detection results
# ref: source code: https://knowledge.udacity.com/questions/242566
pitch = outputs["angle_p_fc"][0]
yaw = outputs["angle_y_fc"][0]
roll = outputs["angle_r_fc"][0]
# Draw output
if ((yaw > -22.5) & (yaw < 22.5) & (pitch > -22.5) & (pitch < 22.5)):
return True, [[yaw, pitch, roll]]
else:
return False, [[0, 0, 0]]
# code source: https://knowledge.udacity.com/questions/171017
def draw_axes(self, img_frame, center_of_face, yaw, pitch, roll):
focal_length = 950.0
scale = 50
yaw *= np.pi / 180.0
pitch *= np.pi / 180.0
roll *= np.pi / 180.0
cx = int(center_of_face[0])
cy = int(center_of_face[1])
Rx = np.array([[1, 0, 0], [0, math.cos(pitch), -math.sin(pitch)],
[0, math.sin(pitch),
math.cos(pitch)]])
Ry = np.array([[math.cos(yaw), 0, -math.sin(yaw)], [0, 1, 0],
[math.sin(yaw), 0, math.cos(yaw)]])
Rz = np.array([[math.cos(roll), -math.sin(roll), 0],
[math.sin(roll), math.cos(roll), 0], [0, 0, 1]])
# R = np.dot(Rz, Ry, Rx)
# ref: https://www.learnopencv.com/rotation-matrix-to-euler-angles/
# R = np.dot(Rz, np.dot(Ry, Rx))
R = Rz @ Ry @ Rx
# print(R)
camera_matrix = self.build_camera_matrix(center_of_face, focal_length)
xaxis = np.array(([1 * scale, 0, 0]), dtype='float32').reshape(3, 1)
yaxis = np.array(([0, -1 * scale, 0]), dtype='float32').reshape(3, 1)
zaxis = np.array(([0, 0, -1 * scale]), dtype='float32').reshape(3, 1)
zaxis1 = np.array(([0, 0, 1 * scale]), dtype='float32').reshape(3, 1)
o = np.array(([0, 0, 0]), dtype='float32').reshape(3, 1)
o[2] = camera_matrix[0][0]
xaxis = np.dot(R, xaxis) + o
yaxis = np.dot(R, yaxis) + o
zaxis = np.dot(R, zaxis) + o
zaxis1 = np.dot(R, zaxis1) + o
xp2 = (xaxis[0] / xaxis[2] * camera_matrix[0][0]) + cx
yp2 = (xaxis[1] / xaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(img_frame, (cx, cy), p2, (0, 0, 255), 2)
xp2 = (yaxis[0] / yaxis[2] * camera_matrix[0][0]) + cx
yp2 = (yaxis[1] / yaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(img_frame, (cx, cy), p2, (0, 255, 0), 2)
xp1 = (zaxis1[0] / zaxis1[2] * camera_matrix[0][0]) + cx
yp1 = (zaxis1[1] / zaxis1[2] * camera_matrix[1][1]) + cy
p1 = (int(xp1), int(yp1))
xp2 = (zaxis[0] / zaxis[2] * camera_matrix[0][0]) + cx
yp2 = (zaxis[1] / zaxis[2] * camera_matrix[1][1]) + cy
p2 = (int(xp2), int(yp2))
cv2.line(img_frame, p1, p2, (255, 0, 0), 2)
cv2.circle(img_frame, p2, 3, (255, 0, 0), 2)
return img_frame
# code source: https://knowledge.udacity.com/questions/171017
def build_camera_matrix(self, center_of_face, focal_length):
cx = int(center_of_face[0])
cy = int(center_of_face[1])
camera_matrix = np.zeros((3, 3), dtype='float32')
camera_matrix[0][0] = focal_length
camera_matrix[0][2] = cx
camera_matrix[1][1] = focal_length
camera_matrix[1][2] = cy
camera_matrix[2][2] = 1
return camera_matrix
def clean(self):
"""
deletes all the open instances
:return: None
"""
del self.plugin
del self.network
del self.exec_network
del self.net
del self.device
class HeadPoseModel:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU',threshold = 0.6 , extensions=None):
self.model_name = model_name
self.device = device
self.extensions = extensions
self.threshold = threshold
self.model_structure = self.model_name
self.model_weights = os.path.splitext(model_name)[0] + ".bin"
self.core = IECore()
self.network = self.core.read_network(model=self.model_structure, weights=self.model_weights)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_names = [o for o in self.network.outputs.keys()]
self.exec_net = None
def load_model(self):
supported_layers = self.core.query_network(network=self.network, device_name=self.device)
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(unsupported_layers)!=0 and self.device=='CPU':
print("unsupported layers found:{}".format(unsupported_layers))
if self.extensions:
self.core.add_extension(self.extensions, self.device)
supported_layers = self.core.query_network(network = self.network, device_name=self.device)
unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers]
if len(unsupported_layers)!=0:
print("Error:Unsupported layers still found:{}".format(unsupported_layers))
exit(1)
else:
print("Error: Exteion layers not found!")
exit(1)
self.exec_net = self.core.load_network(network=self.network, device_name=self.device,num_requests=1)
def predict(self, image):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
image_ = image
img_processed = self.preprocess_input(image_)
input_dict = {self.input_name:img_processed}
outputs = self.exec_net.infer(input_dict)
output = self.preprocess_output(outputs)
return output
def check_model(self):
pass
def preprocess_input(self, image):
image_resized = cv2.resize(image, (self.input_shape[3], self.input_shape[2]))
img_processed = np.transpose(np.expand_dims(image_resized,axis=0), (0,3,1,2))
return img_processed
def preprocess_output(self, outputs):
outputs_ = []
outputs_.append(outputs['angle_p_fc'].tolist()[0][0])
outputs_.append(outputs['angle_r_fc'].tolist()[0][0])
outputs_.append(outputs['angle_y_fc'].tolist()[0][0])
return outputs_
def main():
global args
args = build_argparser().parse_args()
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# ------------- 1. Plugin initialization for specified device and load extensions library if specified -------------
log.info("Creating Inference Engine...")
ie = IECore()
if args.cpu_extension and 'CPU' in args.device:
ie.add_extension(args.cpu_extension, "CPU")
# -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) --------------------
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
print("model loaded")
# ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
format(args.device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(net.inputs.keys()) == 1, "Sample supports only YOLO V3 based single input topologies"
# ---------------------------------------------- 4. Preparing inputs -----------------------------------------------
log.info("Preparing inputs")
input_blob = next(iter(net.inputs))
# Defaulf batch_size is 1
net.batch_size = 1
# Read and pre-process input images
n, c, h, w = net.inputs[input_blob].shape
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
is_async_mode = True
''' CHG '''
wait_rate = rospy.Rate(1000)
global rgb_img_update, depth_img_update, rgb_image, depth_img, bridge, motor_yaw_queue
pub_total=rospy.Publisher("/yolo_ros_real_pose/detected_objects",ObjectsRealPose,queue_size = 1)
corresponding_img_pub=rospy.Publisher("/yolo_ros_real_pose/img_for_detected_objects",Image,queue_size = 1)
pub_time=rospy.Publisher("/yolo_ros_real_pose/detection_time", Float64, queue_size = 1)
time_this = Float64()
info_this = InfoGather()
info_next = InfoGather()
''' END '''
if args.input == "ROS":
while not rospy.is_shutdown():
if rgb_img_update == 1 and depth_img_update==1 and len(rgb_image)!=0 and len(depth_img)!=0 and (not motor_yaw_queue.empty() or not motor_yaw_syncronize):
frame = rgb_image.copy()
depth_this = depth_img.copy()
info_this.uav_pose = uav_pose.pose
info_this.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough")
info_this.img_to_pub.header = time_stamp_header
if motor_yaw_syncronize:
info_this.motor_yaw = motor_yaw_queue.get()
else:
info_this.motor_yaw = motor_yaw_corrected
print("Time difference="+str(uav_pose.header.stamp.to_sec() - time_stamp_header.stamp.to_sec()))
rgb_img_update = 0
depth_img_update = 0
wait_key_code = 1
break
else:
input_stream = 0 if args.input == "cam" else args.input
cap = cv2.VideoCapture(input_stream)
number_input_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
number_input_frames = 1 if number_input_frames != -1 and number_input_frames < 0 else number_input_frames
wait_key_code = 1
# Number of frames in picture is 1 and this will be read in cycle. Sync mode is default value for this case
if number_input_frames != 1:
ret, frame = cap.read()
else:
is_async_mode = False
wait_key_code = 0
# ----------------------------------------- 5. Loading model to the plugin -----------------------------------------
log.info("Loading model to the plugin")
exec_net = ie.load_network(network=net, num_requests=2, device_name=args.device)
cur_request_id = 0
next_request_id = 1
render_time = 0
parsing_time = 0
# ----------------------------------------------- 6. Doing inference -----------------------------------------------
log.info("Starting inference...")
print("To close the application, press 'CTRL+C' here or switch to the output window and press ESC key")
print("To switch between sync/async modes, press TAB key in the output window")
while not rospy.is_shutdown():
# Here is the first asynchronous point: in the Async mode, we capture frame to populate the NEXT infer request
# in the regular mode, we capture frame to the CURRENT infer request
try:
detection_start_time = time()
if args.input == "ROS":
while not rospy.is_shutdown():
if rgb_img_update == 1 and depth_img_update==1 and (not motor_yaw_queue.empty() or not motor_yaw_syncronize):
if len(rgb_image)!=0 and len(depth_img)!=0:
if is_async_mode:
next_frame = rgb_image.copy()
depth_next = depth_img.copy()
info_next.uav_pose = uav_pose.pose
info_next.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough")
info_next.img_to_pub.header = time_stamp_header
if motor_yaw_syncronize:
info_next.motor_yaw = motor_yaw_queue.get()
else:
info_next.motor_yaw = motor_yaw_corrected
else:
frame = rgb_image.copy()
depth_this = depth_img.copy()
info_this.uav_pose = uav_pose.pose
info_this.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough")
info_this.img_to_pub.header = time_stamp_header
if motor_yaw_syncronize:
info_this.motor_yaw = motor_yaw_queue.get()
else:
info_this.motor_yaw = motor_yaw_corrected
print("Time difference="+str(uav_pose.header.stamp.to_sec() - time_stamp_header.stamp.to_sec()))
rgb_img_update = 0
depth_img_update = 0
break
wait_rate.sleep()
else:
if is_async_mode:
ret, next_frame = cap.read()
else:
ret, frame = cap.read()
if not ret:
break
if is_async_mode:
request_id = next_request_id
in_frame = cv2.resize(next_frame, (w, h))
else:
request_id = cur_request_id
in_frame = cv2.resize(frame, (w, h))
# resize input_frame to network size
in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW
in_frame = in_frame.reshape((n, c, h, w))
# Start inference
start_time = time()
exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame})
det_time = time() - start_time
# Collecting object detection results
objects = list()
if exec_net.requests[cur_request_id].wait(-1) == 0:
output = exec_net.requests[cur_request_id].outputs
start_time = time()
for layer_name, out_blob in output.items():
out_blob = out_blob.reshape(net.layers[net.layers[layer_name].parents[0]].shape)
layer_params = YoloParams(net.layers[layer_name].params, out_blob.shape[2])
log.info("Layer {} parameters: ".format(layer_name))
layer_params.log_params()
objects += parse_yolo_region(out_blob, in_frame.shape[2:],
frame.shape[:-1], layer_params,
args.prob_threshold)
parsing_time = time() - start_time
# Filtering overlapping boxes with respect to the --iou_threshold CLI parameter
objects = sorted(objects, key=lambda obj : obj['confidence'], reverse=True)
for i in range(len(objects)):
if objects[i]['confidence'] == 0:
continue
for j in range(i + 1, len(objects)):
if intersection_over_union(objects[i], objects[j]) > args.iou_threshold:
objects[j]['confidence'] = 0
# Drawing objects with respect to the --prob_threshold CLI parameter
objects = [obj for obj in objects if obj['confidence'] >= args.prob_threshold]
if len(objects) and args.raw_output_message:
log.info("\nDetected boxes for batch {}:".format(1))
log.info(" Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR ")
origin_im_size = frame.shape[:-1]
current_total_dect = ObjectsRealPose()
for obj in objects:
# Validation bbox of detected object
if obj['xmax'] > origin_im_size[1] or obj['ymax'] > origin_im_size[0] or obj['xmin'] < 0 or obj['ymin'] < 0:
continue
color = (int(min(obj['class_id'] * 12.5, 255)),
min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255))
det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \
str(obj['class_id'])
if args.raw_output_message:
log.info(
"{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ".format(det_label, obj['confidence'], obj['xmin'],
obj['ymin'], obj['xmax'], obj['ymax'],
color))
cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2)
cv2.putText(frame,
"#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %',
(obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)
'''CHG'''
x_pos=0
y_pos=0
z_pos=0
info=RealPose()
info.label = det_label
info.confidence = obj['confidence']
info.pix_lt_x = obj['xmin']
info.pix_lt_y = obj['ymin']
info.pix_rb_x = obj['xmax']
info.pix_rb_y = obj['ymax']
info.head_yaw = info_this.motor_yaw
info.local_pose = info_this.uav_pose
### Now calculate the real position
if len(depth_this)!=0 and (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])>=0:
### Calculate position here by depth image and camera parameters
depth_box_width=obj['xmax']-obj['xmin']
depth_box_height=obj['ymax']-obj['ymin']
delta_rate=0.1
x_box_min=int(obj['xmin'] + depth_box_width*delta_rate)
y_box_min=int(obj['ymin'] + depth_box_height*delta_rate)
x_box_max=int(obj['xmax'] - depth_box_width*delta_rate)
y_box_max=int(obj['ymax'] - depth_box_height*delta_rate)
after_width=(depth_box_width*(1-2*delta_rate))
after_height=(depth_box_height*(1-2*delta_rate))
''' '''
rect = depth_this[y_box_min:y_box_max, x_box_min:x_box_max] # Simulation
# rect = depth_this[y_box_min:y_box_max, x_box_min:x_box_max] * 0.001
rect[np.where(rect == 0)] = 99
rect[np.where(rect != rect)] = 99
z_pos = rect.min()
x_pos = (0.5 * (obj['xmax'] + obj['xmin']) - cx) * z_pos / fx
y_pos = (0.5 * (obj['ymax'] + obj['ymin']) - cy) * z_pos / fy
info.x = x_pos
info.y = y_pos
info.z = z_pos
current_total_dect.result.append(info)
if len(current_total_dect.result)>0:
current_total_dect.header = info_this.img_to_pub.header
pub_total.publish(current_total_dect)
corresponding_img_pub.publish(info_this.img_to_pub)
time_this.data = time() - detection_start_time
pub_time.publish(time_this)
'''END'''
# Draw performance stats over frame
inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
"Inference time: {:.3f} ms".format(det_time * 1e3)
render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1e3)
async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
"Async mode is off. Processing request {}".format(cur_request_id)
parsing_message = "YOLO parsing time is {:.3f}".format(parsing_time * 1e3)
cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5,
(10, 10, 200), 1)
cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
start_time = time()
cv2.imshow("DetectionResults", frame)
# cv2.imshow("depth_this", depth_this)
render_time = time() - start_time
''' For next '''
if is_async_mode:
cur_request_id, next_request_id = next_request_id, cur_request_id
frame = next_frame
depth_this = depth_next
info_this.uav_pose = info_next.uav_pose
info_this.motor_yaw = info_next.motor_yaw
info_this.img_to_pub = info_next.img_to_pub
info_this.img_to_pub.header = info_next.img_to_pub.header
key = cv2.waitKey(wait_key_code)
# ESC key
if key == 27:
break
# Tab key
if key == 9:
exec_net.requests[cur_request_id].wait()
is_async_mode = not is_async_mode
log.info("Switched to {} mode".format("async" if is_async_mode else "sync"))
except:
print("An error occuered!!!!!!!")
cv2.destroyAllWindows()
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = IECore()
self.exec_network = None
self.net = None
self.input_layer = None
def load_model(self, xml_path, num_requests, device_name, cpu_extension):
### TODO: Load the model ###
self.net = IENetwork(model=xml_path,
weights=xml_path.split('.')[0] + '.bin')
### TODO: Check for supported layers ###
supported_layers = self.plugin.query_network(network=self.net,
device_name=device_name)
unsupported_layers = [
l for l in self.net.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
print("unsupported layers found:{}".format(unsupported_layers))
if not cpu_extension == None:
print("Adding cpu_extension")
self.plugin.add_extension(cpu_extension, device_name)
supported_layers = self.plugin.query_network(
network=self.net, device_name=device_name)
unsupported_layers = [
l for l in self.net.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
print(
"After adding the extension still unsupported layers found"
)
exit(1)
print("After adding the extension the issue is resolved")
else:
print("Give the path of cpu extension")
exit(1)
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
self.exec_network = self.plugin.load_network(network=self.net,
num_requests=num_requests,
device_name=device_name)
### Note: You may need to update the function parameters. ###
self.input_layer = next(iter(self.net.inputs))
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.net.inputs[self.input_layer].shape
def exec_net(self, frame, req_id):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.exec_network.start_async(request_id=req_id,
inputs={self.input_layer: frame})
return
def wait(self, req_id):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.exec_network.requests[req_id].wait(-1)
def get_output(self, req_id):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
out = self.exec_network.requests[req_id].outputs
return [out[i] for i in out.keys()]
class PersonDetect:
'''
Class for the Person Detection Model.
'''
def __init__(self, model_name, device, threshold=0.60):
self.model_weights = model_name + '.bin'
self.model_structure = model_name + '.xml'
self.device = device
self.threshold = threshold
try:
self.model = IENetwork(self.model_structure, self.model_weights)
except Exception as e:
raise ValueError(
"Could not Initialise the network. Have you enterred the correct model path?"
)
self.input_name = next(iter(self.model.inputs))
self.input_shape = self.model.inputs[self.input_name].shape
self.output_name = next(iter(self.model.outputs))
self.output_shape = self.model.outputs[self.output_name].shape
def load_model(self):
self.plugin = IECore()
self.exec_network = self.plugin.load_network(network=self.model,
device_name=self.device)
sl = self.plugin.query_network(network=self.model,
device_name=self.device)
ul = [l for l in self.model.layers.keys() if l not in sl]
if len(ul) != 0:
print("Unsupported layers found: {}".format(ul))
print("Check whether extensions are available to add to IECore.")
exit(1)
def predict(self, image, width, height):
p_image = self.preprocess_input(image)
input_dict = {self.input_name: p_image}
self.exec_network.infer(input_dict)
results = self.exec_network.requests[0].outputs[self.output_name]
coords, image = self.draw_outputs(results, image, width, height)
return coords, image
def draw_outputs(self, result, image, width, height):
coords = []
for box in result[0][0]:
a = []
conf = box[2]
#output_class = box[1]
if conf >= self.threshold:
xmin = int(box[3] * width)
ymin = int(box[4] * height)
xmax = int(box[5] * width)
ymax = int(box[6] * height)
a.append(xmin)
a.append(ymin)
a.append(xmax)
a.append(ymax)
cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 0, 255),
2)
coords.append(a)
return coords, image
def preprocess_input(self, image):
image = cv2.resize(image, (self.input_shape[3], self.input_shape[2]))
image = image.transpose((2, 0, 1))
image = image.reshape(1, *image.shape)
return image
class GazeEstimation:
"""
Class for the Face Detection Model.
"""
def __init__(self, model_name, device='CPU', extensions=None):
self.net = None
self.plugin = None
self.input_blob = None
self.out_blob = None
self.exec_net = None
self.model_name = model_name
self.extensions = extensions
self.device = device
def load_model(self):
"""
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
"""
# Fetch XML model
model_xml = self.model_name
model_bin = os.path.splitext(model_xml)[0] + ".bin"
self.plugin = IECore()
self.net = IENetwork(model=model_xml, weights=model_bin)
# Add CPU extension to self.plugin and check not supported layers
if "CPU" in self.device:
supported_layers = self.plugin.query_network(self.net, self.device)
not_supported_layers = [
layer for layer in self.net.layers.keys()
if layer not in supported_layers
]
if len(not_supported_layers) != 0 and self.device == 'CPU':
print(f"Not supported layers: {not_supported_layers}")
logging.error(f"Unsupported layers: {not_supported_layers}")
# Load model in network
start_time = time.time()
self.exec_net = self.plugin.load_network(network=self.net,
device_name=self.device,
num_requests=1)
end_time = time.time()
# Obtain blob info from network
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
print(f"Gaze Estimation Model Loading Time: {end_time - start_time}")
logging.info(
f"Gaze Estimation Model Loading Time: {end_time - start_time}")
def predict(self, left_eye, right_eye, head_pose):
"""
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
"""
processed_left_eye, processed_right_eye = self.preprocess_input(
left_eye, right_eye)
# infer image
outputs = self.exec_net.infer({
'left_eye_image': processed_left_eye,
'right_eye_image': processed_right_eye,
'head_pose_angles': head_pose
})
(mouse_x,
mouse_y), gaze_vector = self.preprocess_output(outputs, head_pose)
return (mouse_x, mouse_y), gaze_vector
def preprocess_input(self, left_eye, right_eye):
"""
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
"""
try:
left_eye = cv2.resize(left_eye, (60, 60))
right_eye = cv2.resize(right_eye, (60, 60))
preprocessed_left_image = left_eye.transpose((2, 0, 1))
preprocessed_right_image = right_eye.transpose((2, 0, 1))
preprocessed_left_image = preprocessed_left_image.reshape(
1, 3, 60, 60)
preprocessed_right_image = preprocessed_right_image.reshape(
1, 3, 60, 60)
except:
logging.warning(
f"eye image is in wrong shape Left Eye:{left_eye.shape} Right Eye :{right_eye.shape}"
)
return 0, 0
return preprocessed_left_image, preprocessed_right_image
def preprocess_output(self, outputs, head_pose):
"""
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
"""
gaze_vector = outputs[self.out_blob][0]
# HeadPoseEstimation model find angle_r_fc output
rollValue = head_pose[2]
cosValue = math.cos(rollValue * math.pi / 180.0)
sinValue = math.sin(rollValue * math.pi / 180.0)
mouse_x = gaze_vector[0] * cosValue + gaze_vector[1] * sinValue
mouse_y = -gaze_vector[0] * sinValue + gaze_vector[1] * cosValue
# gaze_vec = {'x': outputs['gaze_vector'][0][0].item(),
# 'y': outputs['gaze_vector'][0][1].item(),
# 'z': outputs['gaze_vector'][0][2].item()}
return (mouse_x, mouse_y), gaze_vector
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device="CPU", cpu_extension=None):
### TODO: Load the model ###
### TODO: Check for supported layers ###
### TODO: Add any necessary extensions ###
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Initialize the plugin
self.plugin = IECore()
# Add a CPU extension, if applicable
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
# Read the IR as a IENetwork
self.network = IENetwork(model=model_xml, weights=model_bin)
supported_layers = self.plugin.query_network(network=self.network,
device_name="CPU")
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(unsupported_layers))
print("Check whether extensions are available to add to IECore.")
exit(1)
#if num_of_req == 0:
#self.exec_network = self.plugin.load(network = self.network)
#else:
#self.exec.network = self.plugin.load(network = self.mnetwork,
#num_of_req = num_of_req)
# Load the IENetwork into the plugin
self.exec_network = self.plugin.load_network(self.network, device)
# Get the input layer
self.input_blob = next(iter(self.network.inputs))
self.output_blob = next(iter(self.network.outputs))
return
#def preprocessing(input_image, height, weight):
#image = cv2.resize(input_image, (height, width))
#image = image.transpose((2,0,1))
#image = image.reshape(1,3,height,width)
#return image
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.input_blob].shape
def exec_net(self, image, request_id=0):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.exec_network.start_async(request_id=request_id,
inputs={self.input_blob: image})
return
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.exec_network.requests[0].outputs[self.output_blob]
class FacialLandmarkDetection:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model_name = model_name
self.device = device
self.extensions = extensions
self.model_structure = self.model_name
self.model_weights = self.model_name.split(".")[0] + '.bin'
self.plugin = IECore()
## check if read model without problem
self.check_model(self.model_structure, self.model_weights)
self.exec_net = None
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_names = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_names].shape
## check supported layer and performence counts reference:
# https://gist.github.com/justinshenk/9917891c0433f33967f6e8cd8fcaa49a
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
supported_layers = self.plugin.query_network(network=self.network,
device_name=self.device)
layers_unsupported = [
ul for ul in self.network.layers.keys()
if ul not in supported_layers
]
if len(layers_unsupported) != 0 and self.device == 'CPU':
print("unsupported layers found: {}".format(layers_unsupported))
if self.extensions != None:
print("Adding cpu_extension now")
self.plugin.add_extension(self.extensions, self.device)
supported_layers = self.plugin.query_network(
network=self.network, device_name=self.device)
layers_unsupported = [
ul for ul in self.network.layers.keys()
if ul not in supported_layers
]
if len(layers_unsupported) != 0:
print(
"Please try again! unsupported layers found after adding the extensions. device {}:\n{}"
.format(self.device, ', '.join(layers_unsupported)))
print(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
exit(1)
print("Problem is resolved after adding the extension!")
else:
print("Please give the right path of cpu extension!")
exit(1)
self.exec_net = self.plugin.load_network(network=self.network,
device_name=self.device,
num_requests=1)
def predict(self, image, perf_flag):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image.
'''
processed_input = self.preprocess_input(image.copy())
outputs = self.exec_net.infer({self.input_name: processed_input})
if perf_flag:
self.performance()
coords = self.preprocess_output(outputs)
# print(image.shape)
h, w = image.shape[0], image.shape[1]
coords = coords * np.array([w, h, w, h])
# print(coords) # [ 39.50625494 130.00975445 146.11010522 126.54997483]
coords = coords.astype(
np.int32) #(lefteye_x, lefteye_y, righteye_x, righteye_y)
# print(coords) # [ 39 130 146 126]
## left eye moving range
leye_xmin, leye_ymin = coords[0] - 20, coords[1] - 20
leye_xmax, leye_ymax = coords[0] + 20, coords[1] + 20
## right eye moving range
reye_xmin, reye_ymin = coords[2] - 20, coords[3] - 20
reye_xmax, reye_ymax = coords[2] + 20, coords[3] + 20
## draw left and right eye
# cv2.rectangle(image,(leye_xmin,leye_ymin),(leye_xmax,leye_ymax),(0,255,0), 1)
# cv2.rectangle(image,(reye_xmin,reye_ymin),(reye_xmax,reye_ymax),(0,255,0), 1)
# cv2.imshow("Left Right Eyes",image)
## leye_ymin:leye_ymax, leye_xmin:leye_xmax --> left eye heigh, width
left_eye_box = image[leye_ymin:leye_ymax, leye_xmin:leye_xmax]
## reye_ymin:reye_ymax, reye_xmin:reye_xmax --> right eye heigh, width
right_eye_box = image[reye_ymin:reye_ymax, reye_xmin:reye_xmax]
# print(left_eye_box.shape, right_eye_box.shape) # left eye and right eye image
## [left eye box, right eye box]
eyes_coords = [[leye_xmin, leye_ymin, leye_xmax, leye_ymax],
[reye_xmin, reye_ymin, reye_xmax, reye_ymax]]
return left_eye_box, right_eye_box, eyes_coords
def check_model(self, model_structure, model_weights):
# raise NotImplementedError
try:
# Reads a network from the IR files and creates an IENetwork, load IR files into their related class, architecture with XML and weights with binary file
self.network = self.plugin.read_network(model=model_structure,
weights=model_weights)
except Exception as e:
raise ValueError(
"Error occurred during facial_landmarks_detection network initialization."
)
## check supported layer and performence counts reference:
# https://gist.github.com/justinshenk/9917891c0433f33967f6e8cd8fcaa49a
def performance(self):
perf_counts = self.exec_net.requests[0].get_perf_counts()
# print('\n', perf_counts)
print("\n## Facial landmarks detection model performance:")
print("{:<70} {:<15} {:<15} {:<15} {:<10}".format(
'name', 'layer_type', 'exet_type', 'status', 'real_time, us'))
for layer, stats in perf_counts.items():
print("{:<70} {:<15} {:<15} {:<15} {:<10}".format(
layer, stats['layer_type'], stats['exec_type'],
stats['status'], stats['real_time']))
def preprocess_input(self, image):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
# print(image.shape)
# print(image[2][1])
# cv2.imshow('image',image)
## convert RGB to BGR
image_cvt = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# print(image_cvt.shape) # (374, 238, 3)
# cv2.imshow('cvt',image_cvt)
# print('====',image_cvt[2][1])
# print(self.input_shape) # [1, 3, 48, 48]
H, W = self.input_shape[2], self.input_shape[3]
# print(H, W) # (48, 48)
image_resized = cv2.resize(image_cvt, (W, H))
# print(image_resized.shape) # (48, 48, 3)
## (optional)
# image_processed = np.transpose(np.expand_dims(image_resized, axis=0), (0,3,1,2))
image = image_resized.transpose((2, 0, 1))
# print(image.shape) # (3, 48, 48)
# add 1 dim at very start, then channels then H, W
image_processed = image.reshape(1, 3, self.input_shape[2],
self.input_shape[3])
# print(image_processed.shape) # (1, 3, 48, 48)
return image_processed
def preprocess_output(self, outputs):
'''
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
The net outputs a blob with the shape: [1, 10], containing a row-vector of 10 floating point values
for five landmarks coordinates in the form (x0, y0, x1, y1, ..., x5, y5).
All the coordinates are normalized to be in range [0,1].
'''
# print(output)
# print(outputs[self.output_names].shape) # (1, 10, 1, 1)
# print(outputs[self.output_names][0].shape) # (10, 1, 1)
# print(outputs[self.output_names][0])
# print('-----', outputs[self.output_names][0][0])
## here only need left eye and right eye
outs = outputs[self.output_names][0]
# print(outs.shape)
# print(outs[0][0][0])
# print(outs[0].tolist()) # [[0.37333157658576965]]
# print(outs[0].tolist()[0][0]) # [[0.37333157658576965]]
# print(type(outs)) # numpy.ndarry
leye_x, leye_y = outs[0][0][0], outs[1][0][0]
reye_x, reye_y = outs[2][0][0], outs[3][0][0]
coords_lr = (leye_x, leye_y, reye_x, reye_y)
# print(coords_lr)
return coords_lr
class GenericModel:
'''
Class for controlling similar model characteristics.
'''
def __init__(self, device):
self.device = device
self.net = None
self.plugin = None
self.input_blob = None
self.out_blob = None
self.net_plugin = None
self.infer_request_handle = None
self.outputs = None
def load_model(self, model, cpu_extension=None, plugin=None):
# Obtain model files path:
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
if not plugin:
self.plugin = IECore()
else:
self.plugin = plugin
if cpu_extension and 'CPU' in self.device:
self.plugin.add_extension(cpu_extension, "CPU")
self.net = IENetwork(model=model_xml, weights=model_bin)
# If applicable, add a CPU extension to self.plugin
if "CPU" in self.device:
supported_layers = self.plugin.query_network(self.net, "CPU")
not_supported_layers = [
layer for layer in self.net.layers.keys()
if layer not in supported_layers
]
if len(not_supported_layers) != 0:
sys.exit(1)
# Load the model to the network:
self.net_plugin = self.plugin.load_network(network=self.net,
device_name=self.device)
# Obtain other relevant information:
self.input_blob = next(iter(self.net.inputs))
self.out_blob = next(iter(self.net.outputs))
return self.plugin
def predict(self, image, request_id=0):
preprocessed_image = self.preprocess_input(image)
self.infer_request_handle = self.net_plugin.start_async(
request_id=request_id,
inputs={self.input_blob: preprocessed_image})
return self.net_plugin
def check_model(self):
pass
def preprocess_input(self, image):
input_shape = self.net.inputs[self.input_blob].shape
preprocessed_image = helpers.handle_image(image, input_shape[3],
input_shape[2])
return preprocessed_image
def wait(self, request_id=0):
status = self.net_plugin.requests[request_id].wait(-1)
return status
def get_output(self, request_id=0):
self.outputs = self.net_plugin.requests[request_id].outputs[
self.out_blob]
return self.outputs
class Model_X(ABC):
"""
Abstract Class for the Face Detection Model.
"""
def __init__(self, model_name, device="CPU", threshold=0.5, extensions=None):
self.model_weights = model_name + ".bin"
self.model_structure = model_name + ".xml"
self.device = device
self.threshold = threshold
self.extensions = extensions
@timeit
def load_model(self):
"""
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
"""
self.network = IENetwork(self.model_structure, self.model_weights)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_name = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_name].shape
self.core = IECore()
if self.extensions and "CPU" in self.device:
self.core.add_extension(self.extensions, self.device)
supported_layers = self.core.query_network(
network=self.network, device_name=self.device
)
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if unsupported_layers:
log.error(f"Unsupported layers found: {unsupported_layers}")
exit(1)
self.exec_network = self.core.load_network(self.network, self.device)
@timeit
def predict(self, image):
"""
This abstractmethod is meant for running predictions on the input image.
"""
outputs = self.exec_network.infer(
{self.input_name: self.preprocess_input(image)}
)
return self.preprocess_output(outputs, image)
def check_model(self):
raise NotImplementedError
def preprocess_input(self, image):
"""
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
"""
ret = cv2.resize(image, (self.input_shape[3], self.input_shape[2]))
ret = ret.transpose((2, 0, 1))
ret = ret.reshape(1, *ret.shape)
return ret
@abstractmethod
def preprocess_output(self, outputs):
"""
Before feeding the output of this model to the next model,
you might have to preprocess the output. This function is where you can do that.
"""
pass
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
input_blob = next(iter(net.inputs))
n, c, h, w = net.inputs[input_blob].shape
images = np.ndarray(shape=(n, c, h, w))
images_hw = []
for i in range(n):
image = cv2.imread(args.input[i])
ih, iw = image.shape[:-1]
images_hw.append((ih, iw))
log.info("File was added: ")
log.info(" {}".format(args.input[i]))
if (ih, iw) != (h, w):
image = cv2.resize(image, (w, h))
log.warning("Image {} is resized from {} to {}".format(
args.input[i], image.shape[:-1], (h, w)))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1 or len(net.inputs.keys())
== 2), "Sample supports topologies only with 1 or 2 inputs"
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
input_name, input_info_name = "", ""
for input_key in net.inputs:
if len(net.inputs[input_key].layout) == 4:
input_name = input_key
log.info("Batch size is {}".format(net.batch_size))
net.inputs[input_key].precision = 'U8'
elif len(net.inputs[input_key].layout) == 2:
input_info_name = input_key
net.inputs[input_key].precision = 'FP32'
if net.inputs[input_key].shape[1] != 3 and net.inputs[
input_key].shape[1] != 6 or net.inputs[input_key].shape[
0] != 1:
log.error(
'Invalid input info. Should be 3 or 6 values length.')
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
for output_key in net.outputs:
if net.layers[output_key].type == "DetectionOutput":
output_name, output_info = output_key, net.outputs[output_key]
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
log.info("Creating infer request and starting inference")
res = exec_net.infer(inputs={input_blob: images})
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info("Processing output blobs")
res = res[out_blob]
boxes, classes = {}, {}
data = res[0][0]
for number, proposal in enumerate(data):
if proposal[2] > 0:
imid = np.int(proposal[0])
ih, iw = images_hw[imid]
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(iw * proposal[3])
ymin = np.int(ih * proposal[4])
xmax = np.int(iw * proposal[5])
ymax = np.int(ih * proposal[6])
print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}"\
.format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="")
if proposal[2] > 0.5:
print(" WILL BE PRINTED!")
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
if not imid in classes.keys():
classes[imid] = []
classes[imid].append(label)
else:
print()
for imid in classes:
tmp_image = cv2.imread(args.input[imid])
for box in boxes[imid]:
cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]),
(232, 35, 244), 2)
cv2.imwrite("out.bmp", tmp_image)
log.info("Image out.bmp created!")
# -----------------------------------------------------------------------------------------------------
log.info("Execution successful\n")
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool"
)
def main():
args = build_argparser().parse_args()
# ------------- 1. Plugin initialization for specified device and load extensions library if specified -------------
log.info("Creating Inference Engine...")
ie = IECore()
config_user_specified = {}
config_min_latency = {}
devices_nstreams = {}
if args.num_streams:
devices_nstreams = {device: args.num_streams for device in ['CPU', 'GPU'] if device in args.device} \
if args.num_streams.isdigit() \
else dict([device.split(':') for device in args.num_streams.split(',')])
if 'CPU' in args.device:
if args.cpu_extension:
ie.add_extension(args.cpu_extension, 'CPU')
if args.number_threads is not None:
config_user_specified['CPU_THREADS_NUM'] = str(args.number_threads)
if 'CPU' in devices_nstreams:
config_user_specified['CPU_THROUGHPUT_STREAMS'] = devices_nstreams['CPU'] \
if int(devices_nstreams['CPU']) > 0 \
else 'CPU_THROUGHPUT_AUTO'
config_min_latency['CPU_THROUGHPUT_STREAMS'] = '1'
if 'GPU' in args.device:
if 'GPU' in devices_nstreams:
config_user_specified['GPU_THROUGHPUT_STREAMS'] = devices_nstreams['GPU'] \
if int(devices_nstreams['GPU']) > 0 \
else 'GPU_THROUGHPUT_AUTO'
config_min_latency['GPU_THROUGHPUT_STREAMS'] = '1'
# -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) --------------------
log.info("Loading network")
net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin")
# ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(
net.input_info
) == 1, "Sample supports only YOLO V3 based single input topologies"
# ---------------------------------------------- 4. Preparing inputs -----------------------------------------------
log.info("Preparing inputs")
input_blob = next(iter(net.input_info))
# Read and pre-process input images
if net.input_info[input_blob].input_data.shape[1] == 3:
input_height, input_width = net.input_info[
input_blob].input_data.shape[2:]
nchw_shape = True
else:
input_height, input_width = net.input_info[
input_blob].input_data.shape[1:3]
nchw_shape = False
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
input_stream = 0 if args.input == "cam" else args.input
mode = Mode(Modes.USER_SPECIFIED)
cap = cv2.VideoCapture(input_stream)
wait_key_time = 1
# ----------------------------------------- 5. Loading model to the plugin -----------------------------------------
log.info("Loading model to the plugin")
exec_nets = {}
exec_nets[Modes.USER_SPECIFIED] = ie.load_network(
network=net,
device_name=args.device,
config=config_user_specified,
num_requests=args.num_infer_requests)
exec_nets[Modes.MIN_LATENCY] = ie.load_network(
network=net,
device_name=args.device.split(":")[-1].split(",")[0],
config=config_min_latency,
num_requests=1)
empty_requests = deque(exec_nets[mode.current].requests)
completed_request_results = {}
next_frame_id = 0
next_frame_id_to_show = 0
mode_info = {mode.current: ModeInfo()}
event = threading.Event()
callback_exceptions = []
# ----------------------------------------------- 6. Doing inference -----------------------------------------------
log.info("Starting inference...")
print(
"To close the application, press 'CTRL+C' here or switch to the output window and press ESC key"
)
print(
"To switch between min_latency/user_specified modes, press TAB key in the output window"
)
presenter = monitors.Presenter(
args.utilization_monitors, 55,
(round(cap.get(cv2.CAP_PROP_FRAME_WIDTH) / 4),
round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) / 8)))
while (cap.isOpened() \
or completed_request_results \
or len(empty_requests) < len(exec_nets[mode.current].requests)) \
and not callback_exceptions:
if next_frame_id_to_show in completed_request_results:
frame, output, start_time, is_same_mode = completed_request_results.pop(
next_frame_id_to_show)
next_frame_id_to_show += 1
if is_same_mode:
mode_info[mode.current].frames_count += 1
objects = get_objects(output, net, (input_height, input_width),
frame.shape[:-1], args.prob_threshold,
args.keep_aspect_ratio)
objects = filter_objects(objects, args.iou_threshold,
args.prob_threshold)
if len(objects) and args.raw_output_message:
log.info(
" Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR "
)
origin_im_size = frame.shape[:-1]
presenter.drawGraphs(frame)
for obj in objects:
# Validation bbox of detected object
obj['xmax'] = min(obj['xmax'], origin_im_size[1])
obj['ymax'] = min(obj['ymax'], origin_im_size[0])
obj['xmin'] = max(obj['xmin'], 0)
obj['ymin'] = max(obj['ymin'], 0)
color = (min(obj['class_id'] * 12.5,
255), min(obj['class_id'] * 7,
255), min(obj['class_id'] * 5, 255))
det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \
str(obj['class_id'])
if args.raw_output_message:
log.info(
"{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ".
format(det_label, obj['confidence'], obj['xmin'],
obj['ymin'], obj['xmax'], obj['ymax'], color))
cv2.rectangle(frame, (obj['xmin'], obj['ymin']),
(obj['xmax'], obj['ymax']), color, 2)
cv2.putText(
frame, "#" + det_label + ' ' +
str(round(obj['confidence'] * 100, 1)) + ' %',
(obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX,
0.6, color, 1)
# Draw performance stats over frame
if mode_info[mode.current].frames_count != 0:
fps_message = "FPS: {:.1f}".format(mode_info[mode.current].frames_count / \
(perf_counter() - mode_info[mode.current].last_start_time))
mode_info[
mode.current].latency_sum += perf_counter() - start_time
latency_message = "Latency: {:.1f} ms".format((mode_info[mode.current].latency_sum / \
mode_info[mode.current].frames_count) * 1e3)
put_highlighted_text(frame, fps_message, (15, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.75,
(200, 10, 10), 2)
put_highlighted_text(frame, latency_message, (15, 50),
cv2.FONT_HERSHEY_COMPLEX, 0.75,
(200, 10, 10), 2)
mode_message = "{} mode".format(mode.current.name)
put_highlighted_text(frame, mode_message,
(10, int(origin_im_size[0] - 20)),
cv2.FONT_HERSHEY_COMPLEX, 0.75, (10, 10, 200),
2)
if not args.no_show:
cv2.imshow("Detection Results", frame)
#print(frame.shape)
cv2.imwrite("prob_threshold-0.2.png", frame)
key = cv2.waitKey(wait_key_time)
if key in {ord("q"), ord("Q"), 27}: # ESC key
break
if key == 9: # Tab key
prev_mode = mode.current
mode.next()
await_requests_completion(exec_nets[prev_mode].requests)
empty_requests.clear()
empty_requests.extend(exec_nets[mode.current].requests)
mode_info[prev_mode].last_end_time = perf_counter()
mode_info[mode.current] = ModeInfo()
else:
presenter.handleKey(key)
elif empty_requests and cap.isOpened():
start_time = perf_counter()
ret, frame = cap.read()
if not ret:
if args.loop_input:
cap.open(input_stream)
else:
cap.release()
continue
request = empty_requests.popleft()
# resize input_frame to network size
in_frame = preprocess_frame(frame, input_height, input_width,
nchw_shape, args.keep_aspect_ratio)
# Start inference
request.set_completion_callback(
py_callback=async_callback,
py_data=(request, next_frame_id, mode.current, frame,
start_time, completed_request_results, empty_requests,
mode, event, callback_exceptions))
request.async_infer(inputs={input_blob: in_frame})
next_frame_id += 1
else:
event.wait()
if callback_exceptions:
raise callback_exceptions[0]
for mode_value in mode_info.keys():
log.info("")
log.info("Mode: {}".format(mode_value.name))
end_time = mode_info[mode_value].last_end_time if mode_value in mode_info \
and mode_info[mode_value].last_end_time is not None \
else perf_counter()
log.info("FPS: {:.1f}".format(mode_info[mode_value].frames_count / \
(end_time - mode_info[mode_value].last_start_time)))
log.info("Latency: {:.1f} ms".format((mode_info[mode_value].latency_sum / \
mode_info[mode_value].frames_count) * 1e3))
print(presenter.reportMeans())
for exec_net in exec_nets.values():
await_requests_completion(exec_net.requests)
class Model_Face_Detect:
def __init__(self, model_name, device, extensions=None):
self.model_weights = model_name + '.bin'
self.model_structure = model_name + '.xml'
self.device = device
self.extensions = extensions
try:
self.core = IECore()
self.network = IENetwork(model=self.model_structure,
weights=self.model_weights)
network_layers = self.network.layers.keys()
supported_layers = self.core.query_network(
network=self.network, device_name=self.device).keys()
for layer in network_layers:
if layer in supported_layers:
pass
else:
ext_required = True
break
if self.extensions != None and "CPU" in self.device and ext_required:
self.core.add_extension(self.extensions, self.device)
for layer in network_layers:
if layer in supported_layers:
pass
else:
msg = "Layer extension doesn't support all layers"
log.error(msg)
raise Exception(msg)
except Exception as e:
raise ValueError(
"Could not Initialise the network. Have you enterred the correct model path?"
)
self.input_name = next(iter(self.network.inputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_name = next(iter(self.network.outputs))
self.output_shape = self.network.outputs[self.output_name].shape
def load_model(self):
self.exec_network = self.core.load_network(self.network, self.device)
return
def predict(self, image, threshold):
input_img = self.preprocess_input(image)
input_dict = {self.input_name: input_img}
outputs = self.exec_network.infer(input_dict)[self.output_name]
coords = self.preprocess_outputs(outputs, threshold,
(image.shape[1], image.shape[0]))
return self.face_coord(coords, image)
def face_coord(self, coords, image):
if (len(coords) == 1):
coords = coords[0]
face_coords = image[coords[1]:coords[3], coords[0]:coords[2]]
return coords, face_coords
return False, False
def preprocess_input(self, image):
preprocessed_frame = cv2.resize(
image, (self.input_shape[3], self.input_shape[2]))
preprocessed_frame = preprocessed_frame.transpose((2, 0, 1))
return preprocessed_frame.reshape(1, *preprocessed_frame.shape)
def preprocess_outputs(self, outputs, threshold, dim):
_ = []
for box in outputs[0][0]:
ct = box[2]
if ct > threshold:
xmin = int(box[3] * dim[0])
ymin = int(box[4] * dim[1])
xmax = int(box[5] * dim[0])
ymax = int(box[6] * dim[1])
_.append([xmin, ymin, xmax, ymax])
return _
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.network_inputs = None
self.network_outputs = None
self.exec_network = None
self.infer_request = None
def load_model(self, model, device='CPU', cpu_extension=None):
### TODO: Load the model ###
self.plugin = IECore()
model_bin_path = os.path.splitext(model)[0] + ".bin"
self.network = IENetwork(model=model, weights=model_bin_path)
### TODO: Check for supported layers ###
layers_supported = self.plugin.query_network(self.network,
device_name=device)
layers = self.network.layers.keys()
for layer in layers:
if layer not in layers_supported:
### TODO: Add any necessary extensions ###
self.plugin.add_extension(cpu_extension, device)
break
self.exec_network = self.plugin.load_network(self.network, device)
self.network_inputs = next(iter(self.network.inputs))
self.network_outputs = next(iter(self.network.outputs))
### TODO: Return the loaded inference plugin ###
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
return self.network.inputs[self.network_inputs].shape
def exec_net(self, request_id, net_input):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
self.infer_request_handle = self.exec_network.start_async(
request_id, inputs={self.network_inputs: net_input})
return
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
return self.infer_request_handle.wait()
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
return self.infer_request_handle.outputs[self.network_outputs]
