def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Plugin initialization for specified device and load extensions library if specified log.info("Initializing plugin for {} device...".format(args.device)) ie = IECore() #plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir) if args.cpu_extension and 'CPU' in args.device: log.info("Loading plugins for {} device...".format(args.device)) #ie.add_extension(args.cpu_extension, args.device) // openvino2019-R1 ie.add_extension(args.cpu_extension, "CPU") # Read IR log.info("Reading IR...") net = IENetwork(model=model_xml, weights=model_bin) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) assert len( net.inputs.keys()) == 1, "Sample supports only single input topologies" assert len( net.outputs) == 1, "Sample supports only single output topologies" input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) if args.input == 'cam': input_stream = 0 out_file_name = 'cam' else: input_stream = args.input assert os.path.isfile(args.input), "Specified input file doesn't exist" out_file_name = os.path.splitext(os.path.basename(args.input))[0] log.info("Loading IR to the plugin...") exec_net = ie.load_network(network=net, num_requests=args.number_infer_requests, device_name=args.device) log.info( "Starting inference in async mode, {} requests in parallel...".format( args.number_infer_requests)) job_id = str(os.environ['PBS_JOBID']) result_file = open( os.path.join(args.output_dir, 'output_' + job_id + '.txt'), "w") pre_infer_file = os.path.join(args.output_dir, 'pre_progress_' + job_id + '.txt') infer_file = os.path.join(args.output_dir, 'i_progress_' + job_id + '.txt') processed_vid = '/tmp/processed_vid.bin' # Read and pre-process input image if isinstance(net.inputs[input_blob], list): n, c, h, w = net.inputs[input_blob] else: n, c, h, w = net.inputs[input_blob].shape del net cap = cv2.VideoCapture(input_stream) video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) if video_len < args.number_infer_requests: args.number_infer_requests = video_len #Pre inference processing, read mp4 frame by frame, process using openCV and write to binary file width = int(cap.get(3)) height = int(cap.get(4)) CHUNKSIZE = n * c * w * h id_ = 0 with open(processed_vid, 'w+b') as f: time_start = time.time() while cap.isOpened(): ret, next_frame = cap.read() if not ret: break in_frame = cv2.resize(next_frame, (w, h)) in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) bin_frame = bytearray(in_frame) f.write(bin_frame) id_ += 1 if id_ % 10 == 0: progressUpdate(pre_infer_file, time.time() - time_start, id_, video_len) cap.release() if args.labels: with open(args.labels, 'r') as f: labels_map = [x.strip() for x in f] else: labels_map = None current_inference = 0 previous_inference = 1 - args.number_infer_requests infer_requests = exec_net.requests frame_count = 0 try: infer_time_start = time.time() with open(processed_vid, "rb") as data: while frame_count < video_len: # Read next frame from input stream if available and submit it for inference byte = data.read(CHUNKSIZE) if not byte == b"": deserialized_bytes = np.frombuffer(byte, dtype=np.uint8) in_frame = np.reshape(deserialized_bytes, newshape=(n, c, h, w)) exec_net.start_async(request_id=current_inference, inputs={input_blob: in_frame}) # Retrieve the output of an earlier inference request if previous_inference >= 0: status = infer_requests[previous_inference].wait() if status is not 0: raise Exception( "Infer request not completed successfully") # Parse inference results res = infer_requests[previous_inference].outputs[out_blob] processBoxes(frame_count, res, labels_map, args.prob_threshold, width, height, result_file) frame_count += 1 # Write data to progress tracker if frame_count % 10 == 0: progressUpdate(infer_file, time.time() - infer_time_start, frame_count + 1, video_len + 1) # Increment counter for the inference queue and roll them over if necessary current_inference += 1 if current_inference >= args.number_infer_requests: current_inference = 0 previous_inference += 1 if previous_inference >= args.number_infer_requests: previous_inference = 0 # End while loop total_time = time.time() - infer_time_start with open(os.path.join(args.output_dir, 'stats_{}.txt'.format(job_id)), 'w') as f: f.write('{:.3g} \n'.format(total_time)) f.write('{} \n'.format(frame_count)) result_file.close() finally: log.info("Processing done...") del exec_net
class facelandmarksdetection: def __init__(self, model_name, device='CPU', extensions=None): self.model_name = model_name self.device = device self.extensions = extensions self.model_structure = self.model_name self.model_weights = self.model_name.split('.')[0]+'.bin' self.plugin = None self.network = None self.exec_net = None self.input_name = None self.input_shape = None self.output_names = None self.output_shape = None try: self.model=IENetwork(self.model_structure, self.model_weights) except Exception as e: raise ValueError("Could not Initialise the network. Have you enterred the correct model path?") self.input_name=next(iter(self.model.inputs)) self.input_shape=self.model.inputs[self.input_name].shape self.output_name=next(iter(self.model.outputs)) self.output_shape=self.model.outputs[self.output_name].shape def load_model(self): self.plugin=IECore() supported_layers = self.plugin.query_network(network=self.model, device_name=self.device) unsupported_layers = [l for l in self.model.layers.keys() if l not in supported_layers] if len(unsupported_layers)!=0: print("unsupported layers found") exit(1) self.exec_net=self.plugin.load_network(network=self.model,device_name=self.device,num_requests=1) def predict(self, image): self.processed_image=self.preprocess_input(image) outputs = self.exec_net.infer({self.input_name:self.processed_image}) coords = self.preprocess_output(outputs) h=image.shape[0] w=image.shape[1] coords = coords* np.array([w, h, w, h]) coords = coords.astype(np.int32) l_xmin=coords[0]-10 l_xmax=coords[0]+10 l_ymin=coords[1]-10 l_ymax=coords[1]+10 r_xmin=coords[2]-10 r_xmax=coords[2]+10 r_ymin=coords[3]-10 r_ymax=coords[3]+10 left_eye = image[l_ymin:l_ymax, l_xmin:l_xmax] right_eye = image[r_ymin:r_ymax, r_xmin:r_xmax] coords = [[l_xmin,l_ymin,l_xmax,l_ymax], [r_xmin,r_ymin,r_xmax,r_ymax]] return left_eye, right_eye, coords def check_model(self): raise NotImplementedError def preprocess_input(self, image): image_ct = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) self.image=cv2.resize(image_ct,(self.input_shape[3],self.input_shape[2])) ## cv2.resize(frame, (w, h)) self.image=self.image.transpose((2, 0, 1)) self.image=self.image.reshape(1, *self.image.shape) return self.image def preprocess_output(self, outputs): res=outputs[self.output_name][0] lx = res[0].tolist()[0][0] ly = res[1].tolist()[0][0] rx = res[2].tolist()[0][0] ry = res[3].tolist()[0][0] return(lx,ly,rx,ry)
def main(): # arguments parser = ArgumentParser() parser.add_argument( "-m", "--model", help="Required. Path to an .xml file with a trained model", required=True, type=str) parser.add_argument("-i", "--input", help="Required. Path to a input image file", required=True, type=str) parser.add_argument( "-l", "--cpu_extension", help= "Optional. Required for CPU custom layers. Absolute MKLDNN (CPU)-targeted custom layers. " "Absolute path to a shared library with the kernels implementations", type=str, default=None) parser.add_argument( "-d", "--device", help= "Optional. Specify the target device to infer on; CPU, GPU, FPGA, HDDL or MYRIAD is acceptable. " "Sample will look for a suitable plugin for device specified. Default value is CPU", default="CPU", type=str) args = parser.parse_args() # logging log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) log.info("creating inference engine") ie = IECore() if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("Loading network") net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin") if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify a CPU extensions library path in sample's command line parameters " "using -l or --cpu_extension command line argument") sys.exit(1) assert len( net.input_info) == 1, "Sample supports only single input topologies" assert len( net.outputs) == 1, "Sample supports only single output topologies" log.info("preparing input blobs") input_blob = next(iter(net.input_info)) out_blob = next(iter(net.outputs)) net.batch_size = 1 # read and pre-process input image _, _, height, width = net.input_info[input_blob].input_data.shape image = cv2.imread(args.input, cv2.IMREAD_COLOR) (input_height, input_width) = image.shape[:-1] # resize if (input_height, input_width) != (height, width): log.info("Image is resized from {} to {}".format( image.shape[:-1], (height, width))) image = cv2.resize(image, (width, height), cv2.INTER_CUBIC) # prepare input image = image.astype(np.float32) image = image.transpose((2, 0, 1)) image_input = np.expand_dims(image, 0) # loading model to the plugin log.info("loading model to the plugin") exec_net = ie.load_network(network=net, device_name=args.device) # start sync inference log.info("starting inference") res = exec_net.infer(inputs={input_blob: image_input}) # processing output blob log.info("processing output blob") disp = res[out_blob][0] # resize disp to input resolution disp = cv2.resize(disp, (input_width, input_height), cv2.INTER_CUBIC) # rescale disp disp_min = disp.min() disp_max = disp.max() if disp_max - disp_min > 1e-6: disp = (disp - disp_min) / (disp_max - disp_min) else: disp.fill(0.5) # pfm out = os.path.join(os.path.dirname(__file__), 'disp.pfm') cv2.imwrite(out, disp) log.info("Disparity map was saved to {}".format(out)) # png out = os.path.join(os.path.dirname(__file__), 'disp.png') plt.imsave(out, disp, vmin=0, vmax=1, cmap='inferno') log.info("Color-coded disparity image was saved to {}".format(out)) log.info( "This demo is an API example, for any performance measurements please use " "the dedicated benchmark_app tool from the openVINO toolkit\n")
class face_detection: ''' Class for the Face Detection Model. ''' def __init__(self, model, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model = model self.device = device self.extension = extensions self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None self.device_list = [] self.print_once = True self.network_input_shape = None self.filtered_result = [[]] # ====== Load model files, verify model, and get the input shap parameters at start ====== log.info("<---------- class Facedetection model --------->") self.load_model() self.check_model() self.network_input_shape = self.get_input_shape() def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() log.info("------device avaibility--------") for available_devices in self.plugin.available_devices: #Dont use device variable, conflicts. self.device_list.append(available_devices) log.info("Available device: " + str(self.device_list)) #get name of available devices log.info("---------Plugin version--------") ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info log.info("descr: maj.min.num" + str(ver.description) + "." + str(ver.major) + "." + str(ver.minor) + "." + str(ver.build_number)) ### Load IR files into their related class model_xml = self.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Check if path is not given or model name doesnt ends with .xml if model_xml is not None and model_xml.find('.xml') != -1: f, s = model_xml.rsplit( ".", 1 ) #check from last "." and "r"split only one element from last model_bin = f + ".bin" else: log.error( "Error! Model files are not found or invalid, check paths") log.error("Program stopped") sys.exit() #exit program no further execution log.info("-------------Model path----------") log.info("XML: " + str(model_xml)) log.info("bin: " + str(model_bin)) self.network = IENetwork(model=model_xml, weights=model_bin) log.info("ModelFiles are successfully loaded into IENetwork") return def check_model(self): ''' This function will check the input model for compatibility with hardware, It will check for supported and unsuported layers if any. No return required. ''' ### Check for supported layers ### log.info("Checking for supported Network layers...") # Query network will return all the layer, required all the time if device changes. supported_layers = self.plugin.query_network(network=self.network, device_name="CPU") log.info("------Status of default Network layers--------") log.info("No. of Layers in network: " + str(len(self.network.layers))) log.info("No. of supported layers:" + str(len(supported_layers))) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.info("Unsupported layers found:" + str(unsupported_layers)) log.info("CPU extension required and adding...") #exit(1) ### Adding any necessary extensions ### if self.extension and "CPU" in self.device: self.plugin.add_extension(self.extension, self.device) log.info("Checking for CPU extension compatibility...") # Again Query network will return fresh list of supported layers. supported_layers = self.plugin.query_network( network=self.network, device_name="CPU") log.info( "------Status of Network layers with CPU Extension--------" ) log.info("No. of Layers in network:" + str(len(self.network.layers))) log.info("No. of supported layers:" + str(len(supported_layers))) log.info("CPU extension added sucessfully!") unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.error("Unsupported layers found: " + str((unsupported_layers))) log.error("Error! Model not supported, Program stopped") exit() else: log.error("Error! cpu extension not found") log.error("Program stopped") exit() else: log.info("All the layers are supported, No CPU extension required") # This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference self.exec_network = self.plugin.load_network(self.network, "CPU") print("IR successfully loaded into Inference Engine") return def get_input_shape(self): ''' Get dimensions of inputs ''' ### Get the input layer informations ### log.info("-----Accessing input layer information-----") log.info('Face detection model Network input layers = ' + str(list(self.network.inputs))) log.info('Face detection model Network input layers type: ' + str(type(self.network.inputs))) self.input_blob = next(iter(self.network.inputs)) #Origional log.info("-------------------------------") return self.network.inputs[self.input_blob].shape #Origional def wait(self): ### Wait for the Async request to be complete. ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### Extract and return the output results self.output_blob = next(iter(self.network.outputs)) # First return the name of blob as dictionary and second output of first blob as Nd array return self.exec_network.requests[ 0].outputs, self.exec_network.requests[0].outputs[self.output_blob] def preprocess_input(self, input_frames_raw, network_input_shape_height, network_input_shape_width): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' p_frame = cv2.resize( input_frames_raw, (network_input_shape_height, network_input_shape_width)) #Resize as per network input spec. p_frame = p_frame.transpose((2, 0, 1)) #swap channel cxhxw p_frame = p_frame.reshape( 1, *p_frame.shape) #add one axis 1 to make 4D shape for network input #print(p_frame.shape) #Debug output return p_frame def predict(self, input_frames_raw, input_frame_raw_height, input_frame_raw_width): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image or video frame. input: RBG image in jpg format or video frame. ''' # pre-process origional frame to match network inputs. p_frame = self.preprocess_input(input_frames_raw, self.network_input_shape[3], self.network_input_shape[2]) # Run Async inference self.exec_network.start_async( request_id=0, #Origional inputs={self.input_blob: p_frame}) # run inference # wait until result available if self.wait() == 0: ### the results of the inference request ### blob, result = self.get_output() # Print available blob infirmation blob_list = [] if self.print_once: # Print only Once self.print_once = False for name, output_ in blob.items( ): #Find the possible BLOBS for name, blob_list.append(name) log.info( "The name of available blob of face detection model is: " + str(blob_list)) return result
# Load the model and create the inference engine log.info(f'Loading model') model_xml = args['model'] model_bin = os.path.splitext(model_xml)[0] + '.bin' log.info(f'... model file {model_xml}') log.info(f'... weights file {model_bin}') log.info('Creating inference engine') ie = IECore() net = IENetwork(model=model_xml, weights=model_bin) log.info('...Checking that the network can be run on the selected device') supported_layers = ie.query_network(net, args['device']) not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( '...The following layers are not supported by the device.\n {}'.format( ', '.join(not_supported_layers))) log.info('...Checking that the network has a single input and output') assert len(net.inputs.keys() ) == 1, 'The application supports single input topologies.' assert len( net.outputs) == 1, 'The application supports single output topologies'
landmark_bin = 'models/landmarks-regression-retail-0009/landmarks-regression-retail-0009.bin' head_pose_xml = 'models/head-pose-estimation-adas-0001/head-pose-estimation-adas-0001.xml' head_pose_bin = 'models/head-pose-estimation-adas-0001/head-pose-estimation-adas-0001.bin' gaze_est_xml = 'models/gaze-estimation-adas-0002/gaze-estimation-adas-0002.xml' gaze_est_bin = 'models/gaze-estimation-adas-0002/gaze-estimation-adas-0002.bin' print("INFO: Starting to load models.....") core = IECore() model = IENetwork(model_xml, model_bin) net = core.load_network(model, device, num_requests=1) fps = FPS().start() layers_list = core.query_network(network=model, device_name=device) print(layers_list) model2 = IENetwork(landmark_xml, landmark_bin) net2 = core.load_network(model2, device, num_requests=1) model3 = IENetwork(head_pose_xml, head_pose_bin) net3 = core.load_network(model3, device, num_requests=1) model4 = IENetwork(gaze_est_xml, gaze_est_bin) net4 = core.load_network(model4, device, num_requests=1) print("INFO: All models loaded...") input_name = next(iter(net.inputs)) input_shape = net.inputs[input_name].shape
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None def load_model(self, model, device="CPU", cpu_extension=None): ### TODO: Load the model ### model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Initialize the plugin self.plugin = IECore() # Add a CPU extension, if applicable if cpu_extension and "CPU" in device: self.plugin.add_extension(cpu_extension, device) # Read the IR as a IENetwork self.network = IENetwork(model=model_xml, weights=model_bin) ### TODO: Check for supported layers ### supported_layers = self.plugin.query_network(network=self.network, device_name=device) # Unsupported layers unsupported_layers = [ layer for layer in self.network.layers.keys() if layer not in supported_layers ] ### TODO: Add any necessary extensions ### if unsupported_layers: log.warning( "Unsupported layers found: {}".format(unsupported_layers)) self.plugin.add_extension(cpu_extension, device) # Load the IENetwork into the plugin self.exec_network = self.plugin.load_network(self.network, device) # Get the input layer self.input_blob = next(iter(self.network.inputs)) self.output_blob = next(iter(self.network.outputs)) ### TODO: Return the loaded inference plugin ### ### Note: You may need to update the function parameters. ### return def get_input_shape(self): ### TODO: Return the shape of the input layer ### return self.network.inputs[self.input_blob].shape def exec_net(self, image): ### TODO: Start an asynchronous request ### self.exec_network.start_async(request_id=0, inputs={self.input_blob: image}) ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### return def wait(self): ### TODO: Wait for the request to be complete. ### status = self.exec_network.requests[0].wait(-1) ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### return status def get_output(self): ### TODO: Extract and return the output results ### Note: You may need to update the function parameters. ### return self.exec_network.requests[0].outputs[self.output_blob]
class FaceDetection: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' #From params self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extensions = extensions #For app self.core = None self.network = None self.exec_net = None self.unsupported_layers = None self.prob_threshold = 0.6 self.image = None def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.core = IECore() self.network = self.core.read_network(model=self.model_structure, weights=self.model_weights) supported_layers = self.core.query_network(network=self.network, device_name=self.device) self.unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] self.check_model() logging.info("Checked face-dection model") self.exec_net = self.core.load_network(network=self.network, device_name=self.device, num_requests=1) self.input = next(iter(self.network.inputs)) self.output = next(iter(self.network.outputs)) def predict(self, image): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' img_processed = self.preprocess_input(image.copy()) self.image = image self.exec_net.start_async(request_id=0, inputs={self.input: img_processed}) while self.exec_net.requests[0].wait(-1) == 0: result = self.exec_net.requests[0].outputs[self.output] return self.preprocess_output(result) def check_model(self): if len(self.unsupported_layers) != 0: self.core.add_extension(self.extensions, self.device) supported_layers = self.core.query_network(network=self.network, device_name=self.device) self.unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(self.unsupported_layers) != 0: logging.error("Unsupported layers") exit(1) def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' net_input_shape = [] net_input_shape = self.network.inputs[self.input].shape p_frame = None p_frame = cv2.resize(image, (net_input_shape[3], net_input_shape[2])) p_frame = p_frame.transpose(2, 0, 1) p_frame = p_frame.reshape(1, *p_frame.shape) return p_frame def preprocess_output(self, outputs): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' face_coords = [] outs = outputs[0][0] conf = 0 x_min = 0 x_max = 0 y_min = 0 y_max = 0 height = self.image.shape[0] width = self.image.shape[1] for out in outs: conf = out[2] if conf > self.prob_threshold: x_min = out[3] * width y_min = out[4] * height x_max = out[5] * width y_max = out[6] * height face_coords.append([x_min, y_min, x_max, y_max]) if (len(face_coords) == 0): return None #Only need one person, so take the first box with good conf first_face = face_coords[0] first_face = [round(x) for x in first_face] first_face_box = self.image[first_face[1]:first_face[3], first_face[0]:first_face[2]] return first_face_box, first_face
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None def load_model(self, model_xml, device="CPU"): ### TODO: Load the model ### model_bin = os.path.splitext(model_xml)[0] + ".bin" self.plugin = IECore() self.network = IENetwork(model_xml, model_bin) ### TODO: Check for supported layers ### supported_layers = self.plugin.query_network(self.network, device) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if (len(unsupported_layers) > 0): print("ERROR: Unsupported layers found!") exit(1) ### DEPRECATED ### TODO: Add any necessary extensions ### ### TODO: Return the loaded inference plugin ### self.exec_network = self.plugin.load_network(self.network, device) # print("EXEC_NET INPUTS: " + str(self.network.outputs["detection_output"].shape)) self.input_blob = next( iter(self.network.inputs )) # "image_tensor" #next(iter(self.network.inputs)) self.output_blob = next( iter(self.network.outputs )) # "detection_output" #next(iter(self.network.outputs)) return self.plugin def get_input_shape(self): ### TODO: Return the shape of the input layer ### return self.network.inputs[self.input_blob].shape def exec_net(self, image): ### TODO: Start an asynchronous request ### ### TODO: Return any necessary information ### infer_request = self.exec_network.start_async(0, {self.input_blob: image}) return infer_request def wait(self): ### TODO: Wait for the request to be complete. ### ### TODO: Return any necessary information ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### TODO: Extract and return the output results return self.exec_network.requests[0].outputs[self.output_blob]
class Model_Face_Detection: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, conf=0.5, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_name = model_name self.device = device self.extensions = extensions self.confidence = conf self.ie = None self.network = None self.exec_network = None self.infer_request = None self.input_name = None self.input_shape = None self.output_name = None self.output_shape = None self.model_width = None self.model_height = None self.width = None self.height = None self.model_width = None self.model_height = None self.frame = None def load_model(self): ''' Load the model given IR files. Defaults to CPU as device for use in the workspace. Synchronous requests made within. ''' log.info(f"##### Loading Model: {self.model_name}") model_xml = self.model_name + ".xml" model_bin = self.model_name + ".bin" # Initialize the inference engine self.ie = IECore() # Add a CPU extension, if applicable if self.extensions and "CPU" in self.device: self.ie.add_extension(self.extensions, self.device) # Read the IR as a IENetwork self.network = self.ie.read_network(model=model_xml, weights=model_bin) self.check_model() # Load the IENetwork into the inference engine self.exec_network = self.ie.load_network(self.network, self.device) # Get the layer's info self.input_name = next(iter(self.exec_network.inputs)) self.output_name = next(iter(self.exec_network.outputs)) self.input_shape = self.exec_network.inputs[self.input_name].shape self.output_shape = self.exec_network.outputs[self.output_name].shape log.info(f"Input shape: {self.input_shape}") log.info(f"Output shape: {self.output_shape}") self.model_width = self.input_shape[3] self.model_height = self.input_shape[2] log.info( f'Input image will be resized to ( {self.model_width} x {self.model_height} ) for inference' ) return self.exec_network def get_input_shape(self): ''' Gets the input shape of the network ''' return self.input_shape def get_output_shape(self): ''' Gets the output shape of the network ''' return self.output_shape def predict(self, image): ''' This method is meant for running predictions on the input image. ''' self.frame = image self.height = image.shape[0] self.width = image.shape[1] frame_inference = self.preprocess_input(image) outputs = self.exec_network.infer( inputs={self.input_name: frame_inference}) return self.preprocess_output(outputs[self.output_name]) def check_model(self): ### Check for any unsupported layers, and let the user ### know if anything is missing. Exit the program, if so. supported_layers = self.ie.query_network(network=self.network, device_name=self.device) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.error( "Unsupported layers found: {}".format(unsupported_layers)) log.error( "Check whether extensions are available to add to IECore.") exit(1) def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' frame_inference = cv2.resize(image, (self.model_width, self.model_height)) # Transform the image from the original size to the (1, 3, 320, 544) input shape frame_inference = frame_inference.transpose((2, 0, 1)) frame_inference = frame_inference.reshape(1, *frame_inference.shape) return frame_inference def extract_face(self, box): margin_top = 0 margin = 0 width = self.width height = self.height x_min = int(max(box[3] - box[3] * margin, 0) * width) y_min = int(max(box[4] - box[4] * margin_top, 0) * height) x_max = int(min(box[5] + box[5] * margin, 1) * width) y_max = int(min(box[6] + box[6] * margin, 1) * height) return self.frame[y_min:y_max, x_min:x_max] def preprocess_output(self, outputs): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' # Return only the first face if outputs[0][0][ 0] is not None and outputs[0][0][0][2] >= self.confidence: return self.extract_face(outputs[0][0][0]) else: return None
class Face_Detector: ''' Class for the Face Detection Model. ''' def __init__(self): self.plugin = None self.net = None self.input_name = None self.output_name = None self.exec_net = None self.request = None def load_model(self, model_path, device='CPU', extensions=None): start = time.time() if not os.path.isfile(model_path): log.error("Wrong model xml path specified" + model_path) exit(1) model_xml = model_path model_bin = os.path.splitext(model_xml)[0] + ".bin" self.plugin = IECore() if extensions and "CPU" in device: self.plugin.add_extension(extensions, device) self.net = IENetwork(model=model_xml, weights=model_bin) self.exec_net = self.plugin.load_network(self.net, device) # iter_i = iter(self.net.inputs) # print(next(iter_i)) # print(next(iter_i)) self.input_name = next(iter(self.net.inputs)) self.output_name = next(iter(self.net.outputs)) supported_layers = self.plugin.query_network(network=self.net, device_name="CPU") unsupported_layers = [] for l in self.net.layers.keys(): if l not in supported_layers: unsupported_layers.append(l) if len(unsupported_layers) != 0: log.error( "Unsupported layers found: {}".format(unsupported_layers)) log.error("Check your extensions") exit(1) end = time.time() # print('Face detection model load time',start-end) return def preprocess(self, frame): image = np.copy(frame) shape = self.net.inputs[self.input_name].shape image = cv2.resize(image, (shape[3], shape[2])) image = image.transpose((2, 0, 1)) image = image.reshape(1, *image.shape) return image def predict(self, image, thres): frame = self.preprocess(image) start = time.time() self.request = self.exec_net.start_async( request_id=0, inputs={self.input_name: frame}) status = self.request.wait() if (status == 0): end = time.time() # print('Face detection model inference time',start-end) crop_face, face_count, points = self.preprocess_output( image, thres) return crop_face, face_count, points def check_model(self): raise NotImplementedError def preprocess_output(self, frame, thres): outputs = self.request.outputs[self.output_name] h, w, c = frame.shape faces = 0 crop_face = 0 points = [] for box in outputs[0][0]: conf = box[2] if conf >= thres and box[1] == 1: if (faces > 1): log.error("found more than one face this" "may affect performance of the model") return crop_face, faces xmin = int(box[3] * w) ymin = int(box[4] * h) xmax = int(box[5] * w) ymax = int(box[6] * h) faces += 1 points.append(xmin) points.append(ymin) crop_face = frame[ymin:ymax, xmin:xmax] self.draw_points(frame, (xmin, ymin), (xmax, ymax)) return crop_face, faces, points def draw_points(self, frame, f, s): cv2.rectangle(frame, f, s, (130, 20, 25), 4)
class Gaze_Estimation: ''' Class for the Gaze Estimation Model. ''' def __init__(self, model_name): Model.__init__(self, model_name) # initialize the base class # then create/override the variable #self.model_name = "faceDetectionModel" self.model_name = model_name self.logger = logging.getLogger(__name__) try: self.model = IENetwork(self.model_structure, self.model_weights) except Exception: self.logger.exception("The Network could not be initialized. Check that you have the correct model path") self.load_model() Model.check_model(self, model_name) def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.core=IECore() ### TODO: Check for supported layers ### supported_layers = self.core.query_network(network=self.model, device_name=self.device) unsupported_layers = [l for l in self.model.layers.keys() if l not in supported_layers] if len(unsupported_layers)!=0: ### TODO: Add any necessary extensions ### if self.cpu_extension and "CPU" in self.device: self.core.add_extension(self.cpu_extension, self.device) else: self.logger.debug("Add CPU extension and device type or run layer with original framework") exit(1) self.net=self.core.load_network(network=self.model,device_name=self.device,num_requests=1) self.input_name=[i for i in self.model.inputs.keys()] self.input_shape=self.model.inputs[self.input_name[1]].shape self.output_name=next(iter(self.model.outputs)) self.output_shape=self.model.outputs[self.output_name].shape def predict(self, leftEye, rightEye, headPose): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' self.logger.info("Commence prepocessing of inputs and start inferencing") left_eye_image = self.preprocess_input(leftEye) right_eye_image = self.preprocess_input(rightEye) # start asynchronous inference for specified request self.logger.info("Commence sync request") self.net.infer({'head_pose_angles':headPose, 'left_eye_image': left_eye_image, 'right_eye_image': right_eye_image}) # wait for the result if self.net.requests[0].wait(-1) == 0: outputs=self.net.requests[0].outputs[self.output_name] return self.preprocess_output(outputs) def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' image_size=(self.input_shape[3], self.input_shape[2]) image = cv2.resize(image,(image_size)) image = image.transpose((2,0,1)) image = image.reshape(1,*image.shape) return image #raise NotImplementedError def preprocess_output(self, outputs): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' x = outputs[0][0] y = outputs[0][1] return x,y
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): self.net = None self.plugin = None self.input_blob = None self.out_blob = None self.net_plugin = None self.infer_request_handle = None def load_model(self, model, device, input_size, output_size, num_requests, cpu_extension=None, plugin=None): """ Loads a network and an image to the Inference Engine plugin. :param model: .xml file of pre trained model :param cpu_extension: extension for the CPU device :param device: Target device :param input_size: Number of input layers :param output_size: Number of output layers :param num_requests: Index of Infer request value. Limited to device capabilities. :param plugin: Plugin for specified device :return: Shape of input layer """ model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Plugin initialization for specified device # and load extensions library if specified if not plugin: log.info("Initializing plugin for {} device...".format(device)) self.plugin = IECore() else: self.plugin = plugin if cpu_extension and 'CPU' in device: self.plugin.add_extension(cpu_extension, "CPU") # Read IR log.info("Reading IR...") self.net = IENetwork(model=model_xml, weights=model_bin) # print("Network Layers : ") # print(self.net.layers) log.info("Loading IR to the plugin...") if device == "CPU": supported_layers = self.plugin.query_network(self.net, "CPU") not_supported_layers = \ [l for l in self.net.layers.keys() if l not in supported_layers] if len(not_supported_layers) != 0: log.error("Following layers are not supported by " "the plugin for specified device {}:\n {}".format( device, ', '.join(not_supported_layers))) log.error("Please try to specify cpu extensions library path" " in command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) if num_requests == 0: # Loads network read from IR to theplugin self.net_plugin = self.plugin.load_network(network=self.net, device_name=device) else: self.net_plugin = self.plugin.load_network( network=self.net, num_requests=num_requests, device_name=device) # log.error("num_requests != 0") self.input_blob = next(iter(self.net.inputs)) self.out_blob = next(iter(self.net.outputs)) # assert len(self.net.inputs.keys()) == input_size, \ # "Supports only {} input topologies".format(len(self.net.inputs)) # assert len(self.net.outputs) == output_size, \ # "Supports only {} output topologies".format(len(self.net.outputs)) return self.plugin, self.get_input_shape() def get_input_shape(self): """ Gives the shape of the input layer of the network. :return: None """ return self.net.inputs[self.input_blob].shape def performance_counter(self, request_id): """ Queries performance measures per layer to get feedback of what is the most time consuming layer. :param request_id: Index of Infer request value. Limited to device capabilities :return: Performance of the layer """ perf_count = self.net_plugin.requests[request_id].get_perf_counts() return perf_count def exec_net(self, request_id, frame): """ Starts asynchronous inference for specified request. :param request_id: Index of Infer request value. Limited to device capabilities. :param frame: Input image :return: Instance of Executable Network class """ self.infer_request_handle = self.net_plugin.start_async( request_id=request_id, inputs={self.input_blob: frame}) return self.net_plugin def wait(self, request_id): """ Waits for the result to become available. :param request_id: Index of Infer request value. Limited to device capabilities. :return: Timeout value """ wait_process = self.net_plugin.requests[request_id].wait(-1) return wait_process def get_output(self, request_id, output=None): """ Gives a list of results for the output layer of the network. :param request_id: Index of Infer request value. Limited to device capabilities. :param output: Name of the output layer :return: Results for the specified request """ if output: res = self.infer_request_handle.outputs[output] else: res = self.net_plugin.requests[request_id].outputs[self.out_blob] # print("==" * 30) # print(res[0][0]) # print("==" * 30) return res def clean(self): """ Deletes all the instances :return: None """ del self.net_plugin del self.plugin del self.net
class GazeEstimation: ''' Class for the Gaze Estimation Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' instantiating the necessary variables. ''' self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extensions = extensions self.exec_net = None self.input_name = None self.input_shape = None self.output_name = None self.output_shape = None def load_model(self): ''' This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.core = IECore() self.model = self.core.read_network(self.model_structure, self.model_weights) supported_layers = self.core.query_network(network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0 and self.device == 'CPU': log.error( "Unsupported layers found: {}".format(unsupported_layers)) log.error("Check whether extensions are available to add") self.core.add_extension(self.extension, self.device) supported_layers = self.core.query_network(network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.error("ERROR: Unsupported layer issue not yet resolved") exit(1) self.exec_net = self.core.load_network(network=self.model, device_name=self.device, num_requests=1) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape def predict(self, left_eye_image, right_eye_image, head_pose_output): ''' This method is meant for running predictions on the input image. ''' leye_image, reye_image = self.preprocess_input(left_eye_image.copy(), right_eye_image.copy()) result = self.exec_net.infer({ 'left_eye_image': leye_image, 'right_eye_image': reye_image, 'head_pose_angles': head_pose_output }) mouse_coord, gaze_vector = self.preprocess_output( result, head_pose_output) return mouse_coord, gaze_vector def check_model(self): pass def preprocess_input(self, left_eye_image, right_eye_image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' leye_p_image = cv2.resize(left_eye_image, (60, 60)) leye_p_image = leye_p_image.transpose((2, 0, 1)) leye_p_image = leye_p_image.reshape(1, *leye_p_image.shape) reye_p_image = cv2.resize(right_eye_image, (60, 60)) reye_p_image = reye_p_image.transpose((2, 0, 1)) reye_p_image = reye_p_image.reshape(1, *reye_p_image.shape) return leye_p_image, reye_p_image def preprocess_output(self, outputs, head_pose_output): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' gaze_output = outputs[self.output_name][0] roll_value = head_pose_output[2] cos_value = math.cos(roll_value * math.pi / 180.0) sin_value = math.sin(roll_value * math.pi / 180.0) x_gaze = gaze_output[0] * cos_value + gaze_output[1] * sin_value y_gaze = gaze_output[1] * cos_value - gaze_output[0] * sin_value return (x_gaze, y_gaze), gaze_output
def main(): total_timer = Timer(name='total') total_timer.tic() log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 ), "Sample supports topologies only with 1 input" input_name = next(iter(net.inputs.keys())) input_info = net.inputs[input_name] input_info.precision = 'FP32' log.info('input shape: {}'.format(input_info.shape)) # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') assert (len(net.outputs.keys()) == 2 ), "Sample supports topologies only with 2 output" loc_out_name = "797" class_out_name = "741" assert (loc_out_name in net.outputs.keys()) and (class_out_name in net.outputs.keys()) loc_out_info = net.outputs[loc_out_name] class_out_info = net.outputs[class_out_name] loc_out_info.precision = "FP32" class_out_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------- log.info("Loading model to the device") # cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'} ie.set_config({'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}, args.device) ie.set_config({'CPU_BIND_THREAD': 'YES'}, args.device) exec_net = ie.load_network(network=net, device_name=args.device, num_requests=0) infer_requests = exec_net.requests request_queue = InferRequestsQueue(infer_requests) log.info('nreqs: {}, nstream:{}'.format( len(infer_requests), ie.get_config(args.device, 'CPU_THROUGHPUT_STREAMS'))) # --------------------------- 3. Read and preprocess input -------------------------------------------- # ----------------------------------------------------------------------------------------------------- if not os.path.exists(args.result_dir): os.makedirs(args.result_dir) if args.voc_res_file and os.path.exists(args.voc_res_file): os.remove(args.voc_res_file) load_data_timer = Timer(name='load_data') post_process_timer = Timer(name='post_process') adapter = RetinaNetAdapter(input_shape=args.patch_size) # --------------------------- Performing inference ---------------------------------------------------- result_all_images = defaultdict(list) data_loader = DataLoader(args.image_dir, args.strides, args.patch_size) while True: load_data_timer.tic() input_data = data_loader.next() load_data_timer.toc() if input_data == None: break infer_request = request_queue.get_idle_request() if not infer_request: raise Exception('No idle Infer Requests!') if infer_request.cur_meta == None: infer_request.start_async(input_name, input_data) continue # get result post_process_timer.tic() image_name = infer_request.cur_meta['image_name'] x = infer_request.cur_meta['x'] y = infer_request.cur_meta['y'] loc_out = infer_request.request.outputs[loc_out_name][0] class_out = infer_request.request.outputs[class_out_name][0] ## start infer infer_request.start_async(input_name, input_data) ## post-process result = adapter.process(loc_out, class_out) result, _ = nms(result, thresh=0.5, keep_top_k=100) result[:, 0] += x result[:, 1] += y result[:, 2] += x result[:, 3] += y result_all_images[image_name].append(result) post_process_timer.toc() # wait the latest inference executions request_queue.wait_all() post_process_timer.tic() for infer_request in request_queue.requests: # get result image_name = infer_request.cur_meta['image_name'] x = infer_request.cur_meta['x'] y = infer_request.cur_meta['y'] loc_out = infer_request.request.outputs[loc_out_name][0] class_out = infer_request.request.outputs[class_out_name][0] ## post-process result = adapter.process(loc_out, class_out) result, _ = nms(result, thresh=0.5, keep_top_k=100) result[:, 0] += x result[:, 1] += y result[:, 2] += x result[:, 3] += y result_all_images[image_name].append(result) post_process_timer.toc() post_process_timer.tic() ## process total image result for image_name, result_per_image in result_all_images.items(): result_per_image = np.concatenate(result_per_image, axis=0) nms_result, _ = nms(result_per_image, thresh=0.5) voc_format = '{} {:.4f} {} {} {} {}' pos_all = [] voc_all = [] for i in range(nms_result.shape[0]): x = int(nms_result[i, 0]) y = int(nms_result[i, 1]) w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1) h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1) p = float(nms_result[i, 4]) pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p} pos_all.append(pos) if args.voc_res_file: xmin = x ymin = y xmax = int(nms_result[i, 2]) ymax = int(nms_result[i, 3]) voc_str = voc_format.format( os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax) voc_all.append(voc_str) file_name = os.path.splitext(image_name)[0] + '.json' with open(os.path.join(args.result_dir, file_name), 'w') as f: json.dump(pos_all, f) if args.voc_res_file: with open(args.voc_res_file, 'a') as f: for voc_str in voc_all: f.write(voc_str + '\n') post_process_timer.toc() total_timer.toc() # ----------------------------------------------------------------------------------------------------- all_timers = [] # all_timers.extend([create_anchor_timer, # read_img_timer, # preprocess_timer, # infer_timer, # adapter_timer, # patch_img_nms_timer, # whole_img_nms_timer, # add_offset_timer, # write_result_timer, # total_timer]) all_timers.extend([load_data_timer, post_process_timer, total_timer]) for timer in all_timers: log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format( timer.name, timer.avg * 1000, timer.total)) log.info('infer: {:2f}s'.format(request_queue.get_duration_in_seconds())) log.info("Execution successful\n")
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None def load_model(self, model, device="CPU", cpu_extension=None): ''' Load the model given IR files. Defaults to CPU as device for use in the workspace. Synchronous requests made within. ''' model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Initialize the plugin self.plugin = IECore() # Add a CPU extension, if applicable if cpu_extension and "CPU" in device: self.plugin.add_extension(cpu_extension, device) # Read the IR as a IENetwork network = IENetwork(model=model_xml, weights=model_bin) # Get the supported layers of the network supported_layers = self.plugin.query_network(network=network, device_name="CPU") # Check for unsupported layers unsupported_layers = [l for l in network.layers.keys() if l not in supported_layers] if len(unsupported_layers) != 0: print("Unsupported layers found: {}".format(unsupported_layers)) print("Check whether extensions are available to add to IECore.") exit(1) # Load the IENetwork into the plugin self.exec_network = self.plugin.load_network(network, device) # Get the input layer self.input_blob = next(iter(network.inputs)) self.output_blob = next(iter(network.outputs)) # Return the input shape (to determine preprocessing) return network.inputs[self.input_blob].shape def get_input_shape(self): return self.network.inputs[self.input_blob].shape def async_inference(self, image): self.exec_network.start_async(request_id = 0,inputs = {self.input_blob:image}) return def wait(self): status = self.exec_network.requests[0].wait(-1) return status def extract_output(self): #get_output return self.exec_network.requests[0].outputs[self.output_blob]
class HeadPoseEstimationModel: ''' Class for defining the HeadPoseEstimation Model Attributes. ''' def __init__(self, model_name, threshold, device='CPU', extensions=None, async_mode=True, plugin=None): ''' TODO: Use this to set your instance variables. ''' self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.out_shape = None self.exec_network = None self.threshold = threshold self.device = device self.async_mode = async_mode self.infer_request = None self.net_plugin = None self.net = None self.extensions = extensions self.model_xml = model_name def load_model(self, model_xml, cpu_extension=None): ''' TODO: load models ''' self.model_xml = model_name model_bin = os.path.splitext(model_xml)[0] + ".bin" self.device = device self.extensions = extensions # Initializing the plugins self.plugin = IECore() # Add a CPU extension and any neccessary extension if cpu_extension and "CPU" in device: self.plugin.add_extension(cpu_extension, device) # Reading the Intermediate Representation (IR) model as a IENetwork # IENetwork deprecated in 2020 version self.network = self.plugin.read_network(model=model_xml, weights=model_bin) self.check_plugin(self.plugin) ## Check for supported layers supported_layers = self.plugin.query_network(network=self.network, device_name=device) ## check for unsupported layers unsupported_layers = [ l for l in self.network.layers.keys() if l not in self.plugin.get_supported_layers(self.network) ] if len(unsupported_layers) != 0: print("Unsupported layers found: {}".format(unsupported_layers)) print("Please check for supported extensions.") exit(1) # Loading the IENetwork into the plugin self.exec_network = self.plugin.load_network(self.network, device) # Get the input layer self.input_blob = next(iter(self.network.inputs)) self.output_blob = next(iter(self.network.outputs)) self.out_shape = self.network.outputs[self.output_blob].shape logging.info("Model Head Pose Detection output shape printed : ", self.out_shape) return def predict(self, image, width, height, threshold): ''' TODO: You will need to complete this method. To run predictions on the input image. ''' # [1,3,60,60] tally = 0 valuess = None width = image.shape[1] height = image.shape[0] img_frame = self.preprocess_input(image) if self.async_mode: self.exec_network.requests[0].async_infer( inputs={self.input_blob: img_frame}) else: self.exec_network.requests[0].infer( inputs={self.input_blob: img_frame}) if self.exec_network.requests[0].wait(-1) == 0: outputs = self.exec_network.requests[0].outputs person_in_frame, target_gaze = self.preprocess_output( image, outputs) return person_in_frame, target_gaze def get_input_shape(self): ### Return the shape of the input layer ### return self.network.inputs[self.input_blob].shape def preprocess_input(self, image): ''' TODO: You will need to complete this method. preprocessing the input shape ''' # [1,3,60,60] (n, c, h, w) = self.network.inputs[self.input_blob].shape img_frame = cv2.resize(image, (w, h)) img_frame = img_frame.transpose((2, 0, 1)) img_frame = img_frame.reshape((n, c, h, w)) return img_frame def preprocess_output(self, image, outputs, width, height): ''' TODO: You will need to complete this method. Output layer names in Inference Engine format: name: "angle_y_fc", shape: [1, 1] - Estimated yaw (in degrees). name: "angle_p_fc", shape: [1, 1] - Estimated pitch (in degrees). name: "angle_r_fc", shape: [1, 1] - Estimated roll (in degrees). Each output contains one float value (yaw, pitсh, roll). ''' # To Parse head pose detection results # ref: source code: https://knowledge.udacity.com/questions/242566 pitch = outputs["angle_p_fc"][0] yaw = outputs["angle_y_fc"][0] roll = outputs["angle_r_fc"][0] # Draw output if ((yaw > -22.5) & (yaw < 22.5) & (pitch > -22.5) & (pitch < 22.5)): return True, [[yaw, pitch, roll]] else: return False, [[0, 0, 0]] # code source: https://knowledge.udacity.com/questions/171017 def draw_axes(self, img_frame, center_of_face, yaw, pitch, roll): focal_length = 950.0 scale = 50 yaw *= np.pi / 180.0 pitch *= np.pi / 180.0 roll *= np.pi / 180.0 cx = int(center_of_face[0]) cy = int(center_of_face[1]) Rx = np.array([[1, 0, 0], [0, math.cos(pitch), -math.sin(pitch)], [0, math.sin(pitch), math.cos(pitch)]]) Ry = np.array([[math.cos(yaw), 0, -math.sin(yaw)], [0, 1, 0], [math.sin(yaw), 0, math.cos(yaw)]]) Rz = np.array([[math.cos(roll), -math.sin(roll), 0], [math.sin(roll), math.cos(roll), 0], [0, 0, 1]]) # R = np.dot(Rz, Ry, Rx) # ref: https://www.learnopencv.com/rotation-matrix-to-euler-angles/ # R = np.dot(Rz, np.dot(Ry, Rx)) R = Rz @ Ry @ Rx # print(R) camera_matrix = self.build_camera_matrix(center_of_face, focal_length) xaxis = np.array(([1 * scale, 0, 0]), dtype='float32').reshape(3, 1) yaxis = np.array(([0, -1 * scale, 0]), dtype='float32').reshape(3, 1) zaxis = np.array(([0, 0, -1 * scale]), dtype='float32').reshape(3, 1) zaxis1 = np.array(([0, 0, 1 * scale]), dtype='float32').reshape(3, 1) o = np.array(([0, 0, 0]), dtype='float32').reshape(3, 1) o[2] = camera_matrix[0][0] xaxis = np.dot(R, xaxis) + o yaxis = np.dot(R, yaxis) + o zaxis = np.dot(R, zaxis) + o zaxis1 = np.dot(R, zaxis1) + o xp2 = (xaxis[0] / xaxis[2] * camera_matrix[0][0]) + cx yp2 = (xaxis[1] / xaxis[2] * camera_matrix[1][1]) + cy p2 = (int(xp2), int(yp2)) cv2.line(img_frame, (cx, cy), p2, (0, 0, 255), 2) xp2 = (yaxis[0] / yaxis[2] * camera_matrix[0][0]) + cx yp2 = (yaxis[1] / yaxis[2] * camera_matrix[1][1]) + cy p2 = (int(xp2), int(yp2)) cv2.line(img_frame, (cx, cy), p2, (0, 255, 0), 2) xp1 = (zaxis1[0] / zaxis1[2] * camera_matrix[0][0]) + cx yp1 = (zaxis1[1] / zaxis1[2] * camera_matrix[1][1]) + cy p1 = (int(xp1), int(yp1)) xp2 = (zaxis[0] / zaxis[2] * camera_matrix[0][0]) + cx yp2 = (zaxis[1] / zaxis[2] * camera_matrix[1][1]) + cy p2 = (int(xp2), int(yp2)) cv2.line(img_frame, p1, p2, (255, 0, 0), 2) cv2.circle(img_frame, p2, 3, (255, 0, 0), 2) return img_frame # code source: https://knowledge.udacity.com/questions/171017 def build_camera_matrix(self, center_of_face, focal_length): cx = int(center_of_face[0]) cy = int(center_of_face[1]) camera_matrix = np.zeros((3, 3), dtype='float32') camera_matrix[0][0] = focal_length camera_matrix[0][2] = cx camera_matrix[1][1] = focal_length camera_matrix[1][2] = cy camera_matrix[2][2] = 1 return camera_matrix def clean(self): """ deletes all the open instances :return: None """ del self.plugin del self.network del self.exec_network del self.net del self.device
class HeadPoseModel: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU',threshold = 0.6 , extensions=None): self.model_name = model_name self.device = device self.extensions = extensions self.threshold = threshold self.model_structure = self.model_name self.model_weights = os.path.splitext(model_name)[0] + ".bin" self.core = IECore() self.network = self.core.read_network(model=self.model_structure, weights=self.model_weights) self.input_name = next(iter(self.network.inputs)) self.input_shape = self.network.inputs[self.input_name].shape self.output_names = [o for o in self.network.outputs.keys()] self.exec_net = None def load_model(self): supported_layers = self.core.query_network(network=self.network, device_name=self.device) unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers] if len(unsupported_layers)!=0 and self.device=='CPU': print("unsupported layers found:{}".format(unsupported_layers)) if self.extensions: self.core.add_extension(self.extensions, self.device) supported_layers = self.core.query_network(network = self.network, device_name=self.device) unsupported_layers = [l for l in self.network.layers.keys() if l not in supported_layers] if len(unsupported_layers)!=0: print("Error:Unsupported layers still found:{}".format(unsupported_layers)) exit(1) else: print("Error: Exteion layers not found!") exit(1) self.exec_net = self.core.load_network(network=self.network, device_name=self.device,num_requests=1) def predict(self, image): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' image_ = image img_processed = self.preprocess_input(image_) input_dict = {self.input_name:img_processed} outputs = self.exec_net.infer(input_dict) output = self.preprocess_output(outputs) return output def check_model(self): pass def preprocess_input(self, image): image_resized = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) img_processed = np.transpose(np.expand_dims(image_resized,axis=0), (0,3,1,2)) return img_processed def preprocess_output(self, outputs): outputs_ = [] outputs_.append(outputs['angle_p_fc'].tolist()[0][0]) outputs_.append(outputs['angle_r_fc'].tolist()[0][0]) outputs_.append(outputs['angle_y_fc'].tolist()[0][0]) return outputs_
def main(): global args args = build_argparser().parse_args() model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # ------------- 1. Plugin initialization for specified device and load extensions library if specified ------------- log.info("Creating Inference Engine...") ie = IECore() if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, "CPU") # -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) -------------------- log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) print("model loaded") # ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------ if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers] if len(not_supported_layers) != 0: log.error("Following layers are not supported by the plugin for specified device {}:\n {}". format(args.device, ', '.join(not_supported_layers))) log.error("Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) assert len(net.inputs.keys()) == 1, "Sample supports only YOLO V3 based single input topologies" # ---------------------------------------------- 4. Preparing inputs ----------------------------------------------- log.info("Preparing inputs") input_blob = next(iter(net.inputs)) # Defaulf batch_size is 1 net.batch_size = 1 # Read and pre-process input images n, c, h, w = net.inputs[input_blob].shape if args.labels: with open(args.labels, 'r') as f: labels_map = [x.strip() for x in f] else: labels_map = None is_async_mode = True ''' CHG ''' wait_rate = rospy.Rate(1000) global rgb_img_update, depth_img_update, rgb_image, depth_img, bridge, motor_yaw_queue pub_total=rospy.Publisher("/yolo_ros_real_pose/detected_objects",ObjectsRealPose,queue_size = 1) corresponding_img_pub=rospy.Publisher("/yolo_ros_real_pose/img_for_detected_objects",Image,queue_size = 1) pub_time=rospy.Publisher("/yolo_ros_real_pose/detection_time", Float64, queue_size = 1) time_this = Float64() info_this = InfoGather() info_next = InfoGather() ''' END ''' if args.input == "ROS": while not rospy.is_shutdown(): if rgb_img_update == 1 and depth_img_update==1 and len(rgb_image)!=0 and len(depth_img)!=0 and (not motor_yaw_queue.empty() or not motor_yaw_syncronize): frame = rgb_image.copy() depth_this = depth_img.copy() info_this.uav_pose = uav_pose.pose info_this.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough") info_this.img_to_pub.header = time_stamp_header if motor_yaw_syncronize: info_this.motor_yaw = motor_yaw_queue.get() else: info_this.motor_yaw = motor_yaw_corrected print("Time difference="+str(uav_pose.header.stamp.to_sec() - time_stamp_header.stamp.to_sec())) rgb_img_update = 0 depth_img_update = 0 wait_key_code = 1 break else: input_stream = 0 if args.input == "cam" else args.input cap = cv2.VideoCapture(input_stream) number_input_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) number_input_frames = 1 if number_input_frames != -1 and number_input_frames < 0 else number_input_frames wait_key_code = 1 # Number of frames in picture is 1 and this will be read in cycle. Sync mode is default value for this case if number_input_frames != 1: ret, frame = cap.read() else: is_async_mode = False wait_key_code = 0 # ----------------------------------------- 5. Loading model to the plugin ----------------------------------------- log.info("Loading model to the plugin") exec_net = ie.load_network(network=net, num_requests=2, device_name=args.device) cur_request_id = 0 next_request_id = 1 render_time = 0 parsing_time = 0 # ----------------------------------------------- 6. Doing inference ----------------------------------------------- log.info("Starting inference...") print("To close the application, press 'CTRL+C' here or switch to the output window and press ESC key") print("To switch between sync/async modes, press TAB key in the output window") while not rospy.is_shutdown(): # Here is the first asynchronous point: in the Async mode, we capture frame to populate the NEXT infer request # in the regular mode, we capture frame to the CURRENT infer request try: detection_start_time = time() if args.input == "ROS": while not rospy.is_shutdown(): if rgb_img_update == 1 and depth_img_update==1 and (not motor_yaw_queue.empty() or not motor_yaw_syncronize): if len(rgb_image)!=0 and len(depth_img)!=0: if is_async_mode: next_frame = rgb_image.copy() depth_next = depth_img.copy() info_next.uav_pose = uav_pose.pose info_next.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough") info_next.img_to_pub.header = time_stamp_header if motor_yaw_syncronize: info_next.motor_yaw = motor_yaw_queue.get() else: info_next.motor_yaw = motor_yaw_corrected else: frame = rgb_image.copy() depth_this = depth_img.copy() info_this.uav_pose = uav_pose.pose info_this.img_to_pub = bridge.cv2_to_imgmsg(rgb_image, encoding="passthrough") info_this.img_to_pub.header = time_stamp_header if motor_yaw_syncronize: info_this.motor_yaw = motor_yaw_queue.get() else: info_this.motor_yaw = motor_yaw_corrected print("Time difference="+str(uav_pose.header.stamp.to_sec() - time_stamp_header.stamp.to_sec())) rgb_img_update = 0 depth_img_update = 0 break wait_rate.sleep() else: if is_async_mode: ret, next_frame = cap.read() else: ret, frame = cap.read() if not ret: break if is_async_mode: request_id = next_request_id in_frame = cv2.resize(next_frame, (w, h)) else: request_id = cur_request_id in_frame = cv2.resize(frame, (w, h)) # resize input_frame to network size in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) # Start inference start_time = time() exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame}) det_time = time() - start_time # Collecting object detection results objects = list() if exec_net.requests[cur_request_id].wait(-1) == 0: output = exec_net.requests[cur_request_id].outputs start_time = time() for layer_name, out_blob in output.items(): out_blob = out_blob.reshape(net.layers[net.layers[layer_name].parents[0]].shape) layer_params = YoloParams(net.layers[layer_name].params, out_blob.shape[2]) log.info("Layer {} parameters: ".format(layer_name)) layer_params.log_params() objects += parse_yolo_region(out_blob, in_frame.shape[2:], frame.shape[:-1], layer_params, args.prob_threshold) parsing_time = time() - start_time # Filtering overlapping boxes with respect to the --iou_threshold CLI parameter objects = sorted(objects, key=lambda obj : obj['confidence'], reverse=True) for i in range(len(objects)): if objects[i]['confidence'] == 0: continue for j in range(i + 1, len(objects)): if intersection_over_union(objects[i], objects[j]) > args.iou_threshold: objects[j]['confidence'] = 0 # Drawing objects with respect to the --prob_threshold CLI parameter objects = [obj for obj in objects if obj['confidence'] >= args.prob_threshold] if len(objects) and args.raw_output_message: log.info("\nDetected boxes for batch {}:".format(1)) log.info(" Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR ") origin_im_size = frame.shape[:-1] current_total_dect = ObjectsRealPose() for obj in objects: # Validation bbox of detected object if obj['xmax'] > origin_im_size[1] or obj['ymax'] > origin_im_size[0] or obj['xmin'] < 0 or obj['ymin'] < 0: continue color = (int(min(obj['class_id'] * 12.5, 255)), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \ str(obj['class_id']) if args.raw_output_message: log.info( "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ".format(det_label, obj['confidence'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], color)) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText(frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) '''CHG''' x_pos=0 y_pos=0 z_pos=0 info=RealPose() info.label = det_label info.confidence = obj['confidence'] info.pix_lt_x = obj['xmin'] info.pix_lt_y = obj['ymin'] info.pix_rb_x = obj['xmax'] info.pix_rb_y = obj['ymax'] info.head_yaw = info_this.motor_yaw info.local_pose = info_this.uav_pose ### Now calculate the real position if len(depth_this)!=0 and (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])>=0: ### Calculate position here by depth image and camera parameters depth_box_width=obj['xmax']-obj['xmin'] depth_box_height=obj['ymax']-obj['ymin'] delta_rate=0.1 x_box_min=int(obj['xmin'] + depth_box_width*delta_rate) y_box_min=int(obj['ymin'] + depth_box_height*delta_rate) x_box_max=int(obj['xmax'] - depth_box_width*delta_rate) y_box_max=int(obj['ymax'] - depth_box_height*delta_rate) after_width=(depth_box_width*(1-2*delta_rate)) after_height=(depth_box_height*(1-2*delta_rate)) ''' ''' rect = depth_this[y_box_min:y_box_max, x_box_min:x_box_max] # Simulation # rect = depth_this[y_box_min:y_box_max, x_box_min:x_box_max] * 0.001 rect[np.where(rect == 0)] = 99 rect[np.where(rect != rect)] = 99 z_pos = rect.min() x_pos = (0.5 * (obj['xmax'] + obj['xmin']) - cx) * z_pos / fx y_pos = (0.5 * (obj['ymax'] + obj['ymin']) - cy) * z_pos / fy info.x = x_pos info.y = y_pos info.z = z_pos current_total_dect.result.append(info) if len(current_total_dect.result)>0: current_total_dect.header = info_this.img_to_pub.header pub_total.publish(current_total_dect) corresponding_img_pub.publish(info_this.img_to_pub) time_this.data = time() - detection_start_time pub_time.publish(time_this) '''END''' # Draw performance stats over frame inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \ "Inference time: {:.3f} ms".format(det_time * 1e3) render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1e3) async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \ "Async mode is off. Processing request {}".format(cur_request_id) parsing_message = "YOLO parsing time is {:.3f}".format(parsing_time * 1e3) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) start_time = time() cv2.imshow("DetectionResults", frame) # cv2.imshow("depth_this", depth_this) render_time = time() - start_time ''' For next ''' if is_async_mode: cur_request_id, next_request_id = next_request_id, cur_request_id frame = next_frame depth_this = depth_next info_this.uav_pose = info_next.uav_pose info_this.motor_yaw = info_next.motor_yaw info_this.img_to_pub = info_next.img_to_pub info_this.img_to_pub.header = info_next.img_to_pub.header key = cv2.waitKey(wait_key_code) # ESC key if key == 27: break # Tab key if key == 9: exec_net.requests[cur_request_id].wait() is_async_mode = not is_async_mode log.info("Switched to {} mode".format("async" if is_async_mode else "sync")) except: print("An error occuered!!!!!!!") cv2.destroyAllWindows()
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = IECore() self.exec_network = None self.net = None self.input_layer = None def load_model(self, xml_path, num_requests, device_name, cpu_extension): ### TODO: Load the model ### self.net = IENetwork(model=xml_path, weights=xml_path.split('.')[0] + '.bin') ### TODO: Check for supported layers ### supported_layers = self.plugin.query_network(network=self.net, device_name=device_name) unsupported_layers = [ l for l in self.net.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: print("unsupported layers found:{}".format(unsupported_layers)) if not cpu_extension == None: print("Adding cpu_extension") self.plugin.add_extension(cpu_extension, device_name) supported_layers = self.plugin.query_network( network=self.net, device_name=device_name) unsupported_layers = [ l for l in self.net.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: print( "After adding the extension still unsupported layers found" ) exit(1) print("After adding the extension the issue is resolved") else: print("Give the path of cpu extension") exit(1) ### TODO: Add any necessary extensions ### ### TODO: Return the loaded inference plugin ### self.exec_network = self.plugin.load_network(network=self.net, num_requests=num_requests, device_name=device_name) ### Note: You may need to update the function parameters. ### self.input_layer = next(iter(self.net.inputs)) return def get_input_shape(self): ### TODO: Return the shape of the input layer ### return self.net.inputs[self.input_layer].shape def exec_net(self, frame, req_id): ### TODO: Start an asynchronous request ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### self.exec_network.start_async(request_id=req_id, inputs={self.input_layer: frame}) return def wait(self, req_id): ### TODO: Wait for the request to be complete. ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### return self.exec_network.requests[req_id].wait(-1) def get_output(self, req_id): ### TODO: Extract and return the output results ### Note: You may need to update the function parameters. ### out = self.exec_network.requests[req_id].outputs return [out[i] for i in out.keys()]
class PersonDetect: ''' Class for the Person Detection Model. ''' def __init__(self, model_name, device, threshold=0.60): self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.threshold = threshold try: self.model = IENetwork(self.model_structure, self.model_weights) except Exception as e: raise ValueError( "Could not Initialise the network. Have you enterred the correct model path?" ) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape def load_model(self): self.plugin = IECore() self.exec_network = self.plugin.load_network(network=self.model, device_name=self.device) sl = self.plugin.query_network(network=self.model, device_name=self.device) ul = [l for l in self.model.layers.keys() if l not in sl] if len(ul) != 0: print("Unsupported layers found: {}".format(ul)) print("Check whether extensions are available to add to IECore.") exit(1) def predict(self, image, width, height): p_image = self.preprocess_input(image) input_dict = {self.input_name: p_image} self.exec_network.infer(input_dict) results = self.exec_network.requests[0].outputs[self.output_name] coords, image = self.draw_outputs(results, image, width, height) return coords, image def draw_outputs(self, result, image, width, height): coords = [] for box in result[0][0]: a = [] conf = box[2] #output_class = box[1] if conf >= self.threshold: xmin = int(box[3] * width) ymin = int(box[4] * height) xmax = int(box[5] * width) ymax = int(box[6] * height) a.append(xmin) a.append(ymin) a.append(xmax) a.append(ymax) cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2) coords.append(a) return coords, image def preprocess_input(self, image): image = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) image = image.transpose((2, 0, 1)) image = image.reshape(1, *image.shape) return image
class GazeEstimation: """ Class for the Face Detection Model. """ def __init__(self, model_name, device='CPU', extensions=None): self.net = None self.plugin = None self.input_blob = None self.out_blob = None self.exec_net = None self.model_name = model_name self.extensions = extensions self.device = device def load_model(self): """ TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. """ # Fetch XML model model_xml = self.model_name model_bin = os.path.splitext(model_xml)[0] + ".bin" self.plugin = IECore() self.net = IENetwork(model=model_xml, weights=model_bin) # Add CPU extension to self.plugin and check not supported layers if "CPU" in self.device: supported_layers = self.plugin.query_network(self.net, self.device) not_supported_layers = [ layer for layer in self.net.layers.keys() if layer not in supported_layers ] if len(not_supported_layers) != 0 and self.device == 'CPU': print(f"Not supported layers: {not_supported_layers}") logging.error(f"Unsupported layers: {not_supported_layers}") # Load model in network start_time = time.time() self.exec_net = self.plugin.load_network(network=self.net, device_name=self.device, num_requests=1) end_time = time.time() # Obtain blob info from network self.input_blob = next(iter(self.net.inputs)) self.out_blob = next(iter(self.net.outputs)) print(f"Gaze Estimation Model Loading Time: {end_time - start_time}") logging.info( f"Gaze Estimation Model Loading Time: {end_time - start_time}") def predict(self, left_eye, right_eye, head_pose): """ TODO: You will need to complete this method. This method is meant for running predictions on the input image. """ processed_left_eye, processed_right_eye = self.preprocess_input( left_eye, right_eye) # infer image outputs = self.exec_net.infer({ 'left_eye_image': processed_left_eye, 'right_eye_image': processed_right_eye, 'head_pose_angles': head_pose }) (mouse_x, mouse_y), gaze_vector = self.preprocess_output(outputs, head_pose) return (mouse_x, mouse_y), gaze_vector def preprocess_input(self, left_eye, right_eye): """ Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. """ try: left_eye = cv2.resize(left_eye, (60, 60)) right_eye = cv2.resize(right_eye, (60, 60)) preprocessed_left_image = left_eye.transpose((2, 0, 1)) preprocessed_right_image = right_eye.transpose((2, 0, 1)) preprocessed_left_image = preprocessed_left_image.reshape( 1, 3, 60, 60) preprocessed_right_image = preprocessed_right_image.reshape( 1, 3, 60, 60) except: logging.warning( f"eye image is in wrong shape Left Eye:{left_eye.shape} Right Eye :{right_eye.shape}" ) return 0, 0 return preprocessed_left_image, preprocessed_right_image def preprocess_output(self, outputs, head_pose): """ Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. """ gaze_vector = outputs[self.out_blob][0] # HeadPoseEstimation model find angle_r_fc output rollValue = head_pose[2] cosValue = math.cos(rollValue * math.pi / 180.0) sinValue = math.sin(rollValue * math.pi / 180.0) mouse_x = gaze_vector[0] * cosValue + gaze_vector[1] * sinValue mouse_y = -gaze_vector[0] * sinValue + gaze_vector[1] * cosValue # gaze_vec = {'x': outputs['gaze_vector'][0][0].item(), # 'y': outputs['gaze_vector'][0][1].item(), # 'z': outputs['gaze_vector'][0][2].item()} return (mouse_x, mouse_y), gaze_vector
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None def load_model(self, model, device="CPU", cpu_extension=None): ### TODO: Load the model ### ### TODO: Check for supported layers ### ### TODO: Add any necessary extensions ### ### TODO: Return the loaded inference plugin ### ### Note: You may need to update the function parameters. ### model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Initialize the plugin self.plugin = IECore() # Add a CPU extension, if applicable if cpu_extension and "CPU" in device: self.plugin.add_extension(cpu_extension, device) # Read the IR as a IENetwork self.network = IENetwork(model=model_xml, weights=model_bin) supported_layers = self.plugin.query_network(network=self.network, device_name="CPU") unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: print("Unsupported layers found: {}".format(unsupported_layers)) print("Check whether extensions are available to add to IECore.") exit(1) #if num_of_req == 0: #self.exec_network = self.plugin.load(network = self.network) #else: #self.exec.network = self.plugin.load(network = self.mnetwork, #num_of_req = num_of_req) # Load the IENetwork into the plugin self.exec_network = self.plugin.load_network(self.network, device) # Get the input layer self.input_blob = next(iter(self.network.inputs)) self.output_blob = next(iter(self.network.outputs)) return #def preprocessing(input_image, height, weight): #image = cv2.resize(input_image, (height, width)) #image = image.transpose((2,0,1)) #image = image.reshape(1,3,height,width) #return image def get_input_shape(self): ### TODO: Return the shape of the input layer ### return self.network.inputs[self.input_blob].shape def exec_net(self, image, request_id=0): ### TODO: Start an asynchronous request ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### self.exec_network.start_async(request_id=request_id, inputs={self.input_blob: image}) return def wait(self): ### TODO: Wait for the request to be complete. ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### TODO: Extract and return the output results ### Note: You may need to update the function parameters. ### return self.exec_network.requests[0].outputs[self.output_blob]
class FacialLandmarkDetection: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_name = model_name self.device = device self.extensions = extensions self.model_structure = self.model_name self.model_weights = self.model_name.split(".")[0] + '.bin' self.plugin = IECore() ## check if read model without problem self.check_model(self.model_structure, self.model_weights) self.exec_net = None self.input_name = next(iter(self.network.inputs)) self.input_shape = self.network.inputs[self.input_name].shape self.output_names = next(iter(self.network.outputs)) self.output_shape = self.network.outputs[self.output_names].shape ## check supported layer and performence counts reference: # https://gist.github.com/justinshenk/9917891c0433f33967f6e8cd8fcaa49a def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' supported_layers = self.plugin.query_network(network=self.network, device_name=self.device) layers_unsupported = [ ul for ul in self.network.layers.keys() if ul not in supported_layers ] if len(layers_unsupported) != 0 and self.device == 'CPU': print("unsupported layers found: {}".format(layers_unsupported)) if self.extensions != None: print("Adding cpu_extension now") self.plugin.add_extension(self.extensions, self.device) supported_layers = self.plugin.query_network( network=self.network, device_name=self.device) layers_unsupported = [ ul for ul in self.network.layers.keys() if ul not in supported_layers ] if len(layers_unsupported) != 0: print( "Please try again! unsupported layers found after adding the extensions. device {}:\n{}" .format(self.device, ', '.join(layers_unsupported))) print( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") exit(1) print("Problem is resolved after adding the extension!") else: print("Please give the right path of cpu extension!") exit(1) self.exec_net = self.plugin.load_network(network=self.network, device_name=self.device, num_requests=1) def predict(self, image, perf_flag): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' processed_input = self.preprocess_input(image.copy()) outputs = self.exec_net.infer({self.input_name: processed_input}) if perf_flag: self.performance() coords = self.preprocess_output(outputs) # print(image.shape) h, w = image.shape[0], image.shape[1] coords = coords * np.array([w, h, w, h]) # print(coords) # [ 39.50625494 130.00975445 146.11010522 126.54997483] coords = coords.astype( np.int32) #(lefteye_x, lefteye_y, righteye_x, righteye_y) # print(coords) # [ 39 130 146 126] ## left eye moving range leye_xmin, leye_ymin = coords[0] - 20, coords[1] - 20 leye_xmax, leye_ymax = coords[0] + 20, coords[1] + 20 ## right eye moving range reye_xmin, reye_ymin = coords[2] - 20, coords[3] - 20 reye_xmax, reye_ymax = coords[2] + 20, coords[3] + 20 ## draw left and right eye # cv2.rectangle(image,(leye_xmin,leye_ymin),(leye_xmax,leye_ymax),(0,255,0), 1) # cv2.rectangle(image,(reye_xmin,reye_ymin),(reye_xmax,reye_ymax),(0,255,0), 1) # cv2.imshow("Left Right Eyes",image) ## leye_ymin:leye_ymax, leye_xmin:leye_xmax --> left eye heigh, width left_eye_box = image[leye_ymin:leye_ymax, leye_xmin:leye_xmax] ## reye_ymin:reye_ymax, reye_xmin:reye_xmax --> right eye heigh, width right_eye_box = image[reye_ymin:reye_ymax, reye_xmin:reye_xmax] # print(left_eye_box.shape, right_eye_box.shape) # left eye and right eye image ## [left eye box, right eye box] eyes_coords = [[leye_xmin, leye_ymin, leye_xmax, leye_ymax], [reye_xmin, reye_ymin, reye_xmax, reye_ymax]] return left_eye_box, right_eye_box, eyes_coords def check_model(self, model_structure, model_weights): # raise NotImplementedError try: # Reads a network from the IR files and creates an IENetwork, load IR files into their related class, architecture with XML and weights with binary file self.network = self.plugin.read_network(model=model_structure, weights=model_weights) except Exception as e: raise ValueError( "Error occurred during facial_landmarks_detection network initialization." ) ## check supported layer and performence counts reference: # https://gist.github.com/justinshenk/9917891c0433f33967f6e8cd8fcaa49a def performance(self): perf_counts = self.exec_net.requests[0].get_perf_counts() # print('\n', perf_counts) print("\n## Facial landmarks detection model performance:") print("{:<70} {:<15} {:<15} {:<15} {:<10}".format( 'name', 'layer_type', 'exet_type', 'status', 'real_time, us')) for layer, stats in perf_counts.items(): print("{:<70} {:<15} {:<15} {:<15} {:<10}".format( layer, stats['layer_type'], stats['exec_type'], stats['status'], stats['real_time'])) def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' # print(image.shape) # print(image[2][1]) # cv2.imshow('image',image) ## convert RGB to BGR image_cvt = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # print(image_cvt.shape) # (374, 238, 3) # cv2.imshow('cvt',image_cvt) # print('====',image_cvt[2][1]) # print(self.input_shape) # [1, 3, 48, 48] H, W = self.input_shape[2], self.input_shape[3] # print(H, W) # (48, 48) image_resized = cv2.resize(image_cvt, (W, H)) # print(image_resized.shape) # (48, 48, 3) ## (optional) # image_processed = np.transpose(np.expand_dims(image_resized, axis=0), (0,3,1,2)) image = image_resized.transpose((2, 0, 1)) # print(image.shape) # (3, 48, 48) # add 1 dim at very start, then channels then H, W image_processed = image.reshape(1, 3, self.input_shape[2], self.input_shape[3]) # print(image_processed.shape) # (1, 3, 48, 48) return image_processed def preprocess_output(self, outputs): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. The net outputs a blob with the shape: [1, 10], containing a row-vector of 10 floating point values for five landmarks coordinates in the form (x0, y0, x1, y1, ..., x5, y5). All the coordinates are normalized to be in range [0,1]. ''' # print(output) # print(outputs[self.output_names].shape) # (1, 10, 1, 1) # print(outputs[self.output_names][0].shape) # (10, 1, 1) # print(outputs[self.output_names][0]) # print('-----', outputs[self.output_names][0][0]) ## here only need left eye and right eye outs = outputs[self.output_names][0] # print(outs.shape) # print(outs[0][0][0]) # print(outs[0].tolist()) # [[0.37333157658576965]] # print(outs[0].tolist()[0][0]) # [[0.37333157658576965]] # print(type(outs)) # numpy.ndarry leye_x, leye_y = outs[0][0][0], outs[1][0][0] reye_x, reye_y = outs[2][0][0], outs[3][0][0] coords_lr = (leye_x, leye_y, reye_x, reye_y) # print(coords_lr) return coords_lr
class GenericModel: ''' Class for controlling similar model characteristics. ''' def __init__(self, device): self.device = device self.net = None self.plugin = None self.input_blob = None self.out_blob = None self.net_plugin = None self.infer_request_handle = None self.outputs = None def load_model(self, model, cpu_extension=None, plugin=None): # Obtain model files path: model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" if not plugin: self.plugin = IECore() else: self.plugin = plugin if cpu_extension and 'CPU' in self.device: self.plugin.add_extension(cpu_extension, "CPU") self.net = IENetwork(model=model_xml, weights=model_bin) # If applicable, add a CPU extension to self.plugin if "CPU" in self.device: supported_layers = self.plugin.query_network(self.net, "CPU") not_supported_layers = [ layer for layer in self.net.layers.keys() if layer not in supported_layers ] if len(not_supported_layers) != 0: sys.exit(1) # Load the model to the network: self.net_plugin = self.plugin.load_network(network=self.net, device_name=self.device) # Obtain other relevant information: self.input_blob = next(iter(self.net.inputs)) self.out_blob = next(iter(self.net.outputs)) return self.plugin def predict(self, image, request_id=0): preprocessed_image = self.preprocess_input(image) self.infer_request_handle = self.net_plugin.start_async( request_id=request_id, inputs={self.input_blob: preprocessed_image}) return self.net_plugin def check_model(self): pass def preprocess_input(self, image): input_shape = self.net.inputs[self.input_blob].shape preprocessed_image = helpers.handle_image(image, input_shape[3], input_shape[2]) return preprocessed_image def wait(self, request_id=0): status = self.net_plugin.requests[request_id].wait(-1) return status def get_output(self, request_id=0): self.outputs = self.net_plugin.requests[request_id].outputs[ self.out_blob] return self.outputs
class Model_X(ABC): """ Abstract Class for the Face Detection Model. """ def __init__(self, model_name, device="CPU", threshold=0.5, extensions=None): self.model_weights = model_name + ".bin" self.model_structure = model_name + ".xml" self.device = device self.threshold = threshold self.extensions = extensions @timeit def load_model(self): """ This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. """ self.network = IENetwork(self.model_structure, self.model_weights) self.input_name = next(iter(self.network.inputs)) self.input_shape = self.network.inputs[self.input_name].shape self.output_name = next(iter(self.network.outputs)) self.output_shape = self.network.outputs[self.output_name].shape self.core = IECore() if self.extensions and "CPU" in self.device: self.core.add_extension(self.extensions, self.device) supported_layers = self.core.query_network( network=self.network, device_name=self.device ) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if unsupported_layers: log.error(f"Unsupported layers found: {unsupported_layers}") exit(1) self.exec_network = self.core.load_network(self.network, self.device) @timeit def predict(self, image): """ This abstractmethod is meant for running predictions on the input image. """ outputs = self.exec_network.infer( {self.input_name: self.preprocess_input(image)} ) return self.preprocess_output(outputs, image) def check_model(self): raise NotImplementedError def preprocess_input(self, image): """ Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. """ ret = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) ret = ret.transpose((2, 0, 1)) ret = ret.reshape(1, *ret.shape) return ret @abstractmethod def preprocess_output(self, outputs): """ Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. """ pass
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 3. Read and preprocess input -------------------------------------------- input_blob = next(iter(net.inputs)) n, c, h, w = net.inputs[input_blob].shape images = np.ndarray(shape=(n, c, h, w)) images_hw = [] for i in range(n): image = cv2.imread(args.input[i]) ih, iw = image.shape[:-1] images_hw.append((ih, iw)) log.info("File was added: ") log.info(" {}".format(args.input[i])) if (ih, iw) != (h, w): image = cv2.resize(image, (w, h)) log.warning("Image {} is resized from {} to {}".format( args.input[i], image.shape[:-1], (h, w))) image = image.transpose( (2, 0, 1)) # Change data layout from HWC to CHW images[i] = image # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 or len(net.inputs.keys()) == 2), "Sample supports topologies only with 1 or 2 inputs" input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) input_name, input_info_name = "", "" for input_key in net.inputs: if len(net.inputs[input_key].layout) == 4: input_name = input_key log.info("Batch size is {}".format(net.batch_size)) net.inputs[input_key].precision = 'U8' elif len(net.inputs[input_key].layout) == 2: input_info_name = input_key net.inputs[input_key].precision = 'FP32' if net.inputs[input_key].shape[1] != 3 and net.inputs[ input_key].shape[1] != 6 or net.inputs[input_key].shape[ 0] != 1: log.error( 'Invalid input info. Should be 3 or 6 values length.') # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))] for output_key in net.outputs: if net.layers[output_key].type == "DetectionOutput": output_name, output_info = output_key, net.outputs[output_key] if output_name == "": log.error("Can't find a DetectionOutput layer in the topology") output_dims = output_info.shape if len(output_dims) != 4: log.error("Incorrect output dimensions for SSD model") max_proposal_count, object_size = output_dims[2], output_dims[3] if object_size != 7: log.error("Output item should have 7 as a last dimension") output_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # --------------------------- Performing inference ---------------------------------------------------- log.info("Loading model to the device") exec_net = ie.load_network(network=net, device_name=args.device) log.info("Creating infer request and starting inference") res = exec_net.infer(inputs={input_blob: images}) # ----------------------------------------------------------------------------------------------------- # --------------------------- Read and postprocess output --------------------------------------------- log.info("Processing output blobs") res = res[out_blob] boxes, classes = {}, {} data = res[0][0] for number, proposal in enumerate(data): if proposal[2] > 0: imid = np.int(proposal[0]) ih, iw = images_hw[imid] label = np.int(proposal[1]) confidence = proposal[2] xmin = np.int(iw * proposal[3]) ymin = np.int(ih * proposal[4]) xmax = np.int(iw * proposal[5]) ymax = np.int(ih * proposal[6]) print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}"\ .format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="") if proposal[2] > 0.5: print(" WILL BE PRINTED!") if not imid in boxes.keys(): boxes[imid] = [] boxes[imid].append([xmin, ymin, xmax, ymax]) if not imid in classes.keys(): classes[imid] = [] classes[imid].append(label) else: print() for imid in classes: tmp_image = cv2.imread(args.input[imid]) for box in boxes[imid]: cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]), (232, 35, 244), 2) cv2.imwrite("out.bmp", tmp_image) log.info("Image out.bmp created!") # ----------------------------------------------------------------------------------------------------- log.info("Execution successful\n") log.info( "This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool" )
def main(): args = build_argparser().parse_args() # ------------- 1. Plugin initialization for specified device and load extensions library if specified ------------- log.info("Creating Inference Engine...") ie = IECore() config_user_specified = {} config_min_latency = {} devices_nstreams = {} if args.num_streams: devices_nstreams = {device: args.num_streams for device in ['CPU', 'GPU'] if device in args.device} \ if args.num_streams.isdigit() \ else dict([device.split(':') for device in args.num_streams.split(',')]) if 'CPU' in args.device: if args.cpu_extension: ie.add_extension(args.cpu_extension, 'CPU') if args.number_threads is not None: config_user_specified['CPU_THREADS_NUM'] = str(args.number_threads) if 'CPU' in devices_nstreams: config_user_specified['CPU_THROUGHPUT_STREAMS'] = devices_nstreams['CPU'] \ if int(devices_nstreams['CPU']) > 0 \ else 'CPU_THROUGHPUT_AUTO' config_min_latency['CPU_THROUGHPUT_STREAMS'] = '1' if 'GPU' in args.device: if 'GPU' in devices_nstreams: config_user_specified['GPU_THROUGHPUT_STREAMS'] = devices_nstreams['GPU'] \ if int(devices_nstreams['GPU']) > 0 \ else 'GPU_THROUGHPUT_AUTO' config_min_latency['GPU_THROUGHPUT_STREAMS'] = '1' # -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) -------------------- log.info("Loading network") net = ie.read_network(args.model, os.path.splitext(args.model)[0] + ".bin") # ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------ if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) assert len( net.input_info ) == 1, "Sample supports only YOLO V3 based single input topologies" # ---------------------------------------------- 4. Preparing inputs ----------------------------------------------- log.info("Preparing inputs") input_blob = next(iter(net.input_info)) # Read and pre-process input images if net.input_info[input_blob].input_data.shape[1] == 3: input_height, input_width = net.input_info[ input_blob].input_data.shape[2:] nchw_shape = True else: input_height, input_width = net.input_info[ input_blob].input_data.shape[1:3] nchw_shape = False if args.labels: with open(args.labels, 'r') as f: labels_map = [x.strip() for x in f] else: labels_map = None input_stream = 0 if args.input == "cam" else args.input mode = Mode(Modes.USER_SPECIFIED) cap = cv2.VideoCapture(input_stream) wait_key_time = 1 # ----------------------------------------- 5. Loading model to the plugin ----------------------------------------- log.info("Loading model to the plugin") exec_nets = {} exec_nets[Modes.USER_SPECIFIED] = ie.load_network( network=net, device_name=args.device, config=config_user_specified, num_requests=args.num_infer_requests) exec_nets[Modes.MIN_LATENCY] = ie.load_network( network=net, device_name=args.device.split(":")[-1].split(",")[0], config=config_min_latency, num_requests=1) empty_requests = deque(exec_nets[mode.current].requests) completed_request_results = {} next_frame_id = 0 next_frame_id_to_show = 0 mode_info = {mode.current: ModeInfo()} event = threading.Event() callback_exceptions = [] # ----------------------------------------------- 6. Doing inference ----------------------------------------------- log.info("Starting inference...") print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) print( "To switch between min_latency/user_specified modes, press TAB key in the output window" ) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(cap.get(cv2.CAP_PROP_FRAME_WIDTH) / 4), round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) / 8))) while (cap.isOpened() \ or completed_request_results \ or len(empty_requests) < len(exec_nets[mode.current].requests)) \ and not callback_exceptions: if next_frame_id_to_show in completed_request_results: frame, output, start_time, is_same_mode = completed_request_results.pop( next_frame_id_to_show) next_frame_id_to_show += 1 if is_same_mode: mode_info[mode.current].frames_count += 1 objects = get_objects(output, net, (input_height, input_width), frame.shape[:-1], args.prob_threshold, args.keep_aspect_ratio) objects = filter_objects(objects, args.iou_threshold, args.prob_threshold) if len(objects) and args.raw_output_message: log.info( " Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR " ) origin_im_size = frame.shape[:-1] presenter.drawGraphs(frame) for obj in objects: # Validation bbox of detected object obj['xmax'] = min(obj['xmax'], origin_im_size[1]) obj['ymax'] = min(obj['ymax'], origin_im_size[0]) obj['xmin'] = max(obj['xmin'], 0) obj['ymin'] = max(obj['ymin'], 0) color = (min(obj['class_id'] * 12.5, 255), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \ str(obj['class_id']) if args.raw_output_message: log.info( "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ". format(det_label, obj['confidence'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], color)) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText( frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) # Draw performance stats over frame if mode_info[mode.current].frames_count != 0: fps_message = "FPS: {:.1f}".format(mode_info[mode.current].frames_count / \ (perf_counter() - mode_info[mode.current].last_start_time)) mode_info[ mode.current].latency_sum += perf_counter() - start_time latency_message = "Latency: {:.1f} ms".format((mode_info[mode.current].latency_sum / \ mode_info[mode.current].frames_count) * 1e3) put_highlighted_text(frame, fps_message, (15, 20), cv2.FONT_HERSHEY_COMPLEX, 0.75, (200, 10, 10), 2) put_highlighted_text(frame, latency_message, (15, 50), cv2.FONT_HERSHEY_COMPLEX, 0.75, (200, 10, 10), 2) mode_message = "{} mode".format(mode.current.name) put_highlighted_text(frame, mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.75, (10, 10, 200), 2) if not args.no_show: cv2.imshow("Detection Results", frame) #print(frame.shape) cv2.imwrite("prob_threshold-0.2.png", frame) key = cv2.waitKey(wait_key_time) if key in {ord("q"), ord("Q"), 27}: # ESC key break if key == 9: # Tab key prev_mode = mode.current mode.next() await_requests_completion(exec_nets[prev_mode].requests) empty_requests.clear() empty_requests.extend(exec_nets[mode.current].requests) mode_info[prev_mode].last_end_time = perf_counter() mode_info[mode.current] = ModeInfo() else: presenter.handleKey(key) elif empty_requests and cap.isOpened(): start_time = perf_counter() ret, frame = cap.read() if not ret: if args.loop_input: cap.open(input_stream) else: cap.release() continue request = empty_requests.popleft() # resize input_frame to network size in_frame = preprocess_frame(frame, input_height, input_width, nchw_shape, args.keep_aspect_ratio) # Start inference request.set_completion_callback( py_callback=async_callback, py_data=(request, next_frame_id, mode.current, frame, start_time, completed_request_results, empty_requests, mode, event, callback_exceptions)) request.async_infer(inputs={input_blob: in_frame}) next_frame_id += 1 else: event.wait() if callback_exceptions: raise callback_exceptions[0] for mode_value in mode_info.keys(): log.info("") log.info("Mode: {}".format(mode_value.name)) end_time = mode_info[mode_value].last_end_time if mode_value in mode_info \ and mode_info[mode_value].last_end_time is not None \ else perf_counter() log.info("FPS: {:.1f}".format(mode_info[mode_value].frames_count / \ (end_time - mode_info[mode_value].last_start_time))) log.info("Latency: {:.1f} ms".format((mode_info[mode_value].latency_sum / \ mode_info[mode_value].frames_count) * 1e3)) print(presenter.reportMeans()) for exec_net in exec_nets.values(): await_requests_completion(exec_net.requests)
class Model_Face_Detect: def __init__(self, model_name, device, extensions=None): self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extensions = extensions try: self.core = IECore() self.network = IENetwork(model=self.model_structure, weights=self.model_weights) network_layers = self.network.layers.keys() supported_layers = self.core.query_network( network=self.network, device_name=self.device).keys() for layer in network_layers: if layer in supported_layers: pass else: ext_required = True break if self.extensions != None and "CPU" in self.device and ext_required: self.core.add_extension(self.extensions, self.device) for layer in network_layers: if layer in supported_layers: pass else: msg = "Layer extension doesn't support all layers" log.error(msg) raise Exception(msg) except Exception as e: raise ValueError( "Could not Initialise the network. Have you enterred the correct model path?" ) self.input_name = next(iter(self.network.inputs)) self.input_shape = self.network.inputs[self.input_name].shape self.output_name = next(iter(self.network.outputs)) self.output_shape = self.network.outputs[self.output_name].shape def load_model(self): self.exec_network = self.core.load_network(self.network, self.device) return def predict(self, image, threshold): input_img = self.preprocess_input(image) input_dict = {self.input_name: input_img} outputs = self.exec_network.infer(input_dict)[self.output_name] coords = self.preprocess_outputs(outputs, threshold, (image.shape[1], image.shape[0])) return self.face_coord(coords, image) def face_coord(self, coords, image): if (len(coords) == 1): coords = coords[0] face_coords = image[coords[1]:coords[3], coords[0]:coords[2]] return coords, face_coords return False, False def preprocess_input(self, image): preprocessed_frame = cv2.resize( image, (self.input_shape[3], self.input_shape[2])) preprocessed_frame = preprocessed_frame.transpose((2, 0, 1)) return preprocessed_frame.reshape(1, *preprocessed_frame.shape) def preprocess_outputs(self, outputs, threshold, dim): _ = [] for box in outputs[0][0]: ct = box[2] if ct > threshold: xmin = int(box[3] * dim[0]) ymin = int(box[4] * dim[1]) xmax = int(box[5] * dim[0]) ymax = int(box[6] * dim[1]) _.append([xmin, ymin, xmax, ymax]) return _
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = None self.network = None self.network_inputs = None self.network_outputs = None self.exec_network = None self.infer_request = None def load_model(self, model, device='CPU', cpu_extension=None): ### TODO: Load the model ### self.plugin = IECore() model_bin_path = os.path.splitext(model)[0] + ".bin" self.network = IENetwork(model=model, weights=model_bin_path) ### TODO: Check for supported layers ### layers_supported = self.plugin.query_network(self.network, device_name=device) layers = self.network.layers.keys() for layer in layers: if layer not in layers_supported: ### TODO: Add any necessary extensions ### self.plugin.add_extension(cpu_extension, device) break self.exec_network = self.plugin.load_network(self.network, device) self.network_inputs = next(iter(self.network.inputs)) self.network_outputs = next(iter(self.network.outputs)) ### TODO: Return the loaded inference plugin ### ### Note: You may need to update the function parameters. ### return def get_input_shape(self): ### TODO: Return the shape of the input layer ### return self.network.inputs[self.network_inputs].shape def exec_net(self, request_id, net_input): ### TODO: Start an asynchronous request ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### self.infer_request_handle = self.exec_network.start_async( request_id, inputs={self.network_inputs: net_input}) return def wait(self): ### TODO: Wait for the request to be complete. ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### return self.infer_request_handle.wait() def get_output(self): ### TODO: Extract and return the output results ### Note: You may need to update the function parameters. ### return self.infer_request_handle.outputs[self.network_outputs]