def load_model(self): core = IECore() if self.extensions != None: core.add_extension(self.extensions, self.device) self.net = core.load_network(network=self.model, device_name=self.device, num_requests=1)
class face_detection: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', threshold=0.5, extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.threshold = threshold self.extension = extensions core = IECore() self.model = core.read_network(self.model_structure, self.model_weights) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape #raise NotImplementedError def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() if self.extension and self.device == 'CPU': self.plugin.add_extension(self.extension, self.device) # Check for supported layers ### supported_layers = self.plugin.query_network( network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: logger.error( "Unsupported layers found: {}".format(unsupported_layers)) logger.error( "Check whether extensions are available to add to IECore.") exit(1) self.net = self.plugin.load_network(network=self.model, device_name=self.device, num_requests=1) return self.net #raise NotImplementedError def predict(self, image): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' preprocess_image = self.preprocess_input(image) results = self.net.infer({self.input_name: preprocess_image}) coords = self.preprocess_output(results, image.shape[1], image.shape[0]) if len(coords) > 0: coord = coords[0] # retrieve first face image cropped_image = image[coord[1]:coord[3], coord[0]:coord[2]] return cropped_image, coord #raise NotImplementedError def check_model(self): raise NotImplementedError def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' image = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) image = image.transpose((2, 0, 1)) image = image.reshape(1, *image.shape) return image #raise NotImplementedError def preprocess_output(self, outputs, width, height): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' coords = [] output = outputs['detection_out'] for box in output[0][0]: if box[2] > self.threshold: #box_lst.append(box) xmin = int(box[3] * width) ymin = int(box[4] * height) xmax = int(box[5] * width) ymax = int(box[6] * height) coords.append([xmin, ymin, xmax, ymax]) return coords
class facial_landmarks_detection: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extension = extensions core = IECore() self.model = core.read_network(self.model_structure, self.model_weights) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape #raise NotImplementedError def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() if self.extension and self.device == 'CPU': self.plugin.add_extension(self.extension, self.device) # Check for supported layers ### supported_layers = self.plugin.query_network( network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: logger.error( "Unsupported layers found: {}".format(unsupported_layers)) logger.error( "Check whether extensions are available to add to IECore.") exit(1) self.net = self.plugin.load_network(network=self.model, device_name=self.device, num_requests=1) return self.net #raise NotImplementedError def predict(self, image): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' preprocess_image = self.preprocess_input(image) results = self.net.infer({self.input_name: preprocess_image}) eye_coords = self.preprocess_output(results, image.shape[1], image.shape[0]) lefteye_x_min = eye_coords['left_eye_x_coord'] - 10 lefteye_x_max = eye_coords['left_eye_x_coord'] + 10 lefteye_y_min = eye_coords['left_eye_y_coord'] - 10 lefteye_y_max = eye_coords['left_eye_y_coord'] + 10 righteye_x_min = eye_coords['right_eye_x_coord'] - 10 righteye_x_max = eye_coords['right_eye_x_coord'] + 10 righteye_y_min = eye_coords['right_eye_y_coord'] - 10 righteye_y_max = eye_coords['right_eye_y_coord'] + 10 eye_coord = [[ lefteye_x_min, lefteye_y_min, lefteye_x_max, lefteye_y_max ], [righteye_x_min, righteye_y_min, righteye_x_max, righteye_y_max]] left_eye_image = image[lefteye_x_min:lefteye_x_max, lefteye_y_min:lefteye_y_max] right_eye_image = image[righteye_x_min:righteye_x_max, righteye_y_min:righteye_y_max] return left_eye_image, right_eye_image, eye_coord #raise NotImplementedError def check_model(self): raise NotImplementedError def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' image = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) image = image.transpose(2, 0, 1) image = image.reshape(1, *image.shape) return image #raise NotImplementedError def preprocess_output(self, outputs, width, height): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' eye_coords = {} outputs = outputs[self.output_name][0] eye_coords['left_eye_x_coord'] = int(outputs[0] * width) eye_coords['left_eye_y_coord'] = int(outputs[1] * height) eye_coords['right_eye_x_coord'] = int(outputs[2] * width) eye_coords['right_eye_y_coord'] = int(outputs[3] * height) return eye_coords
class head_pose_estimation: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extension = extensions core = IECore() self.model = core.read_network(self.model_structure, self.model_weights) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape #raise NotImplementedError def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() if self.extension and self.device == 'CPU': self.plugin.add_extension(self.extension, self.device) # Check for supported layers ### supported_layers = self.plugin.query_network( network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: logger.error( "Unsupported layers found: {}".format(unsupported_layers)) logger.error( "Check whether extensions are available to add to IECore.") exit(1) self.net = self.plugin.load_network(network=self.model, device_name=self.device, num_requests=1) return self.net #raise NotImplementedError def predict(self, image): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' preprocess_image = self.preprocess_input(image) results = self.net.infer({self.input_name: preprocess_image}) output_lst = self.preprocess_output(results) return output_lst #raise NotImplementedError def check_model(self): raise NotImplementedError def preprocess_input(self, image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' image = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) image = image.transpose((2, 0, 1)) image = image.reshape(1, *image.shape) return image #raise NotImplementedError def preprocess_output(self, outputs): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' output_lst = [] output_lst.append(outputs['angle_y_fc'].tolist()[0][0]) output_lst.append(outputs['angle_p_fc'].tolist()[0][0]) output_lst.append(outputs['angle_r_fc'].tolist()[0][0]) return output_lst
class BaseModel: def __init__(self, model_name, device='CPU', extensions=None): """ Initialize :param model_name: model path :param device: device to use :param extensions: extensions """ # model_weights self.model_bin = model_name + ".bin" # model_structure self.model_xml = model_name + ".xml" # device to use self.device = device self.plugin = IECore() self.network = None self.net_input = None # extensions to use self.extensions = extensions # Get the input layer self.input_blob = None self.input_shape = None self.output_blob = None self.exec_network = None self.infer_request = None # name of the model extended self.model_name = None def load_model(self): """ To load the model to the specified hardware :return:None """ try: log.info("Loading {} IR to the plugin.".format(self.model_name)) self.network = IENetwork(model=self.model_xml, weights=self.model_bin) self.input_blob = next(iter(self.network.inputs)) self.input_shape = self.network.inputs[self.input_blob].shape self.output_blob = next(iter(self.network.outputs)) # check model for unsupported layers self.check_model() # Load the network into the Inference Engine self.exec_network = self.plugin.load_network( self.network, self.device) except Exception as e: log.error("The loading of the model cannot be completed!".format( self.model_name)) log.error("Exception message during {} load : {}".format( self.model_name, e)) def check_model(self): """ To check the model for unsupported layers and apply necessary extensions :return:None """ # Check for supported layers supported_layers = self.plugin.query_network(network=self.network, device_name=self.device) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.debug("Unsupported network layers found!") # Add necessary extensions self.plugin.add_extension(self.extensions, self.device) def preprocess_input(self, image): """ To preprocess the input for the model. :param image: input frame :return: transformed input frame """ # Pre-process the frame if self.model_name == "Gaze estimation Model": image = cv2.resize(image, (60, 60)) else: image = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) # Change format from HWC to CHW image_to_infer = image.transpose((2, 0, 1)) # prepare according to face_detection model image_to_infer = image_to_infer.reshape(1, *image_to_infer.shape) return image_to_infer def predict(self): """ Perform the inference request. :return:None """ # make a infer request self.infer_request = self.exec_network.start_async( 0, inputs=self.net_input) def wait(self): """ Wait for the request to be complete. :return:status of the inference request """ status = self.exec_network.requests[0].wait(-1) return status def get_network_input_shape(self): """ To get the network input keys and corresponding shape :return:None """ input_name = [i for i in self.network.inputs.keys()] log.debug("The network {} input name : {}".format( self.model_name, input_name)) for name in input_name: input_shape = self.network.inputs[name].shape log.debug("The input shape for {} is {}".format(name, input_shape)) def get_network_output_shape(self): """ To get the network output keys :return:None """ output_name = [i for i in self.network.outputs.keys()] log.debug("The network {} output name : {}".format( self.model_name, output_name)) @abstractmethod def set_net_input(self): """ Prepare the network input according to the model specification :return: None """ pass
class gaze_estimation: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model_weights = model_name + '.bin' self.model_structure = model_name + '.xml' self.device = device self.extension = extensions core = IECore() self.model = core.read_network(self.model_structure, self.model_weights) self.input_name = next(iter(self.model.inputs)) self.input_shape = self.model.inputs[self.input_name].shape self.output_name = next(iter(self.model.outputs)) self.output_shape = self.model.outputs[self.output_name].shape #raise NotImplementedError def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() if self.extension and self.device == 'CPU': self.plugin.add_extension(self.extension, self.device) # Check for supported layers ### supported_layers = self.plugin.query_network( network=self.model, device_name=self.device) unsupported_layers = [ l for l in self.model.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: logger.error( "Unsupported layers found: {}".format(unsupported_layers)) logger.error( "Check whether extensions are available to add to IECore.") exit(1) self.net = self.plugin.load_network(network=self.model, device_name=self.device, num_requests=1) return self.net #raise NotImplementedError def predict(self, left_eye_image, right_eye_image, head_pose_output): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image. ''' left_eye_image_preprocess, right_eye_image_preprocess = self.preprocess_input( left_eye_image, right_eye_image) self.results = self.net.infer( inputs={ 'left_eye_image': left_eye_image_preprocess, 'right_eye_image': right_eye_image_preprocess, 'head_pose_angles': head_pose_output }) mouse_coord, gaze_vector = self.preprocess_output( self.results, head_pose_output) return mouse_coord, gaze_vector #raise NotImplementedError def check_model(self): raise NotImplementedError def preprocess_input(self, left_eye_image, right_eye_image): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' left_eye_image_preprocess = cv2.resize(left_eye_image, (60, 60)) left_eye_image_preprocess = left_eye_image_preprocess.transpose( (2, 0, 1)) left_eye_image_preprocess = left_eye_image_preprocess.reshape( 1, *left_eye_image_preprocess.shape) right_eye_image_preprocess = cv2.resize(right_eye_image, (60, 60)) right_eye_image_preprocess = right_eye_image_preprocess.transpose( (2, 0, 1)) right_eye_image_preprocess = right_eye_image_preprocess.reshape( 1, *right_eye_image_preprocess.shape) return left_eye_image_preprocess, right_eye_image_preprocess #raise NotImplementedError def preprocess_output(self, outputs, head_pose_estimation_output): ''' Before feeding the output of this model to the next model, you might have to preprocess the output. This function is where you can do that. ''' roll_value = head_pose_estimation_output[2] outputs = outputs[self.output_name][0] cos_theta = math.cos(roll_value * math.pi / 180) sin_theta = math.sin(roll_value * math.pi / 180) x_value = outputs[0] * cos_theta + outputs[1] * sin_theta y_value = outputs[1] * cos_theta - outputs[0] * sin_theta return (x_value, y_value), outputs