def test_load_openvino(self): local_path = self.create_temp_dir() model = InferenceModel(1) model_url = data_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.xml" weight_url = data_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.bin" model_path = maybe_download("resnet_v1_50.xml", local_path, model_url) weight_path = maybe_download("resnet_v1_50.bin", local_path, weight_url) model.load_openvino(model_path, weight_path) input_data = np.random.random([4, 1, 224, 224, 3]) model.predict(input_data)
def test_load_openvino(self): local_path = self.create_temp_dir() url = data_url + "/IR_faster_rcnn_resnet101_coco_2018_01_28" maybe_download("frozen_inference_graph.xml", local_path, url + "/frozen_inference_graph.xml") maybe_download("frozen_inference_graph.bin", local_path, url + "/frozen_inference_graph.bin") model = InferenceModel() model.load_openvino(local_path + "/frozen_inference_graph.xml", local_path + "/frozen_inference_graph.bin") input_data = np.random.random([1, 1, 3, 600, 600]) output_data = model.predict(input_data)
def predict(model_path, img_path): model = InferenceModel() model.load_openvino(model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin", batch_size=BATCH_SIZE) sc = init_nncontext("OpenVINO Python resnet_v1_50 Inference Example") # pre-processing infer_transformer = ChainedPreprocessing([ ImageBytesToMat(), ImageResize(256, 256), ImageCenterCrop(224, 224), ImageMatToTensor(format="NHWC", to_RGB=True) ]) image_set = ImageSet.read(img_path, sc).\ transform(infer_transformer).get_image().collect() image_set = np.expand_dims(image_set, axis=1) for i in range(len(image_set) // BATCH_SIZE + 1): index = i * BATCH_SIZE # check whether out of index if index >= len(image_set): break batch = image_set[index] # put 4 images in one batch for j in range(index + 1, min(index + BATCH_SIZE, len(image_set))): batch = np.vstack((batch, image_set[j])) batch = np.expand_dims(batch, axis=0) # predict batch predictions = model.predict(batch) result = predictions[0] # post-processing for Top-1 print("batch_" + str(i)) for r in result: output = {} max_index = np.argmax(r) output["Top-1"] = str(max_index) print("* Predict result " + str(output)) print("finished...") sc.stop()
def classify_process(model_path): # load the pre-trained OpenVINO model print("* Loading model...") model = InferenceModel() model.load_openvino(model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin") print("* Model loaded") # continually pool for new images to classify while True: # attempt to grab a batch of images from the database, then # initialize the image IDs and batch of images themselves queue = DB.lrange(settings.IMAGE_QUEUE, 0, settings.BATCH_SIZE - 1) image_ids = [] batch = None # loop over the queue start_time = time.time() for q in queue: # deserialize the object and obtain the input image q = json.loads(q.decode("utf-8")) image = helpers.base64_decode_image(q["image"], settings.IMAGE_DTYPE) # check to see if the batch list is None if batch is None: batch = image # otherwise, stack the data else: batch = np.vstack([batch, image]) # update the list of image IDs image_ids.append(q["id"]) print("* Pop from redis %d ms" % int(round((time.time() - start_time) * 1000))) # check to see if we need to process the batch if len(image_ids) > 0: # classify the batch batch = np.expand_dims(batch, axis=0) print("* Batch size: {}".format(batch.shape)) # Output is [1, 4, 1000] results = model.predict(batch)[0] print("* Predict a batch %d ms" % int(round((time.time() - start_time) * 1000))) # loop over the image IDs and their corresponding set of # results from our model for (imageID, resultSet) in zip(image_ids, results): # initialize the list of output predictions output = {} # loop over the results and add them to the list of # output predictions # Top 1 max_index = np.argmax(resultSet) output["Top-1"] = str(max_index) output["id"] = imageID print("* Predict result " + str(output)) # store the output predictions in the database, using # the image ID as the key so we can fetch the results DB.lpush(settings.PREDICT_QUEUE, json.dumps(output)) # remove the set of images from our queue print("* Total time used is %d ms" % int(round((time.time() - start_time) * 1000))) DB.ltrim(settings.IMAGE_QUEUE, len(image_ids), -1) # sleep for a small amount time.sleep(settings.SERVER_SLEEP)
if __name__ == "__main__": parser = OptionParser() parser.add_option("--image", type=str, dest="img_path", help="The path where the images are stored, " "can be either a folder or an image path") parser.add_option("--model", type=str, dest="model_path", help="Path to the TensorFlow model file") (options, args) = parser.parse_args(sys.argv) sc = init_nncontext("OpenVINO Object Detection Inference Example") images = ImageSet.read(options.img_path, sc, resize_height=600, resize_width=600).get_image().collect() input_data = np.concatenate( [image.reshape((1, 1) + image.shape) for image in images], axis=0) model_path = options.model_path model = InferenceModel() model.load_openvino(model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin") predictions = model.predict(input_data) # Print the detection result of the first image. print(predictions[0])
class OpenvinoEstimatorWrapper(SparkEstimator): def __init__(self, *, model_path, batch_size=0): self.node_num, self.core_num = get_node_and_core_number() self.path = model_path if batch_size != 0: self.batch_size = batch_size else: import xml.etree.ElementTree as ET tree = ET.parse(model_path) root = tree.getroot() shape_item = root.find('./layers/layer/output/port/dim[1]') if shape_item is None: raise ValueError( "Invalid openVINO IR xml file, please check your model_path" ) self.batch_size = int(shape_item.text) self.model = InferenceModel(supported_concurrent_num=self.core_num) self.model.load_openvino( model_path=model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin", batch_size=batch_size) def fit(self, data, epochs, **kwargs): raise NotImplementedError def predict(self, data, **kwargs): def predict_transform(dict_data, batch_size): assert isinstance(dict_data, dict), "each shard should be an dict" assert "x" in dict_data, "key x should in each shard" feature_data = dict_data["x"] if isinstance(feature_data, np.ndarray): assert feature_data.shape[1] <= batch_size, \ "The batch size of input data (the second dim) should be less than the model " \ "batch size, otherwise some inputs will be ignored." elif isinstance(feature_data, list): for elem in feature_data: assert isinstance(elem, np.ndarray), "Each element in the x list should be " \ "a ndarray, but get " + \ elem.__class__.__name__ assert elem.shape[1] <= batch_size, "The batch size of each input data (the " \ "second dim) should be less than the " \ "model batch size, otherwise some inputs " \ "will be ignored." else: raise ValueError( "x in each shard should be a ndarray or a list of ndarray." ) return dict_data["x"] sc = init_nncontext() if isinstance(data, SparkXShards): assert sc is not None, "You should pass sc(spark context) if data is a XShards." from zoo.orca.learn.utils import convert_predict_to_xshard data = data.transform_shard(predict_transform, self.batch_size) result_rdd = self.model.distributed_predict(data.rdd, sc) return convert_predict_to_xshard(result_rdd) elif isinstance(data, (np.ndarray, list)): total_core_num = self.core_num * self.node_num if isinstance(data, np.ndarray): assert data.shape[1] <= self.batch_size, "The batch size of input data (the " \ "second dim) should be less than the " \ "model batch size, otherwise some " \ "inputs will be ignored." split_num = min(total_core_num, data.shape[0]) arrays = np.array_split(data, split_num) data_rdd = sc.parallelize(arrays, numSlices=split_num) elif isinstance(data, list): flattened = nest.flatten(data) data_length = len(flattened[0]) data_to_be_rdd = [] split_num = min(total_core_num, flattened[0].shape[0]) for i in range(split_num): data_to_be_rdd.append([]) for x in flattened: assert isinstance(x, np.ndarray), "the data in the data list should be " \ "ndarrays, but get " + \ x.__class__.__name__ assert len(x) == data_length, \ "the ndarrays in data must all have the same size in first dimension" \ ", got first ndarray of size {} and another {}".format(data_length, len(x)) assert x.shape[1] <= self.batch_size, "The batch size of each input data (" \ "the second dim) should be less than " \ "the model batch size, otherwise some " \ "inputs will be ignored." x_parts = np.array_split(x, split_num) for idx, x_part in enumerate(x_parts): data_to_be_rdd[idx].append(x_part) data_to_be_rdd = [ nest.pack_sequence_as(data, shard) for shard in data_to_be_rdd ] data_rdd = sc.parallelize(data_to_be_rdd, numSlices=split_num) result_rdd = self.model.distributed_predict(data_rdd, sc) result_arr_list = result_rdd.collect() result_arr = np.concatenate(result_arr_list, axis=0) return result_arr else: raise ValueError( "Only XShards, a numpy array and a list of numpy arrays are supported " "as input data, but get " + data.__class__.__name__) def evaluate(self, data, **kwargs): raise NotImplementedError def get_model(self): raise NotImplementedError def save(self, model_path): raise NotImplementedError def load(self, model_path, batch_size=0): """ Load an openVINO model. :param model_path: String. The file path to the OpenVINO IR xml file. :param batch_size: Int. Set batch Size, default is 0 (use default batch size). :return: """ self.node_num, self.core_num = get_node_and_core_number() self.path = model_path if batch_size != 0: self.batch_size = batch_size else: import xml.etree.ElementTree as ET tree = ET.parse(model_path) root = tree.getroot() shape_item = root.find('./layers/layer/output/port/dim[1]') if shape_item is None: raise ValueError( "Invalid openVINO IR xml file, please check your model_path" ) self.batch_size = int(shape_item.text) self.model = InferenceModel(supported_concurrent_num=self.core_num) self.model.load_openvino( model_path=model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin", batch_size=batch_size) def set_tensorboard(self, log_dir, app_name): raise NotImplementedError def clear_gradient_clipping(self): raise NotImplementedError def set_constant_gradient_clipping(self, min, max): raise NotImplementedError def set_l2_norm_gradient_clipping(self, clip_norm): raise NotImplementedError def get_train_summary(self, tag=None): raise NotImplementedError def get_validation_summary(self, tag=None): raise NotImplementedError def load_orca_checkpoint(self, path, version): raise NotImplementedError def load_latest_orca_checkpoint(self, path): raise NotImplementedError
class OpenvinoEstimator(SparkEstimator): def __init__(self, *, model_path, batch_size=0): self.node_num, self.core_num = get_node_and_core_number() self.path = model_path if batch_size != 0: self.batch_size = batch_size else: import xml.etree.ElementTree as ET tree = ET.parse(model_path) root = tree.getroot() shape_item = root.find('./layers/layer/output/port/dim[1]') if shape_item is None: raise ValueError( "Invalid openVINO IR xml file, please check your model_path" ) self.batch_size = int(shape_item.text) self.model = InferenceModel(supported_concurrent_num=self.core_num) self.model.load_openvino_ng( model_path=model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin", batch_size=self.batch_size) def fit(self, data, epochs, batch_size=32, feature_cols=None, label_cols=None, validation_data=None, checkpoint_trigger=None): """ Fit is not supported in OpenVINOEstimator """ raise NotImplementedError def predict(self, data, feature_cols=None): """ Predict input data :param data: data to be predicted. XShards, Spark DataFrame, numpy array and list of numpy arrays are supported. If data is XShards, each partition is a dictionary of {'x': feature}, where feature(label) is a numpy array or a list of numpy arrays. :param feature_cols: Feature column name(s) of data. Only used when data is a Spark DataFrame. Default: None. :return: predicted result. If the input data is XShards, the predict result is a XShards, each partition of the XShards is a dictionary of {'prediction': result}, where the result is a numpy array or a list of numpy arrays. If the input data is numpy arrays or list of numpy arrays, the predict result is a numpy array or a list of numpy arrays. """ from pyspark.sql import DataFrame def predict_transform(dict_data, batch_size): assert isinstance(dict_data, dict), "each shard should be an dict" assert "x" in dict_data, "key x should in each shard" feature_data = dict_data["x"] if isinstance(feature_data, np.ndarray): assert feature_data.shape[0] <= batch_size, \ "The batch size of input data (the second dim) should be less than the model " \ "batch size, otherwise some inputs will be ignored." elif isinstance(feature_data, list): for elem in feature_data: assert isinstance(elem, np.ndarray), "Each element in the x list should be " \ "a ndarray, but get " + \ elem.__class__.__name__ assert elem.shape[0] <= batch_size, "The batch size of each input data (the " \ "second dim) should be less than the " \ "model batch size, otherwise some inputs " \ "will be ignored." else: raise ValueError( "x in each shard should be a ndarray or a list of ndarray." ) return feature_data sc = init_nncontext() if isinstance(data, DataFrame): from zoo.orca.learn.utils import dataframe_to_xshards, convert_predict_rdd_to_dataframe xshards, _ = dataframe_to_xshards(data, validation_data=None, feature_cols=feature_cols, label_cols=None, mode="predict") transformed_data = xshards.transform_shard(predict_transform, self.batch_size) result_rdd = self.model.distributed_predict( transformed_data.rdd, sc) def delete_useless_result(data): shard, y = data data_length = len(shard["x"]) return y[:data_length] result_rdd = xshards.rdd.zip(result_rdd).map(delete_useless_result) return convert_predict_rdd_to_dataframe( data, result_rdd.flatMap(lambda data: data)) elif isinstance(data, SparkXShards): transformed_data = data.transform_shard(predict_transform, self.batch_size) result_rdd = self.model.distributed_predict( transformed_data.rdd, sc) def update_shard(data): shard, y = data data_length = len(shard["x"]) shard["prediction"] = y[:data_length] return shard return SparkXShards(data.rdd.zip(result_rdd).map(update_shard)) elif isinstance(data, (np.ndarray, list)): if isinstance(data, np.ndarray): split_num = math.ceil(len(data) / self.batch_size) arrays = np.array_split(data, split_num) data_length_list = list(map(lambda arr: len(arr), arrays)) data_rdd = sc.parallelize(arrays, numSlices=split_num) elif isinstance(data, list): flattened = nest.flatten(data) data_length = len(flattened[0]) data_to_be_rdd = [] split_num = math.ceil(flattened[0].shape[0] / self.batch_size) for i in range(split_num): data_to_be_rdd.append([]) for x in flattened: assert isinstance(x, np.ndarray), "the data in the data list should be " \ "ndarrays, but get " + \ x.__class__.__name__ assert len(x) == data_length, \ "the ndarrays in data must all have the same size in first dimension" \ ", got first ndarray of size {} and another {}".format(data_length, len(x)) x_parts = np.array_split(x, split_num) for idx, x_part in enumerate(x_parts): data_to_be_rdd[idx].append(x_part) data_length_list = list( map(lambda arr: len(arr), x_part)) data_to_be_rdd = [ nest.pack_sequence_as(data, shard) for shard in data_to_be_rdd ] data_rdd = sc.parallelize(data_to_be_rdd, numSlices=split_num) result_rdd = self.model.distributed_predict(data_rdd, sc) result_arr_list = result_rdd.collect() for i in range(0, len(result_arr_list)): result_arr_list[i] = result_arr_list[i][:data_length_list[i]] result_arr = np.concatenate(result_arr_list, axis=0) return result_arr else: raise ValueError( "Only XShards, Spark DataFrame, a numpy array and a list of numpy arr" "ays are supported as input data, but get " + data.__class__.__name__) def evaluate(self, data, batch_size=32, feature_cols=None, label_cols=None): """ Evaluate is not supported in OpenVINOEstimator """ raise NotImplementedError def get_model(self): """ Get_model is not supported in OpenVINOEstimator """ raise NotImplementedError def save(self, model_path): """ Save is not supported in OpenVINOEstimator """ raise NotImplementedError def load(self, model_path, batch_size=0): """ Load an openVINO model. :param model_path: String. The file path to the OpenVINO IR xml file. :param batch_size: Int. Set batch Size, default is 0 (use default batch size). :return: """ self.node_num, self.core_num = get_node_and_core_number() self.path = model_path if batch_size != 0: self.batch_size = batch_size else: import xml.etree.ElementTree as ET tree = ET.parse(model_path) root = tree.getroot() shape_item = root.find('./layers/layer/output/port/dim[1]') if shape_item is None: raise ValueError( "Invalid openVINO IR xml file, please check your model_path" ) self.batch_size = int(shape_item.text) self.model = InferenceModel(supported_concurrent_num=self.core_num) self.model.load_openvino( model_path=model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin", batch_size=batch_size) def set_tensorboard(self, log_dir, app_name): """ Set_tensorboard is not supported in OpenVINOEstimator """ raise NotImplementedError def clear_gradient_clipping(self): """ Clear_gradient_clipping is not supported in OpenVINOEstimator """ raise NotImplementedError def set_constant_gradient_clipping(self, min, max): """ Set_constant_gradient_clipping is not supported in OpenVINOEstimator """ raise NotImplementedError def set_l2_norm_gradient_clipping(self, clip_norm): """ Set_l2_norm_gradient_clipping is not supported in OpenVINOEstimator """ raise NotImplementedError def get_train_summary(self, tag=None): """ Get_train_summary is not supported in OpenVINOEstimator """ raise NotImplementedError def get_validation_summary(self, tag=None): """ Get_validation_summary is not supported in OpenVINOEstimator """ raise NotImplementedError def load_orca_checkpoint(self, path, version): """ Load_orca_checkpoint is not supported in OpenVINOEstimator """ raise NotImplementedError
def classify_process(model_path): print("* Loading model...") model = InferenceModel() model.load_openvino(model_path=model_path, weight_path=model_path[:model_path.rindex(".")] + ".bin") print("* Model loaded") # continually poll for new images to classify while True: image_ids = [] batch = None # loop over the queue start_time = time.time() count = 0 while count < 4: # Get message or None or timeout record = pub.get_message() if record and record['type'] == 'message': data = json.loads(record['data'].decode("utf-8")) image = helpers.base64_decode_image(data["image"]) image = helpers.byte_to_mat(image, dtype=settings.IMAGE_DTYPE) image = helpers.image_preprocess(image, settings.IMAGE_WIDTH, settings.IMAGE_HEIGHT) # check to see if the batch list is None if batch is None: batch = image # otherwise, stack the data else: batch = np.vstack([batch, image]) # update the list of image IDs image_ids.append(data["id"]) count += 1 print("* Pop from redis %d ms" % int(round((time.time() - start_time) * 1000))) # check to see if we need to process the batch if len(image_ids) > 0: # classify the batch batch = np.expand_dims(batch, axis=0) print("* Batch size: {}".format(batch.shape)) # Output is [1, 4, 1000] results = model.predict(batch)[0] print("* Predict a batch %d ms" % int(round((time.time() - start_time) * 1000))) # loop over the image IDs and their corresponding set of # results from our model for (imageID, resultSet) in zip(image_ids, results): # initialize the list of output predictions output = {} # loop over the results and add them to the list of # output predictions # Top 1 max_index = np.argmax(resultSet) output["Top-1"] = str(max_index) output["id"] = imageID print("* Predict result " + str(output)) # store the output predictions in the database, using # the image ID as the key so we can fetch the results DB.lpush(settings.PREDICT_QUEUE, json.dumps(output)) # remove the set of images from our queue print("* Total time used is %d ms" % int(round((time.time() - start_time) * 1000))) # sleep for a small amount time.sleep(settings.SERVER_SLEEP)
def load_model(model_path, weight_path, batch_size): model = InferenceModel(supported_concurrent_num=1) model.load_openvino(model_path=model_path, weight_path=weight_path, batch_size=batch_size) return model