def create_warmup_requests_numpy(self, batch_sizes, export_dir): """Creates warm-up requests for a given feature specification. This writes an output file in `export_dir/assets.extra/tf_serving_warmup_requests` for use with Servo. Args: batch_sizes: Batch sizes of warm-up requests to write. export_dir: Base directory for the export. Returns: The filename written. """ feature_spec = self._get_input_features_for_receiver_fn() flat_feature_spec = tensorspec_utils.flatten_spec_structure( feature_spec) tf.io.gfile.makedirs(export_dir) request_filename = os.path.join(export_dir, 'tf_serving_warmup_requests') with tf.python_io.TFRecordWriter(request_filename) as writer: for batch_size in batch_sizes: request = predict_pb2.PredictRequest() request.model_spec.name = self._model_name numpy_feature_specs = tensorspec_utils.make_constant_numpy( flat_feature_spec, constant_value=0, batch_size=batch_size) for key, numpy_spec in numpy_feature_specs.items(): request.inputs[key].CopyFrom( contrib_util.make_tensor_proto(numpy_spec)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString()) return request_filename
def _serving_warm_up(self): _dataset = self.Dataset(self._flags.compression_type, self._flags.label_key, self._flags.schema, self._flags.valid_path, prebatch=self._flags.prebatch, batch_size=1, **self.dataset_args) feature, labels = _dataset.eval_set().make_one_shot_iterator( ).get_next() feature = self._parse_sequence_weight(feature) feature = self.sparse2dense(feature, self._dataset.varlen_list) feature = { name: tensor for name, tensor in feature.items() if name in self._features } with tf.Session(config=self.sess_config) as sess: feature_n = sess.run(feature) del sess del _dataset request = predict_pb2.PredictRequest() for k, v in feature_n.items(): request.inputs[k].CopyFrom(tf.make_tensor_proto(v, shape=v.shape)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) filename = 'tf_serving_warmup_requests' file_dir = self._flags.checkpoint_dir path = os.path.join(file_dir, filename) os.makedirs(file_dir, exist_ok=True) if os.path.exists(path): os.remove(path) with tf.python_io.TFRecordWriter(path) as writer: writer.write(log.SerializeToString()) return {filename: path}
def main(_): channel = grpc.insecure_channel(FLAGS.server) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # Warm up request_w = predict_pb2.PredictRequest() test_batch, y_true = gen_data(FLAGS.test_file, 1) request_w.model_spec.name = FLAGS.model request_w.model_spec.signature_name = FLAGS.signature_name request_w.inputs['examples'].CopyFrom( tf.make_tensor_proto(test_batch, shape=[len(test_batch)])) prediction_log_pb2.PredictionLog(predict_log=prediction_log_pb2.PredictLog( request=request_w)) request = predict_pb2.PredictRequest() request.model_spec.name = FLAGS.model request.model_spec.signature_name = FLAGS.signature_name test_batch, y_true = gen_data(FLAGS.test_file, FLAGS.batch_size) request.inputs['examples'].CopyFrom( tf.make_tensor_proto(test_batch, shape=[len(test_batch)])) start = time() result_future = stub.Predict.future(request, 10.0) elapsed = (time() - start) prediction = result_future.result().outputs['prob'] # print(prediction) print('Batch size: ', FLAGS.batch_size) print('Predict AUC: ', roc_auc_score(y_true, prediction.float_val)) print("Predict time used: {0}ms".format(round(elapsed * 1000, 2)))
def export_serving_model(yolo, path, warmup_path=None, with_tensorrt=False): overwrite_path(path) tf.saved_model.save(yolo.yolo_model, path) if with_tensorrt: params = trt.TrtConversionParams( rewriter_config_template=None, max_workspace_size_bytes=trt.DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES, precision_mode=trt.TrtPrecisionMode.FP16, minimum_segment_size=3, is_dynamic_op=True, maximum_cached_engines=1, use_calibration=True, max_batch_size=1) converter = trt.TrtGraphConverterV2(input_saved_model_dir=path, conversion_params=params) converter.convert() tf.io.gfile.rmtree(path) converter.save(path) asset_extra = os.path.join(path, "assets.extra") tf.io.gfile.mkdir(asset_extra) with tf.io.TFRecordWriter( os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer: request = predict_pb2.PredictRequest() request.model_spec.name = 'detection' request.model_spec.signature_name = 'serving_default' if warmup_path is None: warmup_path = input('Please enter warm up image path:') image = open(warmup_path, 'rb').read() image_data = np.expand_dims(image, 0) request.inputs['predict_image'].CopyFrom( tf.compat.v1.make_tensor_proto(image_data)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def main(): with tf.python_io.TFRecordWriter("tf_serving_warmup_requests") as writer: # replace <request> with one of: # predict_pb2.PredictRequest(..) # classification_pb2.ClassificationRequest(..) # regression_pb2.RegressionRequest(..) # inference_pb2.MultiInferenceRequest(..) request = predict_pb2.PredictRequest() request.model_spec.name = MODEL_NAME request.model_spec.signature_name = 'serving_default' input_name = 'inputs' example1 = tf.train.Example(features=tf.train.Features( feature={ 'user_feature': _bytes_feature(user_feature), 'ctx_features': _bytes_feature(ctx_features), 'item_features': _bytes_feature(item_features), })).SerializeToString() print("example len = {}".format(len(example1))) examples = [example1] request.inputs[input_name].CopyFrom( tf.contrib.util.make_tensor_proto(examples, dtype=tf.string)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def export_serving_model(yolo, path): if tf.io.gfile.exists(path): overwrite = input("Overwrite existed model(yes/no):") if overwrite == 'yes': tf.io.gfile.rmtree(path) else: raise ValueError( "Export directory already exists, and isn't empty. Please choose a different export directory, or delete all the contents of the specified directory: " + path) tf.saved_model.simple_save( yolo.sess, path, inputs={'predict_image:0': yolo.input}, outputs={t.name: t for t in yolo.yolo_model.output}) asset_extra = os.path.join(path, "assets.extra") tf.io.gfile.mkdir(asset_extra) with tf.io.TFRecordWriter( os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer: request = predict_pb2.PredictRequest() request.model_spec.name = 'detection' request.model_spec.signature_name = 'serving_default' image = Image.open('../download/image3.jpeg') scale = yolo.input_shape[0] / max(image.size) if scale < 1: image = image.resize((int(line * scale) for line in image.size), Image.BILINEAR) image_data = np.array(image, dtype='uint8') image_data = np.expand_dims(image_data, 0) request.inputs['predict_image:0'].CopyFrom( tf.make_tensor_proto(image_data)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def create_warmup_records(saved_models_dir: str = 'saved_models'): saved_models_path = Path(saved_models_dir) for target, shape in TARGETS.items(): model_path = saved_models_path / target / '1' # SavedModel warmup records are placed inside the "assets.extra" directory # so we need to create this directory if it doesn't exist yet extra_dir = model_path / 'assets.extra' extra_dir.mkdir(parents=False, exist_ok=True) # extract n_timesteps and n_features from target shape _, n_timesteps, n_features = shape # create a single inference sample to warmup the model # path is converted to string since TFRecordWriter doesn't support path records_path = extra_dir / 'tf_serving_warmup_requests' with tf.io.TFRecordWriter(str(records_path)) as writer: log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog( request=predict_pb2.PredictRequest( inputs={ 'input_1': tf.make_tensor_proto([[[0] * n_features] * n_timesteps], dtype=tf.float32, shape=shape) }))) writer.write(log.SerializeToString())
def main(): max_seq_len = 40 input_id = [ 101, 6821, 3221, 671, 702, 3844, 6407, 4638, 1368, 2094, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] input_mask = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] segment_id = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] if len(input_id) < max_seq_len: input_id.extend([0] * (max_seq_len - len(input_id))) input_mask.extend([0] * (max_seq_len - len(input_mask))) segment_id.extend([0] * (max_seq_len - len(segment_id))) batch_size = 10 input_ids = [] input_masks = [] segment_ids = [] for i in range(batch_size): input_ids.append(input_id) input_masks.append(input_mask) segment_ids.append(segment_id) import numpy as np input_ids = np.array(input_ids) input_mask = np.array(input_masks) segment_ids = np.array(segment_ids) with tf.python_io.TFRecordWriter("tf_serving_warmup_requests") as writer: request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec(name="intent_model", signature_name='serving_default'), inputs={ "input_ids": tf.make_tensor_proto( input_ids, dtype=tf.int32, shape=[input_ids.shape[0], input_ids.shape[1]]), "input_mask": tf.make_tensor_proto( input_mask, dtype=tf.int32, shape=[input_mask.shape[0], input_mask.shape[1]]), "segment_ids": tf.make_tensor_proto( segment_ids, dtype=tf.int32, shape=[segment_ids.shape[0], segment_ids.shape[1]]), "training": tf.make_tensor_proto(False, dtype=tf.bool, shape=[]) }) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def write_warmup_requests(savedmodel_dir, model_name, image_size, batch_sizes=None, num_requests=8, image_format='PNG', input_signature='input'): """Writes warmup requests for inference into a tfrecord file. Args: savedmodel_dir: string, the file to the exported model folder. model_name: string, a model name used inside the model server. image_size: tuple/list or int, size of image. For list/tuple input, assuming it contains image height and width. batch_sizes: list, a list of batch sizes to create different input requests. num_requests: int, number of requests per batch size. image_format: string, the format of the image to write (PNG, JPEG) input_signature: string, input signature defined in exported saved model. Raises: ValueError: if batch_sizes is not a valid integer list. """ from tensorflow_serving.apis import predict_pb2 # pylint: disable=g-import-not-at-top from tensorflow_serving.apis import prediction_log_pb2 # pylint: disable=g-import-not-at-top if not isinstance(batch_sizes, list) or not batch_sizes: raise ValueError('batch sizes should be a valid non-empty list.') extra_assets_dir = os.path.join(savedmodel_dir, 'assets.extra') tf.gfile.MkDir(extra_assets_dir) if isinstance(image_size, int): height = image_size width = image_size elif isinstance(image_size, tuple) or isinstance(image_size, list): height = image_size[0] width = image_size[1] else: raise ValueError('image_size is not a supported type: %s' % type(image_size)) with tf.python_io.TFRecordWriter( os.path.join(extra_assets_dir, 'tf_serving_warmup_requests')) as writer: for batch_size in batch_sizes: for _ in range(num_requests): request = predict_pb2.PredictRequest() image = np.uint8(np.random.rand(height, width, 3) * 255) request.inputs[input_signature].CopyFrom( tf.make_tensor_proto([_encode_image(image, image_format)] * batch_size, shape=[batch_size])) request.model_spec.name = model_name request.model_spec.signature_name = 'serving_default' log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def make_warmup_record(inputs, model_name, signature_name="serving_default"): predict_request = predict_pb2.PredictRequest() predict_request.model_spec.name = model_name predict_request.model_spec.signature_name = signature_name for key, value in inputs.items(): predict_request.inputs[key].CopyFrom( tf.make_tensor_proto(value, get_tf_datatype(value))) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=predict_request)) return log.SerializeToString()
def _post_process( self, elements: Union[Sequence[tf.train.Example], Sequence[tf.train.SequenceExample]], outputs: Mapping[Text, np.ndarray] ) -> Iterable[prediction_log_pb2.PredictLog]: input_tensor_alias = self._io_tensor_spec.input_tensor_alias signature_name = self._signatures[0].name batch_size = len(elements) for output_alias, output in outputs.items(): if len(output.shape) < 1 or output.shape[0] != batch_size: raise ValueError( 'Expected output tensor %s to have at least one ' 'dimension, with the first having a size equal to the input batch ' 'size %s. Instead found %s' % (output_alias, batch_size, output.shape)) predict_log_tmpl = prediction_log_pb2.PredictLog() predict_log_tmpl.request.model_spec.signature_name = signature_name predict_log_tmpl.response.model_spec.signature_name = signature_name input_tensor_proto = predict_log_tmpl.request.inputs[ input_tensor_alias] input_tensor_proto.dtype = tf.string.as_datatype_enum input_tensor_proto.tensor_shape.dim.add().size = 1 result = [] for i in range(batch_size): predict_log = prediction_log_pb2.PredictLog() predict_log.CopyFrom(predict_log_tmpl) predict_log.request.inputs[input_tensor_alias].string_val.append( elements[i].SerializeToString()) for output_alias, output in outputs.items(): # Mimic tensor::Split tensor_proto = tf.make_tensor_proto( values=output[i], dtype=tf.as_dtype(output[i].dtype).as_datatype_enum, shape=np.expand_dims(output[i], axis=0).shape) predict_log.response.outputs[output_alias].CopyFrom( tensor_proto) result.append(predict_log) return result
def main(_): assets_dir = make_assets_dir(tf.flags.FLAGS.export_dir) with tf.Session() as session: random_tensors = load_saved_model(session, tf.flags.FLAGS.export_dir) with tf.python_io.TFRecordWriter(os.path.join(assets_dir, 'tf_serving_warmup_requests')) as writer: for _ in range(tf.flags.FLAGS.batch_size): request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec(name=tf.flags.FLAGS.name), inputs={k: tf.make_tensor_proto(v) for k, v in session.run(random_tensors).items()} ) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def main(): """Generate TFRecords for warming up.""" with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer: image_bytes = get_image_bytes() predict_request = predict_pb2.PredictRequest() predict_request.model_spec.name = 'resnet' predict_request.model_spec.signature_name = 'serving_default' predict_request.inputs['image_bytes'].CopyFrom( tensor_util.make_tensor_proto([image_bytes], tf.string)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=predict_request)) for r in range(NUM_RECORDS): writer.write(log.SerializeToString())
def _convert_to_prediction_log(request: iv_types.Request): """Try convert infra validation request to TF-Serving PredictionLog.""" if isinstance(request, classification_pb2.ClassificationRequest): return prediction_log_pb2.PredictionLog( classify_log=prediction_log_pb2.ClassifyLog(request=request)) elif isinstance(request, regression_pb2.RegressionRequest): return prediction_log_pb2.PredictionLog( regress_log=prediction_log_pb2.RegressLog(request=request)) elif isinstance(request, predict_pb2.PredictRequest): return prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) else: raise NotImplementedError( f'Cannot convert {type(request)} to PredictionLog')
def main(): """Generate TFRecords for warming up.""" with tf.io.TFRecordWriter( f"models/{MODEL_NAME}/{MODEL_VERSION}/assets.extra/tf_serving_warmup_requests" ) as writer: predict_request = predict_pb2.PredictRequest() predict_request.model_spec.name = f'{MODEL_NAME}' predict_request.model_spec.signature_name = 'serving_default' predict_request.inputs["input_1"].CopyFrom( tf.make_tensor_proto(tf.reshape(X_new, (-1, 224, 224, 3)))) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=predict_request)) for r in range(NUM_RECORDS): writer.write(log.SerializeToString())
def main(): # Build a batch of images. image_data = img_data(image_path) imag_data = [] for i in range(len(image_data)): imag_data.append(image_data[i].numpy().tolist()) with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer: request = predict_pb2.PredictRequest() request.model_spec.name = 'model1' request.model_spec.signature_name = 'serving_default' request.inputs['input_1'].CopyFrom( tf.make_tensor_proto(imag_data, shape=[len(imag_data),224,224,3])) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def main(): with tf.python_io.TFRecordWriter( "/home/tata/Projects/hand_detector/inference_graph/1/assets.extra/tf_serving_warmup_requests" ) as writer: request = predict_pb2.PredictRequest() request.model_spec.name = serving_config.model_name image = cv2.imread('/home/tata/hand2.jpg') image_batch = np.array([image] * 5) # Use a batch of 5 images for warmup request.inputs['inputs'].CopyFrom( tf.contrib.util.make_tensor_proto(image_batch, shape=image_batch.shape)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def _post_process( self, elements: List[Union[tf.train.Example, tf.train.SequenceExample]], outputs: Sequence[Mapping[Text, Any]] ) -> Iterable[prediction_log_pb2.PredictLog]: result = [] for output in outputs: predict_log = prediction_log_pb2.PredictLog() for output_alias, values in output.items(): values = np.array(values) tensor_proto = tf.make_tensor_proto( values=values, dtype=tf.as_dtype(values.dtype).as_datatype_enum, shape=np.expand_dims(values, axis=0).shape) predict_log.response.outputs[output_alias].CopyFrom(tensor_proto) result.append(predict_log) return result
def main(): if len(sys.argv) != 2 or sys.argv[-1].startswith('-'): print('Usage: resnet_warmup.py saved_model_dir') sys.exit(-1) model_dir = sys.argv[-1] if not os.path.isdir(model_dir): print('The saved model directory: %s does not exist. ' 'Specify the path of an existing model.' % model_dir) sys.exit(-1) # Create the assets.extra directory, assuming model_dir is the versioned # directory containing the SavedModel assets_dir = os.path.join(model_dir, 'assets.extra') if not os.path.exists(assets_dir): os.mkdir(assets_dir) warmup_file = os.path.join(assets_dir, 'tf_serving_warmup_requests') with tf.io.TFRecordWriter(warmup_file) as writer: for image in IMAGE_URLS: # Download the image dl_request = requests.get(image, stream=True) dl_request.raise_for_status() data = dl_request.content if not MODEL_ACCEPT_JPG: data = Image.open(io.BytesIO(dl_request.content)) # Normalize and batchify the image data = np.array(data) / 255.0 data = np.expand_dims(data, 0) data = data.astype(np.float32) # Create the inference request request = predict_pb2.PredictRequest() request.model_spec.name = 'resnet' request.model_spec.signature_name = 'serving_default' request.inputs['input_1'].CopyFrom( tf.make_tensor_proto(data)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString()) print('Created the file \'%s\', restart tensorflow_model_server to warmup ' 'the ResNet SavedModel.' % warmup_file)
def main(argv): count = 0 images = [] files = [path.join(path.join(FLAGS.dataset, f)) for f in os.listdir(FLAGS.dataset) if path.isfile(path.join(FLAGS.dataset, f)) ] files = [f for f in files if f.endswith(('.png', '.jpg', '.jpeg'))] for file in files: img_raw = tf.image.decode_image(open(file, 'rb').read(), channels=3) image = preprocess_image(img_raw, FLAGS.input_size) image = tf.expand_dims(image, 0) images.append(image) count += 1 if count == FLAGS.size: break input_tensor = tf.concat(images, 0) with tf.io.TFRecordWriter('tf_serving_warmup_requests') as writer: request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec( name=FLAGS.model_name ), inputs={ FLAGS.input_tensor: tf.make_tensor_proto( input_tensor, shape=input_tensor.shape, dtype=input_tensor.dtype ) } ) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request) ) writer.write(log.SerializeToString()) logging.info('"tf_serving_warmup_requests" created with success!') logging.info('to use it paste it to the "<model>/<version>/assets.extra" folder on the serving configuration folder')
def main(): """Generate TFRecords for warming up.""" with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer: with open(IMAGE_PATH, 'rb') as f: # image_data = base64.b64encode(f.read()).decode('utf-8') image_data = f.read() predict_request = predict_pb2.PredictRequest() predict_request.model_spec.name = 'resnet' predict_request.model_spec.signature_name = 'serving_default' predict_request.inputs['string_inp'].CopyFrom( tensor_util.make_tensor_proto(image_data, shape=[ 1, ])) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=predict_request)) for r in range(NUM_RECORDS): writer.write(log.SerializeToString())
def build_warmup_data(savemodel_dir, config_dict): serving_batch = config_dict['SERVING']['SERVE_BATCH'] model_spec_name = config_dict['SERVING']['NAME'] output_paras = config_dict['OUTPUT'] experiment_base_dir = os.path.join(output_paras['OUTPUT_SAVE_DIR'], output_paras['EXPERIMENT_NAME']) model_save_dir = os.path.join(experiment_base_dir, 'weights') label_file = os.path.join(model_save_dir, 'labels.txt') trainset_root_dir = config_dict['DATASET']['DATASET_ROOT_DIR'] warmup_data_dir = _get_warmup_data_dir(trainset_root_dir, label_file) assets_extra_dir = os.path.join(savemodel_dir, 'assets.extra') mkdir_if_nonexist(assets_extra_dir, raise_error=False) warmup_save_path = os.path.join(assets_extra_dir, 'tf_serving_warmup_requests') # load warmup images cnt = 1 image_str_list = [] for img_name in os.listdir(warmup_data_dir): if cnt > serving_batch: break img_path = os.path.join(warmup_data_dir, img_name) image_str = (open(img_path, 'rb').read()) image_str_list.append(image_str) cnt += 1 with tf.python_io.TFRecordWriter(warmup_save_path) as writer: request = predict_pb2.PredictRequest() request.model_spec.name = model_spec_name request.model_spec.signature_name = "serving_default" #request.inputs['input'].ParseFromString(tf.make_tensor_proto(image_str, shape=[1]).SerializeToString()) request.inputs['input'].CopyFrom( tf.contrib.util.make_tensor_proto(image_str_list, dtype=tf.string)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def main(): with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer: # replace <request> with one of: # # classification_pb2.ClassificationRequest(..) # regression_pb2.RegressionRequest(..) # inference_pb2.MultiInferenceRequest(..) request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec(name="youtube_dnn", signature_name="serving_default"), inputs={ "sample": tf.compat.v1.make_tensor_proto([[1, 2, 3, 4, 5]], dtype=tf.int64), "label": tf.compat.v1.make_tensor_proto([[0]], dtype=tf.int64) } ) print(request) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) for r in range(100): writer.write(log.SerializeToString()) writer.write(log.SerializeToString())
def main(): ### Generate TFRecords for warming up. ### global filename for root, dirs, files in os.walk(img_path): for img_file in files: if img_file.endswith(".png") or img_file.endswith(".bmp") or img_file.endswith(".jpg"): filename = os.path.join(root, img_file) break print(filename) with tf.io.TFRecordWriter(f"{target_path}tf_serving_warmup_requests") as writer: # replace <request> with one of: # predict_pb2.PredictRequest(..) # classification_pb2.ClassificationRequest(..) # regression_pb2.RegressionRequest(..) # inference_pb2.MultiInferenceRequest(..) ### Method 1 ### if model_origin == "saiap": request = predict_pb2.PredictRequest() request.model_spec.name = MODEL_NAME request.model_spec.signature_name = SIG request.inputs[INPUTS].CopyFrom( tensor_util.make_tensor_proto([saiap_open_image(filename)], tf.string) ) elif model_origin == "wzsda": ## Method 2 ### request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec(name=MODEL_NAME, signature_name=SIG), inputs={ INPUTS: tensor_util.make_tensor_proto([open_image(filename)], tf.string), "degree": tensor_util.make_tensor_proto(["0"], tf.string) } ) else: print(f"arg 'model_origin' {model_origin} is not supported!") log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def write_warmup_requests(savedmodel_dir, model_name, image_size, batch_sizes=None, num_requests=8): """Writes warmup requests for inference into a tfrecord file. Args: savedmodel_dir: string, the file to the exported model folder. model_name: string, a model name used inside the model server. image_size: int, size of image, assuming image height and width. batch_sizes: list, a list of batch sizes to create different input requests. num_requests: int, number of requests per batch size. Raises: ValueError: if batch_sizes is not a valid integer list. """ if not isinstance(batch_sizes, list) or not batch_sizes: raise ValueError('batch sizes should be a valid non-empty list.') extra_assets_dir = os.path.join(savedmodel_dir, 'assets.extra') tf.gfile.MkDir(extra_assets_dir) with tf.python_io.TFRecordWriter( os.path.join(extra_assets_dir, 'tf_serving_warmup_requests')) as writer: for batch_size in batch_sizes: for _ in range(num_requests): request = predict_pb2.PredictRequest() image = np.uint8( np.random.rand(image_size, image_size, 3) * 255) request.inputs['input'].CopyFrom( tf.make_tensor_proto([_encode_image(image)] * batch_size, shape=[batch_size])) request.model_spec.name = model_name request.model_spec.signature_name = 'serving_default' log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def export_serving_model(yolo, path): if tf.version.VERSION.startswith('1.'): tf.saved_model.simple_save( yolo.sess, path, inputs={'predict_image:0': yolo.input}, outputs={t.name: t for t in yolo.yolo_model.output}) else: signature_def_map = { tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.saved_model.signature_def_utils.predict_signature_def( {'predict_image:0': yolo.input}, {t.name: t for t in yolo.yolo_model.output}) } tf.saved_model.experimental.save(yolo.yolo_model, path, signature_def_map) asset_extra = os.path.join(path, "assets.extra") tf.io.gfile.mkdir(asset_extra) with tf.io.TFRecordWriter( os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer: request = predict_pb2.PredictRequest() request.model_spec.name = 'detection' request.model_spec.signature_name = 'serving_default' image = Image.open('../download/image3.jpeg') scale = yolo.input_shape[0] / max(image.size) if scale < 1: image = image.resize((int(line * scale) for line in image.size), Image.BILINEAR) image_data = np.array(image, dtype='uint8') image_data = np.expand_dims(image_data, 0) request.inputs['predict_image:0'].CopyFrom( tf.make_tensor_proto(image_data)) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())
def main(): if len(sys.argv) != 2 or sys.argv[-1].startswith('-'): print('Usage: resnet_warmup.py saved_model_dir') sys.exit(-1) model_dir = sys.argv[-1] if not os.path.isdir(model_dir): print('The saved model directory: %s does not exist. ' 'Specify the path of an existing model.' % model_dir) sys.exit(-1) # Create the assets.extra directory assets_dir = os.path.join(model_dir, 'assets.extra') if not os.path.exists(assets_dir): os.mkdir(assets_dir) warmup_file = os.path.join(assets_dir, 'tf_serving_warmup_requests') with tf.io.TFRecordWriter(warmup_file) as writer: for image in IMAGE_URLS: # Download the image dl_request = requests.get(image, stream=True) dl_request.raise_for_status() data = dl_request.content # Create the inference request request = predict_pb2.PredictRequest() request.model_spec.name = 'resnet' request.model_spec.signature_name = 'serving_default' request.inputs['image_bytes'].CopyFrom( tf.make_tensor_proto(data, shape=[1])) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString()) print('Created the file \'%s\', restart tensorflow_model_server to warmup ' 'the ResNet SavedModel.' % warmup_file)
import tensorflow as tf from tensorflow_serving.apis import classification_pb2 from tensorflow_serving.apis import inference_pb2 from tensorflow_serving.apis import model_pb2 from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_log_pb2 from tensorflow_serving.apis import regression_pb2 with tf.io.TFRecordWriter("../testdata/tf_serving_warmup_requests") as writer: predict_request = predict_pb2.PredictRequest() predict_request.model_spec.name = 'some_model' predict_request.model_spec.signature_name = 'some_signature' example_proto = tf.train.Example(features=tf.train.Features(feature={})) predict_request.inputs['inputs'].CopyFrom( tf.make_tensor_proto([example_proto.SerializeToString()])) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=predict_request)) writer.write(log.SerializeToString())
def export(model_dir, vocab_dir, max_length, theta, emb_size, dim, num_oov_buckets): while True: cur = os.path.join(model_dir, str(int(time.time()))) if not tf.gfile.Exists(cur): break print("export model path: %s" % cur) method_name = tf.saved_model.signature_constants.PREDICT_METHOD_NAME rowkeys_file = os.path.join(vocab_dir, "rowkeys.vocab") embedding_file = os.path.join(vocab_dir, "emb.vocab") with tf.Graph().as_default(), tf.Session() as sess: rowkeys = tf.placeholder(dtype=tf.string, name="rowkeys") algorithm_ids = tf.placeholder(dtype=tf.uint32, name="algorithm_id") scores = tf.placeholder(dtype=tf.float32, name="scores") # rowkeys = tf.constant(["2785c0f6f0e592ah", # "8605d35a21f857bk", # "7915d39c6f9755ap", # "3155d3846cb468bk", # "4285d39597b375bk"], dtype=tf.string) # algorithm_ids = tf.constant([2081, # 2803, # 2086, # 2803, # 2086], dtype=tf.uint32) # scores = tf.constant([ # 0.1, # 0.2, # 0.3, # 0.11, # 0.7 # ], dtype=float) with open(rowkeys_file, encoding="utf8") as fi: lines = fi.readlines() vocab_size = len(lines) print(vocab_size) emb = tf.Variable(np.loadtxt(embedding_file, delimiter=' '), dtype=tf.float32) print(emb.shape) table = tf.contrib.lookup.index_table_from_file( vocabulary_file=rowkeys_file, vocab_size=vocab_size - num_oov_buckets, hasher_spec=tf.contrib.lookup.FastHashSpec, num_oov_buckets=num_oov_buckets) rowkey_ids = table.lookup(rowkeys) rowkeys_embedding = tf.nn.embedding_lookup(emb, rowkey_ids) rowkeys_embedding /= tf.linalg.norm(rowkeys_embedding, axis=1, keepdims=True) rowkeys_embedding = tf.where(tf.is_nan(rowkeys_embedding), tf.zeros_like(rowkeys_embedding), rowkeys_embedding) similarities = tf.cast( tf.matmul(rowkeys_embedding, rowkeys_embedding, transpose_b=True), tf.float32) kernel_matrix = tf.reshape( scores, [-1, 1]) * similarities * tf.reshape(scores, [1, -1]) # alpha = theta / (2 * (1 - theta)) # kernel_matrix = tf.math.exp(alpha * tf.reshape(scores, [-1, 1])) * similarities * tf.math.exp(alpha * tf.reshape(scores, [1, -1])) indices = dpp(kernel_matrix, max_length) predict_rowkeys = tf.gather(rowkeys, indices) predict_scores = tf.gather(scores, indices) predict_algorithm_ids = tf.gather(algorithm_ids, indices) predict_positions = indices sess.run(tf.global_variables_initializer()) sess.run(tf.tables_initializer()) #sess.run(emb, feed_dict={embedding_placeholder: emb_dict}) # print(sess.run(predict_rowkeys)) # print(sess.run(predict_scores)) signature_def_map = signature({ "prediction": { "inputs": { 'rowkeys': rowkeys, "scores": scores, "algorithm_ids": algorithm_ids }, "outputs": { "rowkeys": predict_rowkeys, "scores": predict_scores, "algorithm_ids": predict_algorithm_ids, "origin_position": predict_positions }, "method_name": method_name }, }) builder = tf.saved_model.builder.SavedModelBuilder(cur) builder.add_meta_graph_and_variables( sess, tags=[tf.saved_model.tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), main_op=tf.tables_initializer()) builder.save() os.mkdir(os.path.join(cur, "assets.extra")) with tf.python_io.TFRecordWriter( os.path.join( cur, "assets.extra/tf_serving_warmup_requests")) as writer: request = predict_pb2.PredictRequest( model_spec=model_pb2.ModelSpec(name="kva_dpp_model", signature_name="prediction"), inputs={ "rowkeys": tf.make_tensor_proto( ["2785c0f6f0e592ah", "2785c0f6f0e592ah"], dtype=tf.string), "algorithm_ids": tf.make_tensor_proto([2081, 2081], dtype=tf.uint32), "scores": tf.make_tensor_proto([0.7, 0.7], dtype=tf.float32) }) print(request) log = prediction_log_pb2.PredictionLog( predict_log=prediction_log_pb2.PredictLog(request=request)) writer.write(log.SerializeToString())