def create_warmup_requests_numpy(self, batch_sizes, export_dir):
"""Creates warm-up requests for a given feature specification.
This writes an output file in
`export_dir/assets.extra/tf_serving_warmup_requests` for use with Servo.
Args:
batch_sizes: Batch sizes of warm-up requests to write.
export_dir: Base directory for the export.
Returns:
The filename written.
"""
feature_spec = self._get_input_features_for_receiver_fn()
flat_feature_spec = tensorspec_utils.flatten_spec_structure(
feature_spec)
tf.io.gfile.makedirs(export_dir)
request_filename = os.path.join(export_dir,
'tf_serving_warmup_requests')
with tf.python_io.TFRecordWriter(request_filename) as writer:
for batch_size in batch_sizes:
request = predict_pb2.PredictRequest()
request.model_spec.name = self._model_name
numpy_feature_specs = tensorspec_utils.make_constant_numpy(
flat_feature_spec, constant_value=0, batch_size=batch_size)
for key, numpy_spec in numpy_feature_specs.items():
request.inputs[key].CopyFrom(
contrib_util.make_tensor_proto(numpy_spec))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
return request_filename
def _serving_warm_up(self):
_dataset = self.Dataset(self._flags.compression_type,
self._flags.label_key,
self._flags.schema,
self._flags.valid_path,
prebatch=self._flags.prebatch,
batch_size=1,
**self.dataset_args)
feature, labels = _dataset.eval_set().make_one_shot_iterator(
).get_next()
feature = self._parse_sequence_weight(feature)
feature = self.sparse2dense(feature, self._dataset.varlen_list)
feature = {
name: tensor
for name, tensor in feature.items() if name in self._features
}
with tf.Session(config=self.sess_config) as sess:
feature_n = sess.run(feature)
del sess
del _dataset
request = predict_pb2.PredictRequest()
for k, v in feature_n.items():
request.inputs[k].CopyFrom(tf.make_tensor_proto(v, shape=v.shape))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
filename = 'tf_serving_warmup_requests'
file_dir = self._flags.checkpoint_dir
path = os.path.join(file_dir, filename)
os.makedirs(file_dir, exist_ok=True)
if os.path.exists(path):
os.remove(path)
with tf.python_io.TFRecordWriter(path) as writer:
writer.write(log.SerializeToString())
return {filename: path}
def main(_):
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Warm up
request_w = predict_pb2.PredictRequest()
test_batch, y_true = gen_data(FLAGS.test_file, 1)
request_w.model_spec.name = FLAGS.model
request_w.model_spec.signature_name = FLAGS.signature_name
request_w.inputs['examples'].CopyFrom(
tf.make_tensor_proto(test_batch, shape=[len(test_batch)]))
prediction_log_pb2.PredictionLog(predict_log=prediction_log_pb2.PredictLog(
request=request_w))
request = predict_pb2.PredictRequest()
request.model_spec.name = FLAGS.model
request.model_spec.signature_name = FLAGS.signature_name
test_batch, y_true = gen_data(FLAGS.test_file, FLAGS.batch_size)
request.inputs['examples'].CopyFrom(
tf.make_tensor_proto(test_batch, shape=[len(test_batch)]))
start = time()
result_future = stub.Predict.future(request, 10.0)
elapsed = (time() - start)
prediction = result_future.result().outputs['prob']
# print(prediction)
print('Batch size: ', FLAGS.batch_size)
print('Predict AUC: ', roc_auc_score(y_true, prediction.float_val))
print("Predict time used: {0}ms".format(round(elapsed * 1000, 2)))
def export_serving_model(yolo, path, warmup_path=None, with_tensorrt=False):
overwrite_path(path)
tf.saved_model.save(yolo.yolo_model, path)
if with_tensorrt:
params = trt.TrtConversionParams(
rewriter_config_template=None,
max_workspace_size_bytes=trt.DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
precision_mode=trt.TrtPrecisionMode.FP16,
minimum_segment_size=3,
is_dynamic_op=True,
maximum_cached_engines=1,
use_calibration=True,
max_batch_size=1)
converter = trt.TrtGraphConverterV2(input_saved_model_dir=path,
conversion_params=params)
converter.convert()
tf.io.gfile.rmtree(path)
converter.save(path)
asset_extra = os.path.join(path, "assets.extra")
tf.io.gfile.mkdir(asset_extra)
with tf.io.TFRecordWriter(
os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'detection'
request.model_spec.signature_name = 'serving_default'
if warmup_path is None:
warmup_path = input('Please enter warm up image path:')
image = open(warmup_path, 'rb').read()
image_data = np.expand_dims(image, 0)
request.inputs['predict_image'].CopyFrom(
tf.compat.v1.make_tensor_proto(image_data))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def main():
with tf.python_io.TFRecordWriter("tf_serving_warmup_requests") as writer:
# replace <request> with one of:
# predict_pb2.PredictRequest(..)
# classification_pb2.ClassificationRequest(..)
# regression_pb2.RegressionRequest(..)
# inference_pb2.MultiInferenceRequest(..)
request = predict_pb2.PredictRequest()
request.model_spec.name = MODEL_NAME
request.model_spec.signature_name = 'serving_default'
input_name = 'inputs'
example1 = tf.train.Example(features=tf.train.Features(
feature={
'user_feature': _bytes_feature(user_feature),
'ctx_features': _bytes_feature(ctx_features),
'item_features': _bytes_feature(item_features),
})).SerializeToString()
print("example len = {}".format(len(example1)))
examples = [example1]
request.inputs[input_name].CopyFrom(
tf.contrib.util.make_tensor_proto(examples, dtype=tf.string))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def export_serving_model(yolo, path):
if tf.io.gfile.exists(path):
overwrite = input("Overwrite existed model(yes/no):")
if overwrite == 'yes':
tf.io.gfile.rmtree(path)
else:
raise ValueError(
"Export directory already exists, and isn't empty. Please choose a different export directory, or delete all the contents of the specified directory: "
+ path)
tf.saved_model.simple_save(
yolo.sess,
path,
inputs={'predict_image:0': yolo.input},
outputs={t.name: t
for t in yolo.yolo_model.output})
asset_extra = os.path.join(path, "assets.extra")
tf.io.gfile.mkdir(asset_extra)
with tf.io.TFRecordWriter(
os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'detection'
request.model_spec.signature_name = 'serving_default'
image = Image.open('../download/image3.jpeg')
scale = yolo.input_shape[0] / max(image.size)
if scale < 1:
image = image.resize((int(line * scale) for line in image.size),
Image.BILINEAR)
image_data = np.array(image, dtype='uint8')
image_data = np.expand_dims(image_data, 0)
request.inputs['predict_image:0'].CopyFrom(
tf.make_tensor_proto(image_data))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def create_warmup_records(saved_models_dir: str = 'saved_models'):
saved_models_path = Path(saved_models_dir)
for target, shape in TARGETS.items():
model_path = saved_models_path / target / '1'
# SavedModel warmup records are placed inside the "assets.extra" directory
# so we need to create this directory if it doesn't exist yet
extra_dir = model_path / 'assets.extra'
extra_dir.mkdir(parents=False, exist_ok=True)
# extract n_timesteps and n_features from target shape
_, n_timesteps, n_features = shape
# create a single inference sample to warmup the model
# path is converted to string since TFRecordWriter doesn't support path
records_path = extra_dir / 'tf_serving_warmup_requests'
with tf.io.TFRecordWriter(str(records_path)) as writer:
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(
request=predict_pb2.PredictRequest(
inputs={
'input_1':
tf.make_tensor_proto([[[0] * n_features] *
n_timesteps],
dtype=tf.float32,
shape=shape)
})))
writer.write(log.SerializeToString())
def main():
max_seq_len = 40
input_id = [
101, 6821, 3221, 671, 702, 3844, 6407, 4638, 1368, 2094, 102, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0
]
input_mask = [
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
segment_id = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
if len(input_id) < max_seq_len:
input_id.extend([0] * (max_seq_len - len(input_id)))
input_mask.extend([0] * (max_seq_len - len(input_mask)))
segment_id.extend([0] * (max_seq_len - len(segment_id)))
batch_size = 10
input_ids = []
input_masks = []
segment_ids = []
for i in range(batch_size):
input_ids.append(input_id)
input_masks.append(input_mask)
segment_ids.append(segment_id)
import numpy as np
input_ids = np.array(input_ids)
input_mask = np.array(input_masks)
segment_ids = np.array(segment_ids)
with tf.python_io.TFRecordWriter("tf_serving_warmup_requests") as writer:
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(name="intent_model",
signature_name='serving_default'),
inputs={
"input_ids":
tf.make_tensor_proto(
input_ids,
dtype=tf.int32,
shape=[input_ids.shape[0], input_ids.shape[1]]),
"input_mask":
tf.make_tensor_proto(
input_mask,
dtype=tf.int32,
shape=[input_mask.shape[0], input_mask.shape[1]]),
"segment_ids":
tf.make_tensor_proto(
segment_ids,
dtype=tf.int32,
shape=[segment_ids.shape[0], segment_ids.shape[1]]),
"training":
tf.make_tensor_proto(False, dtype=tf.bool, shape=[])
})
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def write_warmup_requests(savedmodel_dir,
model_name,
image_size,
batch_sizes=None,
num_requests=8,
image_format='PNG',
input_signature='input'):
"""Writes warmup requests for inference into a tfrecord file.
Args:
savedmodel_dir: string, the file to the exported model folder.
model_name: string, a model name used inside the model server.
image_size: tuple/list or int, size of image. For list/tuple input, assuming
it contains image height and width.
batch_sizes: list, a list of batch sizes to create different input requests.
num_requests: int, number of requests per batch size.
image_format: string, the format of the image to write (PNG, JPEG)
input_signature: string, input signature defined in exported saved model.
Raises:
ValueError: if batch_sizes is not a valid integer list.
"""
from tensorflow_serving.apis import predict_pb2 # pylint: disable=g-import-not-at-top
from tensorflow_serving.apis import prediction_log_pb2 # pylint: disable=g-import-not-at-top
if not isinstance(batch_sizes, list) or not batch_sizes:
raise ValueError('batch sizes should be a valid non-empty list.')
extra_assets_dir = os.path.join(savedmodel_dir, 'assets.extra')
tf.gfile.MkDir(extra_assets_dir)
if isinstance(image_size, int):
height = image_size
width = image_size
elif isinstance(image_size, tuple) or isinstance(image_size, list):
height = image_size[0]
width = image_size[1]
else:
raise ValueError('image_size is not a supported type: %s' %
type(image_size))
with tf.python_io.TFRecordWriter(
os.path.join(extra_assets_dir,
'tf_serving_warmup_requests')) as writer:
for batch_size in batch_sizes:
for _ in range(num_requests):
request = predict_pb2.PredictRequest()
image = np.uint8(np.random.rand(height, width, 3) * 255)
request.inputs[input_signature].CopyFrom(
tf.make_tensor_proto([_encode_image(image, image_format)] *
batch_size,
shape=[batch_size]))
request.model_spec.name = model_name
request.model_spec.signature_name = 'serving_default'
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def make_warmup_record(inputs, model_name, signature_name="serving_default"):
predict_request = predict_pb2.PredictRequest()
predict_request.model_spec.name = model_name
predict_request.model_spec.signature_name = signature_name
for key, value in inputs.items():
predict_request.inputs[key].CopyFrom(
tf.make_tensor_proto(value, get_tf_datatype(value)))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=predict_request))
return log.SerializeToString()
def _post_process(
self, elements: Union[Sequence[tf.train.Example],
Sequence[tf.train.SequenceExample]],
outputs: Mapping[Text, np.ndarray]
) -> Iterable[prediction_log_pb2.PredictLog]:
input_tensor_alias = self._io_tensor_spec.input_tensor_alias
signature_name = self._signatures[0].name
batch_size = len(elements)
for output_alias, output in outputs.items():
if len(output.shape) < 1 or output.shape[0] != batch_size:
raise ValueError(
'Expected output tensor %s to have at least one '
'dimension, with the first having a size equal to the input batch '
'size %s. Instead found %s' %
(output_alias, batch_size, output.shape))
predict_log_tmpl = prediction_log_pb2.PredictLog()
predict_log_tmpl.request.model_spec.signature_name = signature_name
predict_log_tmpl.response.model_spec.signature_name = signature_name
input_tensor_proto = predict_log_tmpl.request.inputs[
input_tensor_alias]
input_tensor_proto.dtype = tf.string.as_datatype_enum
input_tensor_proto.tensor_shape.dim.add().size = 1
result = []
for i in range(batch_size):
predict_log = prediction_log_pb2.PredictLog()
predict_log.CopyFrom(predict_log_tmpl)
predict_log.request.inputs[input_tensor_alias].string_val.append(
elements[i].SerializeToString())
for output_alias, output in outputs.items():
# Mimic tensor::Split
tensor_proto = tf.make_tensor_proto(
values=output[i],
dtype=tf.as_dtype(output[i].dtype).as_datatype_enum,
shape=np.expand_dims(output[i], axis=0).shape)
predict_log.response.outputs[output_alias].CopyFrom(
tensor_proto)
result.append(predict_log)
return result
def main(_):
assets_dir = make_assets_dir(tf.flags.FLAGS.export_dir)
with tf.Session() as session:
random_tensors = load_saved_model(session, tf.flags.FLAGS.export_dir)
with tf.python_io.TFRecordWriter(os.path.join(assets_dir, 'tf_serving_warmup_requests')) as writer:
for _ in range(tf.flags.FLAGS.batch_size):
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(name=tf.flags.FLAGS.name),
inputs={k: tf.make_tensor_proto(v) for k, v in session.run(random_tensors).items()}
)
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def main():
"""Generate TFRecords for warming up."""
with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer:
image_bytes = get_image_bytes()
predict_request = predict_pb2.PredictRequest()
predict_request.model_spec.name = 'resnet'
predict_request.model_spec.signature_name = 'serving_default'
predict_request.inputs['image_bytes'].CopyFrom(
tensor_util.make_tensor_proto([image_bytes], tf.string))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=predict_request))
for r in range(NUM_RECORDS):
writer.write(log.SerializeToString())
def _convert_to_prediction_log(request: iv_types.Request):
"""Try convert infra validation request to TF-Serving PredictionLog."""
if isinstance(request, classification_pb2.ClassificationRequest):
return prediction_log_pb2.PredictionLog(
classify_log=prediction_log_pb2.ClassifyLog(request=request))
elif isinstance(request, regression_pb2.RegressionRequest):
return prediction_log_pb2.PredictionLog(
regress_log=prediction_log_pb2.RegressLog(request=request))
elif isinstance(request, predict_pb2.PredictRequest):
return prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
else:
raise NotImplementedError(
f'Cannot convert {type(request)} to PredictionLog')
def main():
"""Generate TFRecords for warming up."""
with tf.io.TFRecordWriter(
f"models/{MODEL_NAME}/{MODEL_VERSION}/assets.extra/tf_serving_warmup_requests"
) as writer:
predict_request = predict_pb2.PredictRequest()
predict_request.model_spec.name = f'{MODEL_NAME}'
predict_request.model_spec.signature_name = 'serving_default'
predict_request.inputs["input_1"].CopyFrom(
tf.make_tensor_proto(tf.reshape(X_new, (-1, 224, 224, 3))))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=predict_request))
for r in range(NUM_RECORDS):
writer.write(log.SerializeToString())
def main():
# Build a batch of images.
image_data = img_data(image_path)
imag_data = []
for i in range(len(image_data)):
imag_data.append(image_data[i].numpy().tolist())
with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'model1'
request.model_spec.signature_name = 'serving_default'
request.inputs['input_1'].CopyFrom(
tf.make_tensor_proto(imag_data, shape=[len(imag_data),224,224,3]))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def main():
with tf.python_io.TFRecordWriter(
"/home/tata/Projects/hand_detector/inference_graph/1/assets.extra/tf_serving_warmup_requests"
) as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = serving_config.model_name
image = cv2.imread('/home/tata/hand2.jpg')
image_batch = np.array([image] *
5) # Use a batch of 5 images for warmup
request.inputs['inputs'].CopyFrom(
tf.contrib.util.make_tensor_proto(image_batch,
shape=image_batch.shape))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def _post_process(
self, elements: List[Union[tf.train.Example, tf.train.SequenceExample]],
outputs: Sequence[Mapping[Text, Any]]
) -> Iterable[prediction_log_pb2.PredictLog]:
result = []
for output in outputs:
predict_log = prediction_log_pb2.PredictLog()
for output_alias, values in output.items():
values = np.array(values)
tensor_proto = tf.make_tensor_proto(
values=values,
dtype=tf.as_dtype(values.dtype).as_datatype_enum,
shape=np.expand_dims(values, axis=0).shape)
predict_log.response.outputs[output_alias].CopyFrom(tensor_proto)
result.append(predict_log)
return result
def main():
if len(sys.argv) != 2 or sys.argv[-1].startswith('-'):
print('Usage: resnet_warmup.py saved_model_dir')
sys.exit(-1)
model_dir = sys.argv[-1]
if not os.path.isdir(model_dir):
print('The saved model directory: %s does not exist. '
'Specify the path of an existing model.' % model_dir)
sys.exit(-1)
# Create the assets.extra directory, assuming model_dir is the versioned
# directory containing the SavedModel
assets_dir = os.path.join(model_dir, 'assets.extra')
if not os.path.exists(assets_dir):
os.mkdir(assets_dir)
warmup_file = os.path.join(assets_dir, 'tf_serving_warmup_requests')
with tf.io.TFRecordWriter(warmup_file) as writer:
for image in IMAGE_URLS:
# Download the image
dl_request = requests.get(image, stream=True)
dl_request.raise_for_status()
data = dl_request.content
if not MODEL_ACCEPT_JPG:
data = Image.open(io.BytesIO(dl_request.content))
# Normalize and batchify the image
data = np.array(data) / 255.0
data = np.expand_dims(data, 0)
data = data.astype(np.float32)
# Create the inference request
request = predict_pb2.PredictRequest()
request.model_spec.name = 'resnet'
request.model_spec.signature_name = 'serving_default'
request.inputs['input_1'].CopyFrom(
tf.make_tensor_proto(data))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
print('Created the file \'%s\', restart tensorflow_model_server to warmup '
'the ResNet SavedModel.' % warmup_file)
def main(argv):
count = 0
images = []
files = [path.join(path.join(FLAGS.dataset, f))
for f in os.listdir(FLAGS.dataset)
if path.isfile(path.join(FLAGS.dataset, f))
]
files = [f for f in files if f.endswith(('.png', '.jpg', '.jpeg'))]
for file in files:
img_raw = tf.image.decode_image(open(file, 'rb').read(), channels=3)
image = preprocess_image(img_raw, FLAGS.input_size)
image = tf.expand_dims(image, 0)
images.append(image)
count += 1
if count == FLAGS.size:
break
input_tensor = tf.concat(images, 0)
with tf.io.TFRecordWriter('tf_serving_warmup_requests') as writer:
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(
name=FLAGS.model_name
),
inputs={
FLAGS.input_tensor: tf.make_tensor_proto(
input_tensor,
shape=input_tensor.shape,
dtype=input_tensor.dtype
)
}
)
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request)
)
writer.write(log.SerializeToString())
logging.info('"tf_serving_warmup_requests" created with success!')
logging.info('to use it paste it to the "<model>/<version>/assets.extra" folder on the serving configuration folder')
def main():
"""Generate TFRecords for warming up."""
with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer:
with open(IMAGE_PATH, 'rb') as f:
# image_data = base64.b64encode(f.read()).decode('utf-8')
image_data = f.read()
predict_request = predict_pb2.PredictRequest()
predict_request.model_spec.name = 'resnet'
predict_request.model_spec.signature_name = 'serving_default'
predict_request.inputs['string_inp'].CopyFrom(
tensor_util.make_tensor_proto(image_data, shape=[
1,
]))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=predict_request))
for r in range(NUM_RECORDS):
writer.write(log.SerializeToString())
def build_warmup_data(savemodel_dir, config_dict):
serving_batch = config_dict['SERVING']['SERVE_BATCH']
model_spec_name = config_dict['SERVING']['NAME']
output_paras = config_dict['OUTPUT']
experiment_base_dir = os.path.join(output_paras['OUTPUT_SAVE_DIR'],
output_paras['EXPERIMENT_NAME'])
model_save_dir = os.path.join(experiment_base_dir, 'weights')
label_file = os.path.join(model_save_dir, 'labels.txt')
trainset_root_dir = config_dict['DATASET']['DATASET_ROOT_DIR']
warmup_data_dir = _get_warmup_data_dir(trainset_root_dir, label_file)
assets_extra_dir = os.path.join(savemodel_dir, 'assets.extra')
mkdir_if_nonexist(assets_extra_dir, raise_error=False)
warmup_save_path = os.path.join(assets_extra_dir,
'tf_serving_warmup_requests')
# load warmup images
cnt = 1
image_str_list = []
for img_name in os.listdir(warmup_data_dir):
if cnt > serving_batch:
break
img_path = os.path.join(warmup_data_dir, img_name)
image_str = (open(img_path, 'rb').read())
image_str_list.append(image_str)
cnt += 1
with tf.python_io.TFRecordWriter(warmup_save_path) as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = model_spec_name
request.model_spec.signature_name = "serving_default"
#request.inputs['input'].ParseFromString(tf.make_tensor_proto(image_str, shape=[1]).SerializeToString())
request.inputs['input'].CopyFrom(
tf.contrib.util.make_tensor_proto(image_str_list, dtype=tf.string))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def main():
with tf.io.TFRecordWriter("tf_serving_warmup_requests") as writer:
# replace <request> with one of:
#
# classification_pb2.ClassificationRequest(..)
# regression_pb2.RegressionRequest(..)
# inference_pb2.MultiInferenceRequest(..)
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(name="youtube_dnn", signature_name="serving_default"),
inputs={
"sample": tf.compat.v1.make_tensor_proto([[1, 2, 3, 4, 5]], dtype=tf.int64),
"label": tf.compat.v1.make_tensor_proto([[0]], dtype=tf.int64)
}
)
print(request)
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
for r in range(100):
writer.write(log.SerializeToString())
writer.write(log.SerializeToString())
def main():
### Generate TFRecords for warming up. ###
global filename
for root, dirs, files in os.walk(img_path):
for img_file in files:
if img_file.endswith(".png") or img_file.endswith(".bmp") or img_file.endswith(".jpg"):
filename = os.path.join(root, img_file)
break
print(filename)
with tf.io.TFRecordWriter(f"{target_path}tf_serving_warmup_requests") as writer:
# replace <request> with one of:
# predict_pb2.PredictRequest(..)
# classification_pb2.ClassificationRequest(..)
# regression_pb2.RegressionRequest(..)
# inference_pb2.MultiInferenceRequest(..)
### Method 1 ###
if model_origin == "saiap":
request = predict_pb2.PredictRequest()
request.model_spec.name = MODEL_NAME
request.model_spec.signature_name = SIG
request.inputs[INPUTS].CopyFrom(
tensor_util.make_tensor_proto([saiap_open_image(filename)], tf.string)
)
elif model_origin == "wzsda":
## Method 2 ###
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(name=MODEL_NAME, signature_name=SIG),
inputs={
INPUTS: tensor_util.make_tensor_proto([open_image(filename)], tf.string),
"degree": tensor_util.make_tensor_proto(["0"], tf.string)
}
)
else:
print(f"arg 'model_origin' {model_origin} is not supported!")
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def write_warmup_requests(savedmodel_dir,
model_name,
image_size,
batch_sizes=None,
num_requests=8):
"""Writes warmup requests for inference into a tfrecord file.
Args:
savedmodel_dir: string, the file to the exported model folder.
model_name: string, a model name used inside the model server.
image_size: int, size of image, assuming image height and width.
batch_sizes: list, a list of batch sizes to create different input requests.
num_requests: int, number of requests per batch size.
Raises:
ValueError: if batch_sizes is not a valid integer list.
"""
if not isinstance(batch_sizes, list) or not batch_sizes:
raise ValueError('batch sizes should be a valid non-empty list.')
extra_assets_dir = os.path.join(savedmodel_dir, 'assets.extra')
tf.gfile.MkDir(extra_assets_dir)
with tf.python_io.TFRecordWriter(
os.path.join(extra_assets_dir,
'tf_serving_warmup_requests')) as writer:
for batch_size in batch_sizes:
for _ in range(num_requests):
request = predict_pb2.PredictRequest()
image = np.uint8(
np.random.rand(image_size, image_size, 3) * 255)
request.inputs['input'].CopyFrom(
tf.make_tensor_proto([_encode_image(image)] * batch_size,
shape=[batch_size]))
request.model_spec.name = model_name
request.model_spec.signature_name = 'serving_default'
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def export_serving_model(yolo, path):
if tf.version.VERSION.startswith('1.'):
tf.saved_model.simple_save(
yolo.sess,
path,
inputs={'predict_image:0': yolo.input},
outputs={t.name: t
for t in yolo.yolo_model.output})
else:
signature_def_map = {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.saved_model.signature_def_utils.predict_signature_def(
{'predict_image:0': yolo.input},
{t.name: t
for t in yolo.yolo_model.output})
}
tf.saved_model.experimental.save(yolo.yolo_model, path,
signature_def_map)
asset_extra = os.path.join(path, "assets.extra")
tf.io.gfile.mkdir(asset_extra)
with tf.io.TFRecordWriter(
os.path.join(asset_extra, "tf_serving_warmup_requests")) as writer:
request = predict_pb2.PredictRequest()
request.model_spec.name = 'detection'
request.model_spec.signature_name = 'serving_default'
image = Image.open('../download/image3.jpeg')
scale = yolo.input_shape[0] / max(image.size)
if scale < 1:
image = image.resize((int(line * scale) for line in image.size),
Image.BILINEAR)
image_data = np.array(image, dtype='uint8')
image_data = np.expand_dims(image_data, 0)
request.inputs['predict_image:0'].CopyFrom(
tf.make_tensor_proto(image_data))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def main():
if len(sys.argv) != 2 or sys.argv[-1].startswith('-'):
print('Usage: resnet_warmup.py saved_model_dir')
sys.exit(-1)
model_dir = sys.argv[-1]
if not os.path.isdir(model_dir):
print('The saved model directory: %s does not exist. '
'Specify the path of an existing model.' % model_dir)
sys.exit(-1)
# Create the assets.extra directory
assets_dir = os.path.join(model_dir, 'assets.extra')
if not os.path.exists(assets_dir):
os.mkdir(assets_dir)
warmup_file = os.path.join(assets_dir, 'tf_serving_warmup_requests')
with tf.io.TFRecordWriter(warmup_file) as writer:
for image in IMAGE_URLS:
# Download the image
dl_request = requests.get(image, stream=True)
dl_request.raise_for_status()
data = dl_request.content
# Create the inference request
request = predict_pb2.PredictRequest()
request.model_spec.name = 'resnet'
request.model_spec.signature_name = 'serving_default'
request.inputs['image_bytes'].CopyFrom(
tf.make_tensor_proto(data, shape=[1]))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
print('Created the file \'%s\', restart tensorflow_model_server to warmup '
'the ResNet SavedModel.' % warmup_file)
import tensorflow as tf
from tensorflow_serving.apis import classification_pb2
from tensorflow_serving.apis import inference_pb2
from tensorflow_serving.apis import model_pb2
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_log_pb2
from tensorflow_serving.apis import regression_pb2
with tf.io.TFRecordWriter("../testdata/tf_serving_warmup_requests") as writer:
predict_request = predict_pb2.PredictRequest()
predict_request.model_spec.name = 'some_model'
predict_request.model_spec.signature_name = 'some_signature'
example_proto = tf.train.Example(features=tf.train.Features(feature={}))
predict_request.inputs['inputs'].CopyFrom(
tf.make_tensor_proto([example_proto.SerializeToString()]))
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=predict_request))
writer.write(log.SerializeToString())
def export(model_dir, vocab_dir, max_length, theta, emb_size, dim,
num_oov_buckets):
while True:
cur = os.path.join(model_dir, str(int(time.time())))
if not tf.gfile.Exists(cur):
break
print("export model path: %s" % cur)
method_name = tf.saved_model.signature_constants.PREDICT_METHOD_NAME
rowkeys_file = os.path.join(vocab_dir, "rowkeys.vocab")
embedding_file = os.path.join(vocab_dir, "emb.vocab")
with tf.Graph().as_default(), tf.Session() as sess:
rowkeys = tf.placeholder(dtype=tf.string, name="rowkeys")
algorithm_ids = tf.placeholder(dtype=tf.uint32, name="algorithm_id")
scores = tf.placeholder(dtype=tf.float32, name="scores")
# rowkeys = tf.constant(["2785c0f6f0e592ah",
# "8605d35a21f857bk",
# "7915d39c6f9755ap",
# "3155d3846cb468bk",
# "4285d39597b375bk"], dtype=tf.string)
# algorithm_ids = tf.constant([2081,
# 2803,
# 2086,
# 2803,
# 2086], dtype=tf.uint32)
# scores = tf.constant([
# 0.1,
# 0.2,
# 0.3,
# 0.11,
# 0.7
# ], dtype=float)
with open(rowkeys_file, encoding="utf8") as fi:
lines = fi.readlines()
vocab_size = len(lines)
print(vocab_size)
emb = tf.Variable(np.loadtxt(embedding_file, delimiter=' '),
dtype=tf.float32)
print(emb.shape)
table = tf.contrib.lookup.index_table_from_file(
vocabulary_file=rowkeys_file,
vocab_size=vocab_size - num_oov_buckets,
hasher_spec=tf.contrib.lookup.FastHashSpec,
num_oov_buckets=num_oov_buckets)
rowkey_ids = table.lookup(rowkeys)
rowkeys_embedding = tf.nn.embedding_lookup(emb, rowkey_ids)
rowkeys_embedding /= tf.linalg.norm(rowkeys_embedding,
axis=1,
keepdims=True)
rowkeys_embedding = tf.where(tf.is_nan(rowkeys_embedding),
tf.zeros_like(rowkeys_embedding),
rowkeys_embedding)
similarities = tf.cast(
tf.matmul(rowkeys_embedding, rowkeys_embedding, transpose_b=True),
tf.float32)
kernel_matrix = tf.reshape(
scores, [-1, 1]) * similarities * tf.reshape(scores, [1, -1])
# alpha = theta / (2 * (1 - theta))
# kernel_matrix = tf.math.exp(alpha * tf.reshape(scores, [-1, 1])) * similarities * tf.math.exp(alpha * tf.reshape(scores, [1, -1]))
indices = dpp(kernel_matrix, max_length)
predict_rowkeys = tf.gather(rowkeys, indices)
predict_scores = tf.gather(scores, indices)
predict_algorithm_ids = tf.gather(algorithm_ids, indices)
predict_positions = indices
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
#sess.run(emb, feed_dict={embedding_placeholder: emb_dict})
# print(sess.run(predict_rowkeys))
# print(sess.run(predict_scores))
signature_def_map = signature({
"prediction": {
"inputs": {
'rowkeys': rowkeys,
"scores": scores,
"algorithm_ids": algorithm_ids
},
"outputs": {
"rowkeys": predict_rowkeys,
"scores": predict_scores,
"algorithm_ids": predict_algorithm_ids,
"origin_position": predict_positions
},
"method_name": method_name
},
})
builder = tf.saved_model.builder.SavedModelBuilder(cur)
builder.add_meta_graph_and_variables(
sess,
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=ops.get_collection(
ops.GraphKeys.ASSET_FILEPATHS),
main_op=tf.tables_initializer())
builder.save()
os.mkdir(os.path.join(cur, "assets.extra"))
with tf.python_io.TFRecordWriter(
os.path.join(
cur, "assets.extra/tf_serving_warmup_requests")) as writer:
request = predict_pb2.PredictRequest(
model_spec=model_pb2.ModelSpec(name="kva_dpp_model",
signature_name="prediction"),
inputs={
"rowkeys":
tf.make_tensor_proto(
["2785c0f6f0e592ah", "2785c0f6f0e592ah"],
dtype=tf.string),
"algorithm_ids":
tf.make_tensor_proto([2081, 2081], dtype=tf.uint32),
"scores":
tf.make_tensor_proto([0.7, 0.7], dtype=tf.float32)
})
print(request)
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
