def text(self, tag, textdata, step):
"""Saves a text summary.
Args:
tag: str: label for this data
textdata: string, or 1D/2D list/numpy array of strings
step: int: training step
Note: markdown formatting is rendered by tensorboard.
"""
smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData(
plugin_name='text'))
if isinstance(textdata, (str, bytes)):
tensor = tf.make_tensor_proto(
values=[textdata.encode(encoding='utf_8')], shape=(1, ))
else:
textdata = onp.array(textdata) # convert lists, jax arrays, etc.
datashape = onp.shape(textdata)
if len(datashape) == 1:
tensor = tf.make_tensor_proto(
values=[td.encode(encoding='utf_8') for td in textdata],
shape=(datashape[0], ))
elif len(datashape) == 2:
tensor = tf.make_tensor_proto(values=[
td.encode(encoding='utf_8')
for td in onp.reshape(textdata, -1)
],
shape=(datashape[0],
datashape[1]))
summary = Summary(
value=[Summary.Value(tag=tag, metadata=smd, tensor=tensor)])
self._add_summary(summary, step)
def denoise_images(self, images, timeout_per_request=10):
processed_images = [preprocess_unet(img) for img in images]
request_data_list = [
make_tensor_proto(np.array([pr_img]).astype(np.float32),
shape=[1, 224, 64, 3])
for pr_img in processed_images
]
future_responses = list()
request = predict_pb2.PredictRequest()
request.model_spec.signature_name = 'serving_default'
request.model_spec.name = 'unet'
for request_data in request_data_list:
request.inputs['input_images'].CopyFrom(request_data)
future_responses.append(
self.stub.Predict.future(request, timeout_per_request))
# Chờ lấy kết quả
response_objects = [r.result() for r in future_responses]
# Decode kết quả trả về
final_result = list()
for result_obj in response_objects:
# Decode tensor shape
tf_shape_grpc_obj_list = result_obj.outputs[
'predictions'].tensor_shape.dim
resp_tensor_shape = [tmp.size for tmp in tf_shape_grpc_obj_list]
# Decode response result
model_result_flatten = np.array(
result_obj.outputs['predictions'].float_val._values)
model_result = np.reshape(model_result_flatten, resp_tensor_shape)
final_result.append(np.squeeze(model_result, axis=0))
final_result = np.stack(final_result)
return final_result
def _python_type_to_attr_list_elem(
list_value, # type: tf.AttrValue.ListValue
elem, # type: Any
attr_name # type: String
):
"""
Subroutine of python_type_to_attr_value(). Converts one element of a Python
list to one element of a `tf.AttrValue.ListValue` protobuf.
Args:
list_value: ListValue proto being populated for use within an AttrValue
proto. Modified in place.
elem: Original value to convert.
"""
if isinstance(elem, string_types):
list_value.s.append(tf.compat.as_bytes(elem))
# Must check for bool before int because bool is a subclass of int in Python
elif isinstance(elem, bool):
list_value.b.append(elem)
elif isinstance(elem, int):
list_value.i.append(elem)
elif isinstance(elem, float):
list_value.f.append(elem)
elif isinstance(elem, tf.DType):
list_value.type.append(elem.as_datatype_enum)
elif isinstance(elem, tf.TensorShape):
list_value.shape.add().CopyFrom(elem.as_proto())
elif isinstance(elem, np.ndarray) or isinstance(elem, list):
list_value.tensor.add().CopyFrom(tf.make_tensor_proto(values=elem))
# TODO(frreiss): Populate the "func" field of the union here
else:
raise ValueError("Don't know how to convert a {} to "
"tf.AttrValue.ListValue for attribute {}".format(
type(elem), attr_name))
def do_inference(hostport, work_dir, concurrency, num_tests):
"""Tests PredictionService with concurrent requests.
Args:
hostport: Host:port address of the PredictionService.
work_dir: The full path of working directory for test data set.
concurrency: Maximum number of concurrent requests.
num_tests: Number of test images to use.
Returns:
The classification error rate.
Raises:
IOError: An error occurred processing test data set.
"""
test_data_set = mnist_input_data.read_data_sets(work_dir).test
channel = grpc.insecure_channel(hostport)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
result_counter = _ResultCounter(num_tests, concurrency)
for _ in range(num_tests):
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mnist'
request.model_spec.signature_name = 'predict_images'
image, label = test_data_set.next_batch(1)
request.inputs['images'].CopyFrom(
tf.make_tensor_proto(image[0], shape=[1, image[0].size]))
result_counter.throttle()
result_future = stub.Predict.future(request, 5.0) # 5 seconds
result_future.add_done_callback(
_create_rpc_callback(label[0], result_counter))
return result_counter.get_error_rate()
def pb(scalars_layout):
"""Creates a summary that contains a layout.
When users navigate to the custom scalars dashboard, they will see a layout
based on the proto provided to this function.
Args:
scalars_layout: The scalars_layout_pb2.Layout proto that specifies the
layout.
Returns:
A summary proto containing the layout.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
assert isinstance(scalars_layout, layout_pb2.Layout)
tensor = tf.make_tensor_proto(
scalars_layout.SerializeToString(), dtype=tf.string)
tf_summary_metadata = tf.SummaryMetadata.FromString(
metadata.create_summary_metadata().SerializeToString())
summary = tf.Summary()
summary.value.add(tag=metadata.CONFIG_SUMMARY_TAG,
metadata=tf_summary_metadata,
tensor=tensor)
return summary
def _CreatePredictResponse(values: List[float],
shape: List[int]) -> predict_pb2.PredictResponse:
"""Returns a PredictResponse used in below unit tests."""
tensor_proto = tf.make_tensor_proto([], shape=shape, dtype=np.float32)
tensor_proto.float_val.extend(values)
resp = predict_pb2.PredictResponse(
outputs={_MODEL_CONFIG.response[0].key: tensor_proto})
return resp
def generate_grpc_request_from_dictionary(self, row_dict):
"""Generate gRPC inference request with payload."""
request = predict_pb2.PredictRequest()
request.model_spec.name = self._model_name
request.model_spec.signature_name = self._signature_name
for key, value in row_dict.items():
proto = None
if self._default_int_type and (clients.utils.is_subtype(value, int) or
"/values" in key or "/indices" in key):
proto = tf.make_tensor_proto(value, dtype=self._default_int_type)
elif self._default_float_type and clients.utils.is_subtype(value, float):
proto = tf.make_tensor_proto(value, dtype=self._default_float_type)
else:
proto = tf.make_tensor_proto(value)
request.inputs[key].CopyFrom(proto)
return request
def raw_data_pb(name,
true_positive_counts,
false_positive_counts,
true_negative_counts,
false_negative_counts,
precision,
recall,
num_thresholds=None,
display_name=None,
description=None):
"""Create a PR curves summary protobuf from raw data values.
Args:
name: A tag attached to the summary. Used by TensorBoard for organization.
true_positive_counts: A rank-1 numpy array of true positive counts. Must
contain `num_thresholds` elements and be castable to float32.
false_positive_counts: A rank-1 numpy array of false positive counts. Must
contain `num_thresholds` elements and be castable to float32.
true_negative_counts: A rank-1 numpy array of true negative counts. Must
contain `num_thresholds` elements and be castable to float32.
false_negative_counts: A rank-1 numpy array of false negative counts. Must
contain `num_thresholds` elements and be castable to float32.
precision: A rank-1 numpy array of precision values. Must contain
`num_thresholds` elements and be castable to float32.
recall: A rank-1 numpy array of recall values. Must contain `num_thresholds`
elements and be castable to float32.
num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to
compute PR metrics for. Should be an int `>= 2`.
display_name: Optional name for this summary in TensorBoard, as a `str`.
Defaults to `name`.
description: Optional long-form description for this summary, as a `str`.
Markdown is supported. Defaults to empty.
Returns:
A summary operation for use in a TensorFlow graph. See docs for the `op`
method for details on the float32 tensor produced by this summary.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name if display_name is not None else name,
description=description or '',
num_thresholds=num_thresholds)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString())
summary = tf.Summary()
data = np.stack(
(true_positive_counts, false_positive_counts, true_negative_counts,
false_negative_counts, precision, recall))
tensor = tf.make_tensor_proto(np.float32(data), dtype=tf.float32)
summary.value.add(tag='%s/pr_curves' % name,
metadata=tf_summary_metadata,
tensor=tensor)
return summary
def generate_grpc_request_from_tfrecord(self, tfrecord_row):
"""Generate gRPC inference request with payload."""
request = predict_pb2.PredictRequest()
request.model_spec.name = self._model_name
request.model_spec.signature_name = self._signature_name
request.inputs[self._input_name].CopyFrom(
tf.make_tensor_proto(tfrecord_row, dtype=tf.string))
return request
def make_request(self, input_batch) -> PredictRequest:
input_batch = np.stack(input_batch)
request = PredictRequest()
request.model_spec.name = self.model_info.architecture
request.model_spec.signature_name = self.signature_name
for input_ in self.model_info.inputs:
tensor_proto = tf.make_tensor_proto(input_batch, shape=input_batch.shape)
request.inputs[input_.name].CopyFrom(tensor_proto)
return request
def main(_):
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Send request
request = predict_pb2.PredictRequest()
request.model_spec.name = 'half_plus_two_with_rpop'
request.model_spec.signature_name = 'serving_default'
request.inputs['x'].CopyFrom(tf.make_tensor_proto([10.0], shape=[1]))
result = stub.Predict(request, 30)
print(result)
def pb(name, data, bucket_count=None, display_name=None, description=None):
"""Create a legacy histogram summary protobuf.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
data: A `np.array` or array-like form of any shape. Must have type
castable to `float`.
bucket_count: Optional positive `int`. The output will have this
many buckets, except in two edge cases. If there is no data, then
there are no buckets. If there is data but all points have the
same value, then there is one bucket whose left and right
endpoints are the same.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
if bucket_count is None:
bucket_count = summary_v2.DEFAULT_BUCKET_COUNT
data = np.array(data).flatten().astype(float)
if data.size == 0:
buckets = np.array([]).reshape((0, 3))
else:
min_ = np.min(data)
max_ = np.max(data)
range_ = max_ - min_
bucket_num = tf.minimum(bucket_count, data.size()[0])
left_edges = tf.linspace(0, bucket_num - 1, bucket_num)
right_edges = tf.linspace(1, bucket_num, bucket_num)
buckets = np.array([left_edges, right_edges,
data[0:bucket_num]]).transpose()
tensor = tf.make_tensor_proto(
buckets, dtype=tf.float64) #dtype是float，那么需要查询bucketcounts怎么绘制，是否需要int
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString())
summary = tf.Summary()
summary.value.add(tag='%s/bar_summary' % name,
metadata=tf_summary_metadata,
tensor=tensor)
return summary
def python_type_to_attr_value(
value, #type: Any
attr_name #type: String
):
# type (...) -> tf.AttrValue
"""
Convert a Python object or scalar value to a TensorFlow `tf.AttrValue`
protocol buffer message.
Args:
value: Python object to be converted
Returns:
An AttrValue object that wraps the contents of `value` in the most
appropriate way available.
"""
if isinstance(value, list) or isinstance(value, tuple):
if 0 == len(value):
return tf.AttrValue(list=tf.AttrValue.ListValue())
else:
# Nonempty list
list_value = tf.AttrValue.ListValue()
for elem in value:
# TODO(frreiss): Should we disallow heterogeneous types in lists?
_python_type_to_attr_list_elem(list_value, elem, attr_name)
return tf.AttrValue(list=list_value)
elif isinstance(value, tf.AttrValue):
# TODO(frreiss): Should this case result in an error?
return value
# Scalar types, in the order they appear in the .proto file
elif isinstance(value, string_types):
return tf.AttrValue(s=tf.compat.as_bytes(value))
# Must check for bool before int because bool is a subclass of int in Python
elif isinstance(value, bool):
return tf.AttrValue(b=value)
elif (isinstance(value, int) or isinstance(value, np.int32)
or isinstance(value, np.int64)):
return tf.AttrValue(i=value)
elif isinstance(value, float) or isinstance(value, np.float):
return tf.AttrValue(f=value)
elif isinstance(value, tf.DType):
return tf.AttrValue(type=value.as_datatype_enum)
elif isinstance(value, np.dtype):
return tf.AttrValue(type=tf.as_dtype(value).as_datatype_enum)
elif isinstance(value, tf.TensorShape):
return tf.AttrValue(shape=value.as_proto())
elif isinstance(value, np.ndarray):
return tf.AttrValue(tensor=tf.make_tensor_proto(values=value))
# TODO(frreiss): Populate the "func" and "placeholder" fields of the union
# here
else:
raise ValueError("Don't know how to convert a {} to "
"tf.AttrValue for attribute {}".format(
type(value), attr_name))
def write_warmup_requests(savedmodel_dir,
model_name,
image_size,
batch_sizes=None,
num_requests=8,
image_format='PNG',
input_signature='input'):
"""Writes warmup requests for inference into a tfrecord file.
Args:
savedmodel_dir: string, the file to the exported model folder.
model_name: string, a model name used inside the model server.
image_size: tuple/list or int, size of image. For list/tuple input, assuming
it contains image height and width.
batch_sizes: list, a list of batch sizes to create different input requests.
num_requests: int, number of requests per batch size.
image_format: string, the format of the image to write (PNG, JPEG)
input_signature: string, input signature defined in exported saved model.
Raises:
ValueError: if batch_sizes is not a valid integer list.
"""
from tensorflow_serving.apis import predict_pb2 # pylint: disable=g-import-not-at-top
from tensorflow_serving.apis import prediction_log_pb2 # pylint: disable=g-import-not-at-top
if not isinstance(batch_sizes, list) or not batch_sizes:
raise ValueError('batch sizes should be a valid non-empty list.')
extra_assets_dir = os.path.join(savedmodel_dir, 'assets.extra')
tf.gfile.MkDir(extra_assets_dir)
if isinstance(image_size, int):
height = image_size
width = image_size
elif isinstance(image_size, tuple) or isinstance(image_size, list):
height = image_size[0]
width = image_size[1]
else:
raise ValueError('image_size is not a supported type: %s' %
type(image_size))
with tf.python_io.TFRecordWriter(
os.path.join(extra_assets_dir,
'tf_serving_warmup_requests')) as writer:
for batch_size in batch_sizes:
for _ in range(num_requests):
request = predict_pb2.PredictRequest()
image = np.uint8(np.random.rand(height, width, 3) * 255)
request.inputs[input_signature].CopyFrom(
tf.make_tensor_proto([_encode_image(image, image_format)] *
batch_size,
shape=[batch_size]))
request.model_spec.name = model_name
request.model_spec.signature_name = 'serving_default'
log = prediction_log_pb2.PredictionLog(
predict_log=prediction_log_pb2.PredictLog(request=request))
writer.write(log.SerializeToString())
def pb(name, images, max_outputs=3, display_name=None, description=None):
"""Create a legacy image summary protobuf.
This behaves as if you were to create an `op` with the same arguments
(wrapped with constant tensors where appropriate) and then execute
that summary op in a TensorFlow session.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
images: An `np.array` representing pixel data with shape
`[k, h, w, c]`, where `k` is the number of images, `w` and `h` are
the width and height of the images, and `c` is the number of
channels, which should be 1, 3, or 4.
max_outputs: Optional `int`. At most this many images will be
emitted. If more than this many images are provided, the first
`max_outputs` many images will be used and the rest silently
discarded.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
images = np.array(images).astype(np.uint8)
if images.ndim != 4:
raise ValueError("Shape %r must have rank 4" % (images.shape, ))
limited_images = images[:max_outputs]
encoded_images = [encoder.encode_png(image) for image in limited_images]
(width, height) = (images.shape[2], images.shape[1])
content = [str(width), str(height)] + encoded_images
tensor = tf.make_tensor_proto(content, dtype=tf.string)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString())
summary = tf.Summary()
summary.value.add(
tag="%s/image_summary" % name,
metadata=tf_summary_metadata,
tensor=tensor,
)
return summary
def generate_grpc_request():
"""Generate gRPC inference request with payload."""
request = predict_pb2.PredictRequest()
request.model_spec.name = FLAGS.model_name
request.model_spec.signature_name = 'serving_default'
image = get_image_payload()
request.inputs[FLAGS.input_name].CopyFrom(
tf.make_tensor_proto([image] * FLAGS.batch_size,
shape=[FLAGS.batch_size]))
return request
def change_T(new_node, *attr):
for a in attr:
if a in ("T", 'DstT', 'Tindices', 'TI'):
new_node.attr[a].type = types_pb2.DT_INT32
elif a in ("dtype", ):
new_node.attr['dtype'].CopyFrom(
attr_value_pb2.AttrValue(type=types_pb2.DT_INT32))
elif a in ("value", ):
values = tf.make_ndarray(new_node.attr['value'].tensor).astype(
np.int32)
new_node.attr['value'].tensor.CopyFrom(
tf.make_tensor_proto(values, dtype=tf.int32))
def main(_):
samples={"age":[37.0,37.0],"workclass":["Private","Private"],"education":["Some-college","Some-college"],"education-num":["10","10"],"marital-status":["Married-civ-spouse","Married-civ-spouse"],"occupation":["Exec-managerial","Exec-managerial"],"relationship":["Husband","Husband"],"race":["Black","Black"],"sex":["Male","Male"],"capital-gain":[0.0,10.0],"capital-loss":[0.0,100.0],"hours-per-week":[80.0,45.0],"native-country":["United-States","United-States"]}
channel = grpc.insecure_channel(FLAGS.server)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Send request
# See prediction_service.proto for gRPC request/response details.
request = predict_pb2.PredictRequest()
request.model_spec.name = 'deepfm'
request.model_spec.signature_name = 'serving_default'
for key,value in samples.items():
request.inputs[key].CopyFrom(
tf.make_tensor_proto(value, shape=[2]))
result = stub.Predict(request, 10.0) # 10 secs timeout
print(result)
def caption_example(image):
"""Convert image caption data into an Example proto.
Args:
image: A ImageMetadata instance.
Returns:
example: An Example proto with serialized tensor data.
"""
# Collect image object information from metadata.
image_features, positions = read_object(image.objects, image.image_id)
# Serialize multi-dimensional tensor data.
captions_proto = tf.make_tensor_proto(np.array(image.captions))
features_proto = tf.make_tensor_proto(image_features)
positions_proto = tf.make_tensor_proto(positions)
# Create final features dict.
features = dict(
image_id=int64_feature(image.image_id),
captions=bytes_feature(captions_proto.SerializeToString()),
object_features=bytes_feature(features_proto.SerializeToString()),
object_positions=bytes_feature(positions_proto.SerializeToString()))
return tf.train.Example(features=tf.train.Features(feature=features))
def _make_grpc_request(examples):
"""Builds and sends request to TensorFlow model server."""
request = predict_pb2.PredictRequest()
request.model_spec.name = servable_name
request.inputs["input"].CopyFrom(
tf.make_tensor_proto([ex.SerializeToString() for ex in examples],
shape=[len(examples)]))
response = stub.Predict(request, timeout_secs)
outputs = tf.make_ndarray(response.outputs["outputs"])
scores = tf.make_ndarray(response.outputs["scores"])
assert len(outputs) == len(scores)
return [
{ # pylint: disable=g-complex-comprehension
"outputs": output,
"scores": score
} for output, score in zip(outputs, scores)
]
def _predict_using_grpc(context, instance):
request = predict_pb2.PredictRequest()
request.model_spec.name = 'model'
request.model_spec.signature_name = 'serving_default'
request.inputs['input_1'].CopyFrom(make_tensor_proto(instance))
options = [('grpc.max_send_message_length', MAX_GRPC_MESSAGE_LENGTH),
('grpc.max_receive_message_length', MAX_GRPC_MESSAGE_LENGTH)]
channel = grpc.insecure_channel(f'0.0.0.0:{context.grpc_port}',
options=options)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
result_future = stub.Predict.future(request, 30) # 5 seconds
output_tensor_proto = result_future.result().outputs['output']
output_shape = [dim.size for dim in output_tensor_proto.tensor_shape.dim]
output_np = np.array(output_tensor_proto.float_val).reshape(output_shape)
predicted_class_idx = argmax(output_np)
print(f' Predicted class: {predicted_class_idx}')
prediction_json = {'predictions': output_np.tolist()}
return json.dumps(prediction_json)
def pb(name, data, display_name=None, description=None):
"""Create a legacy text summary protobuf.
Arguments:
name: A name for the generated node. Will also serve as a series name in
TensorBoard.
data: A Python bytestring (of type bytes), or Unicode string. Or a numpy
data array of those types.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Raises:
ValueError: If the type of the data is unsupported.
Returns:
A `tf.Summary` protobuf object.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
try:
tensor = tf.make_tensor_proto(data, dtype=tf.string)
except TypeError as e:
raise ValueError(e)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description
)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString()
)
summary = tf.Summary()
summary.value.add(
tag="%s/text_summary" % name,
metadata=tf_summary_metadata,
tensor=tensor,
)
return summary
def pb(name, data, display_name=None, description=None):
"""Create a legacy scalar summary protobuf.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
data: A rank-0 `np.array` or array-like form (so raw `int`s and
`float`s are fine, too).
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
data = np.array(data)
if data.shape != ():
raise ValueError("Expected scalar shape for data, saw shape: %s." %
data.shape)
if data.dtype.kind not in ("b", "i", "u", "f"): # bool, int, uint, float
raise ValueError("Cast %s to float is not supported" % data.dtype.name)
tensor = tf.make_tensor_proto(data.astype(np.float32))
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString())
summary = tf.Summary()
summary.value.add(
tag="%s/scalar_summary" % name,
metadata=tf_summary_metadata,
tensor=tensor,
)
return summary
def _Predict(self, input_data: Any, num_inputs: int,
model_config: ModelConfig) -> List[List[float]]:
"""Get prediction from the model server.
Send the provided input data to the model server over gRPC and returns
the response.
Args:
input_data: Input data fed into the model.
num_inputs: Number of model inputs (should be set to 1 for non-batch).
model_config: Configuration for the model to be called.
Returns:
The prediction response from TF serving.
Raises:
exception.CustomExceptionError: If the shape of response is incompatible
or if the model server returns an error and it sets the status code to
INTERNAL.
"""
try:
req = predict_pb2.PredictRequest()
req.model_spec.name = model_config.name
req.model_spec.signature_name = model_config.signature
req.inputs[model_config.input_key].CopyFrom(
tf.make_tensor_proto(input_data, shape=[num_inputs]))
resp = self._InvokePredictRequest(req)
except Exception as e:
raise exception.CustomExceptionError(str(e),
code_pb2.Code.INTERNAL)
floats = []
for r in model_config.response:
value = resp.outputs[r.key]
shape = tensor_util.TensorShapeProtoToList(value.tensor_shape)
if not tf.TensorShape(shape).is_compatible_with(r.shape):
raise exception.CustomExceptionError(
'Model returned invalid shape {}, want {}'.format(
shape, r.shape), code_pb2.Code.INTERNAL)
floats.append(value.float_val[:])
return floats
def batch_predict(self, inputs, version=None):
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_name
request.model_spec.signature_name = self.signature_name
if version: request.model_spec.version.value = version
for name, data in inputs.items():
request.inputs[name].CopyFrom(
make_tensor_proto(data,
shape=list(data.shape),
dtype=data.dtype))
try:
response = self.prediction_service_stub.Predict(request,
timeout=10.0)
except:
import traceback
traceback.print_exc()
return None
results = {
key: tf.make_ndarray(tensor_proto)
for key, tensor_proto in response.outputs.items()
}
return results
def strip_pruning_vars_fn(input_graph_def, output_node_names):
"""Removes mask variable from the graph.
Replaces the masked_weight tensor with element-wise multiplication of mask
and the corresponding weight variable.
Args:
input_graph_def: A GraphDef in which the variables have been converted to
constants. This is typically the output of
tf.graph_util.convert_variables_to_constant()
output_node_names: List of name strings for the result nodes of the graph
Returns:
A GraphDef in which pruning-related variables have been removed
"""
masked_weights_dict = _get_masked_weights(input_graph_def)
pruned_graph_def = tf.GraphDef()
# Replace masked_weight with a const op containing the
# result of tf.multiply(mask,weight)
for node in input_graph_def.node:
output_node = tf.NodeDef()
if 'masked_weight' in node.name:
output_node.op = 'Const'
output_node.name = node.name
dtype = node.attr['T']
data = masked_weights_dict[node.name]
output_node.attr['dtype'].CopyFrom(dtype)
output_node.attr['value'].CopyFrom(
tf.AttrValue(tensor=tf.make_tensor_proto(data)))
else:
output_node.CopyFrom(node)
pruned_graph_def.node.extend([output_node])
# Remove stranded nodes: mask and weights
return tf.graph_util.extract_sub_graph(pruned_graph_def, output_node_names)
def _predict_request(
data: Union[ORIGINAL_DATA_TYPE, NEW_DATA_TYPE],
model_name: str,
output_names=None,
model_signature_name=None,
):
req = predict_pb2.PredictRequest()
req.model_spec.name = model_name
if model_signature_name is not None:
req.model_spec.signature_name = model_signature_name
for datum in data:
if isinstance(datum, PredictInput):
# old way to pass input
name = datum.name
value = datum.value
else:
name = datum
value = data[datum]
copy_message(tf.make_tensor_proto(value), req.inputs[name])
if output_names is not None:
for output_name in output_names:
req.output_filter.append(output_name)
return req
def vqa_example(image):
"""Convert visual qa data into an Example proto.
Args:
image: An ImageMetadata instance.
Returns:
example: An Example proto with serialized tensor data.
"""
# Collect image object information from metadata.
image_features, positions = read_object(image.objects, image.image_id)
# Serialize multi-dimensional tensor data.
captions_proto = tf.make_tensor_proto(np.array(image.captions))
question_ids_proto = tf.make_tensor_proto(np.array(image.question_ids))
questions_proto = tf.make_tensor_proto(np.array(image.questions))
features_proto = tf.make_tensor_proto(image_features)
positions_proto = tf.make_tensor_proto(positions)
# Take the first answer always for simplicity.
# This is only used for training and unofficial eval.
answers = copy.deepcopy(image.answers)
for i, answer in enumerate(answers):
answers[i] = answer[0]
answers_proto = tf.make_tensor_proto(np.array(answers))
# Create final features dict.
features = dict(
image_id=int64_feature(image.image_id),
question_ids=bytes_feature(question_ids_proto.SerializeToString()),
questions=bytes_feature(questions_proto.SerializeToString()),
answers=bytes_feature(answers_proto.SerializeToString()),
captions=bytes_feature(captions_proto.SerializeToString()),
object_features=bytes_feature(features_proto.SerializeToString()),
object_positions=bytes_feature(positions_proto.SerializeToString()))
return tf.train.Example(features=tf.train.Features(feature=features))
def pb(name, data, bucket_count=None, display_name=None, description=None):
"""Create a legacy histogram summary protobuf.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
data: A `np.array` or array-like form of any shape. Must have type
castable to `float`.
bucket_count: Optional positive `int`. The output will have this
many buckets, except in two edge cases. If there is no data, then
there are no buckets. If there is data but all points have the
same value, then there is one bucket whose left and right
endpoints are the same.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
# TODO(nickfelt): remove on-demand imports once dep situation is fixed.
import tensorflow.compat.v1 as tf
if bucket_count is None:
bucket_count = summary_v2.DEFAULT_BUCKET_COUNT
data = np.array(data).flatten().astype(float)
if data.size == 0:
buckets = np.array([]).reshape((0, 3))
else:
min_ = np.min(data)
max_ = np.max(data)
range_ = max_ - min_
if range_ == 0:
center = min_
buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]])
else:
bucket_width = range_ / bucket_count
offsets = data - min_
bucket_indices = np.floor(offsets / bucket_width).astype(int)
clamped_indices = np.minimum(bucket_indices, bucket_count - 1)
one_hots = (np.array([clamped_indices]).transpose()
== np.arange(0, bucket_count)) # broadcast
assert one_hots.shape == (data.size, bucket_count), (
one_hots.shape, (data.size, bucket_count))
bucket_counts = np.sum(one_hots, axis=0)
edges = np.linspace(min_, max_, bucket_count + 1)
left_edges = edges[:-1]
right_edges = edges[1:]
buckets = np.array([left_edges, right_edges, bucket_counts]).transpose()
tensor = tf.make_tensor_proto(buckets, dtype=tf.float64)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
tf_summary_metadata = tf.SummaryMetadata.FromString(
summary_metadata.SerializeToString())
summary = tf.Summary()
summary.value.add(tag='%s/histogram_summary' % name,
metadata=tf_summary_metadata,
tensor=tensor)
return summary
def send(self, iter_id, name, x):
msg = tws_pb.TrainerWorkerMessage(data=tws_pb.DataMessage(
iter_id=iter_id, name=name, tensor=tf.make_tensor_proto(x)))
self._transmit(msg)
logging.debug('Data: send %s for iter %d. seq_num=%d.', name, iter_id,
msg.seq_num)
