def _infer(self, request):
"""Returns JSON for the `vz-line-chart`s for a feature.
Args:
request: A request that should contain 'inference_address', 'model_name',
'model_type, 'model_version', 'model_signature' and 'label_vocab_path'.
Returns:
A list of JSON objects, one for each chart.
"""
vocab_path = request.args.get('label_vocab_path')
if vocab_path:
try:
with tf.gfile.GFile(vocab_path, 'r') as f:
label_vocab = [line.rstrip('\n') for line in f]
except tf.errors.NotFoundError as err:
tf.logging.error('error reading vocab file: %s', err)
label_vocab = []
else:
label_vocab = []
try:
if request.method != 'GET':
tf.logging.error('%s requests are forbidden.', request.method)
return http_util.Respond(request, {'error': 'invalid non-GET request'},
'application/json', code=405)
serving_bundle = inference_utils.ServingBundle(
request.args.get('inference_address'),
request.args.get('model_name'), request.args.get('model_type'),
request.args.get('model_version'),
request.args.get('model_signature'),
request.args.get('use_predict') == 'true',
request.args.get('predict_input_tensor'),
request.args.get('predict_output_tensor'))
indices_to_infer = sorted(self.updated_example_indices)
examples_to_infer = [self.examples[index] for index in indices_to_infer]
# Get inference results proto and combine with indices of inferred
# examples and respond with this data as json.
inference_result_proto = platform_utils.call_servo(
examples_to_infer, serving_bundle)
new_inferences = inference_utils.wrap_inference_results(
inference_result_proto)
infer_json = json_format.MessageToJson(
new_inferences, including_default_value_fields=True)
infer_obj = json.loads(infer_json)
resp = {'indices': indices_to_infer, 'results': infer_obj}
self.updated_example_indices = set()
return http_util.Respond(request, {'inferences': json.dumps(resp),
'vocab': json.dumps(label_vocab)},
'application/json')
except common_utils.InvalidUserInputError as e:
return http_util.Respond(request, {'error': e.message},
'application/json', code=400)
except AbortionError as e:
return http_util.Respond(request, {'error': e.details},
'application/json', code=400)
def chart_for_index(index_to_mutate):
mutant_features, mutant_examples = make_mutant_tuples(
example_proto, original_feature, index_to_mutate, viz_params)
inference_result_proto = platform_utils.call_servo(
mutant_examples, serving_bundle)
return make_json_formatted_for_single_chart(mutant_features,
inference_result_proto,
index_to_mutate)
def chart_for_index(index_to_mutate):
mutant_features, mutant_examples = make_mutant_tuples(
example_proto, original_feature, index_to_mutate, viz_params)
inference_result_proto = platform_utils.call_servo(
mutant_examples, serving_bundle)
return make_json_formatted_for_single_chart(mutant_features,
inference_result_proto,
index_to_mutate)
def _infer(self, request):
"""Returns JSON for the `vz-line-chart`s for a feature.
Args:
request: A request that should contain 'inference_address', 'model_name',
'model_type, 'model_version', 'model_signature' and 'label_vocab_path'.
Returns:
A list of JSON objects, one for each chart.
"""
vocab_path = request.args.get('label_vocab_path')
if vocab_path:
try:
with tf.gfile.GFile(vocab_path, 'r') as f:
label_vocab = [line.rstrip('\n') for line in f]
except tf.errors.NotFoundError as err:
tf.logging.error('error reading vocab file: %s', err)
label_vocab = []
else:
label_vocab = []
try:
if request.method != 'GET':
tf.logging.error('%s requests are forbidden.', request.method)
return http_util.Respond(request, {'error': 'invalid non-GET request'},
'application/json', code=405)
serving_bundle = inference_utils.ServingBundle(
request.args.get('inference_address'),
request.args.get('model_name'), request.args.get('model_type'),
request.args.get('model_version'),
request.args.get('model_signature'))
indices_to_infer = sorted(self.updated_example_indices)
examples_to_infer = [self.examples[index] for index in indices_to_infer]
# Get inference results proto and combine with indices of inferred
# examples and respond with this data as json.
inference_result_proto = platform_utils.call_servo(
examples_to_infer, serving_bundle)
new_inferences = inference_utils.wrap_inference_results(
inference_result_proto)
infer_json = json_format.MessageToJson(
new_inferences, including_default_value_fields=True)
infer_obj = json.loads(infer_json)
resp = {'indices': indices_to_infer, 'results': infer_obj}
self.updated_example_indices = set()
return http_util.Respond(request, {'inferences': json.dumps(resp),
'vocab': json.dumps(label_vocab)},
'application/json')
except common_utils.InvalidUserInputError as e:
return http_util.Respond(request, {'error': e.message},
'application/json', code=400)
except AbortionError as e:
return http_util.Respond(request, {'error': e.details},
'application/json', code=400)
def run_inference(examples, serving_bundle):
"""Run inference on examples given model information
Args:
examples: A list of examples that matches the model spec.
serving_bundle: A `ServingBundle` object that contains the information to
make the inference request.
Returns:
A tuple with the first entry being the ClassificationResponse or
RegressionResponse proto and the second entry being a dictionary of extra
data for each example, such as attributions, or None if no data exists.
"""
batch_size = 64
if serving_bundle.estimator and serving_bundle.feature_spec:
# If provided an estimator and feature spec then run inference locally.
preds = serving_bundle.estimator.predict(
lambda: tf.data.Dataset.from_tensor_slices(
tf.io.parse_example([
ex.SerializeToString() for ex in examples
], serving_bundle.feature_spec)).batch(batch_size))
if serving_bundle.use_predict:
preds_key = serving_bundle.predict_output_tensor
elif serving_bundle.model_type == 'regression':
preds_key = 'predictions'
else:
preds_key = 'probabilities'
values = []
for pred in preds:
values.append(pred[preds_key])
return (common_utils.convert_prediction_values(values,
serving_bundle), None)
elif serving_bundle.custom_predict_fn:
# If custom_predict_fn is provided, pass examples directly for local
# inference.
values = serving_bundle.custom_predict_fn(examples)
extra_results = None
# If the custom prediction function returned a dict, then parse out the
# prediction scores. If it is just a list, then the results are the
# prediction results without attributions or other data.
if isinstance(values, dict):
preds = values.pop('predictions')
extra_results = values
else:
preds = values
return (common_utils.convert_prediction_values(preds, serving_bundle),
extra_results)
else:
return (platform_utils.call_servo(examples, serving_bundle), None)
def run_inference(examples, serving_bundle):
"""Run inference on examples given model information
Args:
examples: A list of examples that matches the model spec.
serving_bundle: A `ServingBundle` object that contains the information to
make the inference request.
Returns:
A ClassificationResponse or RegressionResponse proto.
"""
batch_size = 64
if serving_bundle.estimator and serving_bundle.feature_spec:
# If provided an estimator and feature spec then run inference locally.
preds = serving_bundle.estimator.predict(
lambda: tf.data.Dataset.from_tensor_slices(
tf.parse_example([ex.SerializeToString()
for ex in examples], serving_bundle.
feature_spec)).batch(batch_size))
if serving_bundle.use_predict:
preds_key = serving_bundle.predict_output_tensor
elif serving_bundle.model_type == 'regression':
preds_key = 'predictions'
else:
preds_key = 'probabilities'
values = []
for pred in preds:
values.append(pred[preds_key])
return common_utils.convert_prediction_values(values, serving_bundle)
elif serving_bundle.custom_predict_fn:
# If custom_predict_fn is provided, pass examples directly for local
# inference.
values = serving_bundle.custom_predict_fn(examples)
return common_utils.convert_prediction_values(values, serving_bundle)
else:
return platform_utils.call_servo(examples, serving_bundle)
def run_inference(examples, serving_bundle):
"""Run inference on examples given model information.
Args:
examples: A list of examples that matches the model spec.
serving_bundle: A `ServingBundle` object that contains the information to
make the inference request.
Returns:
A tuple with the first entry being the ClassificationResponse or
RegressionResponse proto and the second entry being a dictionary of extra
data for each example, such as attributions, or None if no data exists.
"""
batch_size = 64
if serving_bundle.estimator and serving_bundle.feature_spec:
# If provided an estimator and feature spec then run inference locally.
preds = serving_bundle.estimator.predict(
lambda: tf.data.Dataset.from_tensor_slices(
tf.io.parse_example(
[ex.SerializeToString() for ex in examples],
serving_bundle.feature_spec,
)
).batch(batch_size)
)
# Use the specified key if one is provided.
key_to_use = (
serving_bundle.predict_output_tensor
if serving_bundle.use_predict
else None
)
values = []
for pred in preds:
if key_to_use is None:
# If the prediction dictionary only contains one key, use it.
returned_keys = list(pred.keys())
if len(returned_keys) == 1:
key_to_use = returned_keys[0]
# Use default keys if necessary.
elif serving_bundle.model_type == "classification":
key_to_use = "probabilities"
else:
key_to_use = "predictions"
if key_to_use not in pred:
raise KeyError(
'"%s" not found in model predictions dictionary'
% key_to_use
)
values.append(pred[key_to_use])
return (
common_utils.convert_prediction_values(values, serving_bundle),
None,
)
elif serving_bundle.custom_predict_fn:
# If custom_predict_fn is provided, pass examples directly for local
# inference.
values = serving_bundle.custom_predict_fn(examples)
extra_results = None
# If the custom prediction function returned a dict, then parse out the
# prediction scores. If it is just a list, then the results are the
# prediction results without attributions or other data.
if isinstance(values, dict):
preds = values.pop("predictions")
extra_results = values
else:
preds = values
return (
common_utils.convert_prediction_values(preds, serving_bundle),
extra_results,
)
else:
return (platform_utils.call_servo(examples, serving_bundle), None)
