def test_mutant_charts_for_feature_with_feature_index_pattern(
self, mock_call_servo, mock_make_json_formatted_for_single_chart):
example = self.make_and_write_fake_example()
serving_bundle = inference_utils.ServingBundle('', '', 'classification',
'', '')
num_mutants = 10
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[example],
num_mutants=num_mutants,
feature_index_pattern='0 , 2-3')
mock_call_servo = lambda _, __: None
mock_make_json_formatted_for_single_chart = lambda _, __: {}
charts = inference_utils.mutant_charts_for_feature(
example, 'repeated_float', serving_bundle, viz_params)
self.assertEqual('numeric', charts['chartType'])
self.assertEqual(3, len(charts['data']))
# These should error out because they don't have 4 fields.
with self.assertRaises(common_utils.InvalidUserInputError):
charts = inference_utils.mutant_charts_for_feature(
example, 'repeated_int', serving_bundle, viz_params)
with self.assertRaises(common_utils.InvalidUserInputError):
charts = inference_utils.mutant_charts_for_feature(
example, 'single_int', serving_bundle, viz_params)
def _infer_mutants_handler(self, request):
"""Returns JSON for the `vz-line-chart`s for a feature.
Args:
request: A request that should contain 'feature_name', 'example_index',
'inference_address', 'model_name', 'model_type', 'model_version', and
'model_signature'.
Returns:
A list of JSON objects, one for each chart.
"""
try:
if request.method != 'GET':
tf.logging.error('%s requests are forbidden.', request.method)
return http_util.Respond(request, {'error': 'invalid non-GET request'},
'application/json', code=405)
example_index = int(request.args.get('example_index', '0'))
feature_name = request.args.get('feature_name')
example = self.examples[example_index]
serving_bundle = inference_utils.ServingBundle(
request.args.get('inference_address'), request.args.get('model_name'),
request.args.get('model_type'),
request.args.get('model_version'),
request.args.get('model_signature'))
viz_params = inference_utils.VizParams(
request.args.get('x_min'), request.args.get('x_max'),
self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS,
request.args.get('feature_index_pattern'))
json_mapping = inference_utils.mutant_charts_for_feature(
example, feature_name, serving_bundle, viz_params)
return http_util.Respond(request, json_mapping, 'application/json')
except common_utils.InvalidUserInputError as e:
return http_util.Respond(request, {'error': e.message},
'application/json', code=400)
def test_mutant_charts_for_feature(self, mock_call_servo,
mock_make_json_formatted_for_single_chart):
example = self.make_and_write_fake_example()
serving_bundle = inference_utils.ServingBundle('', '', 'classification',
'', '')
num_mutants = 10
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[example],
num_mutants=num_mutants,
feature_index_pattern=None)
mock_call_servo = lambda _, __: None
mock_make_json_formatted_for_single_chart = lambda _, __: {}
charts = inference_utils.mutant_charts_for_feature(
example, 'repeated_float', serving_bundle, viz_params)
self.assertEqual('numeric', charts['chartType'])
self.assertEqual(4, len(charts['data']))
charts = inference_utils.mutant_charts_for_feature(
example, 'repeated_int', serving_bundle, viz_params)
self.assertEqual('numeric', charts['chartType'])
self.assertEqual(2, len(charts['data']))
charts = inference_utils.mutant_charts_for_feature(
example, 'single_int', serving_bundle, viz_params)
self.assertEqual('numeric', charts['chartType'])
self.assertEqual(1, len(charts['data']))
charts = inference_utils.mutant_charts_for_feature(
example, 'non_numeric', serving_bundle, viz_params)
self.assertEqual('categorical', charts['chartType'])
self.assertEqual(3, len(charts['data']))
def infer_mutants_impl(self, info):
"""Performs mutant inference on specified examples."""
example_index = int(info["example_index"])
feature_name = info["feature_name"]
examples = (self.examples
if example_index == -1 else [self.examples[example_index]])
examples = [self.json_to_proto(ex) for ex in examples]
scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]]
serving_bundles = []
serving_bundles.append(
inference_utils.ServingBundle(
self.config.get("inference_address"),
self.config.get("model_name"),
self.config.get("model_type"),
self.config.get("model_version"),
self.config.get("model_signature"),
self.config.get("uses_predict_api"),
self.config.get("predict_input_tensor"),
self.config.get("predict_output_tensor"),
self.estimator_and_spec.get("estimator"),
self.estimator_and_spec.get("feature_spec"),
self.custom_predict_fn,
))
if ("inference_address_2" in self.config
or self.compare_estimator_and_spec.get("estimator")
or self.compare_custom_predict_fn):
serving_bundles.append(
inference_utils.ServingBundle(
self.config.get("inference_address_2"),
self.config.get("model_name_2"),
self.config.get("model_type"),
self.config.get("model_version_2"),
self.config.get("model_signature_2"),
self.config.get("uses_predict_api"),
self.config.get("predict_input_tensor"),
self.config.get("predict_output_tensor"),
self.compare_estimator_and_spec.get("estimator"),
self.compare_estimator_and_spec.get("feature_spec"),
self.compare_custom_predict_fn,
))
viz_params = inference_utils.VizParams(
info["x_min"],
info["x_max"],
scan_examples,
10,
info["feature_index_pattern"],
)
self.running_mutant_infer = True
charts = inference_utils.mutant_charts_for_feature(
examples, feature_name, serving_bundles, viz_params)
self.running_mutant_infer = False
return charts
def test_vizparams_pattern_parser(self):
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern=None)
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='1-3')
self.assertEqual([1, 2, 3], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='3-1')
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='1-1')
self.assertEqual([1], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='3, 1')
self.assertEqual([1, 3], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='0-')
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='0-error')
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern='0-3-5')
self.assertEqual([], viz_params.feature_indices)
def _infer_mutants_handler(self, request):
"""Returns JSON for the `vz-line-chart`s for a feature.
Args:
request: A request that should contain 'feature_name', 'example_index',
'inference_address', 'model_name', 'model_type', 'model_version', and
'model_signature'.
Returns:
A list of JSON objects, one for each chart.
"""
try:
if request.method != 'GET':
logger.error('%s requests are forbidden.', request.method)
return http_util.Respond(request,
{'error': 'invalid non-GET request'},
'application/json',
code=405)
example_index = int(request.args.get('example_index', '0'))
feature_name = request.args.get('feature_name')
examples = (self.examples if example_index == -1 else
[self.examples[example_index]])
(inference_addresses, model_names, model_versions,
model_signatures) = self._parse_request_arguments(request)
serving_bundles = []
for model_num in xrange(len(inference_addresses)):
serving_bundles.append(
inference_utils.ServingBundle(
inference_addresses[model_num], model_names[model_num],
request.args.get('model_type'),
model_versions[model_num], model_signatures[model_num],
request.args.get('use_predict') == 'true',
request.args.get('predict_input_tensor'),
request.args.get('predict_output_tensor')))
viz_params = inference_utils.VizParams(
request.args.get('x_min'), request.args.get('x_max'),
self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS,
request.args.get('feature_index_pattern'))
json_mapping = inference_utils.mutant_charts_for_feature(
examples, feature_name, serving_bundles, viz_params)
return http_util.Respond(request, json_mapping, 'application/json')
except common_utils.InvalidUserInputError as e:
return http_util.Respond(request, {'error': e.message},
'application/json',
code=400)
def test_make_mutant_tuples_float_list(self):
example = self.make_and_write_fake_example()
index_to_mutate = 1
num_mutants = 10
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[example],
num_mutants=num_mutants,
feature_index_pattern=None,
)
original_feature = inference_utils.parse_original_feature_from_example(
example, "repeated_float")
mutant_features, mutant_examples = inference_utils.make_mutant_tuples(
[example],
original_feature,
index_to_mutate=index_to_mutate,
viz_params=viz_params,
)
# Check that values in mutant_features and mutant_examples are as expected.
expected_values = np.linspace(1, 10, num_mutants)
np.testing.assert_almost_equal(
expected_values,
[
mutant_feature.mutant_value
for mutant_feature in mutant_features
],
)
np.testing.assert_almost_equal(
expected_values,
[
mutant_example.features.feature["repeated_float"].float_list.
value[index_to_mutate] for mutant_example in mutant_examples
],
)
# Check that the example (other than the mutant value) is the same.
for expected_value, mutant_example in zip(expected_values,
mutant_examples):
mutant_values = test_utils.value_from_example(
mutant_example, "repeated_float")
original_values = test_utils.value_from_example(
example, "repeated_float")
original_values[index_to_mutate] = expected_value
self.assertEqual(original_values, mutant_values)
def _infer_mutants(self, change):
info = self.infer_mutants
example_index = int(info['example_index'])
feature_name = info['feature_name']
examples = (self.examples if example_index == -1
else [self.examples[example_index]])
examples = [self.json_to_proto(ex) for ex in examples]
scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]]
serving_bundles = []
serving_bundles.append(inference_utils.ServingBundle(
self.config.get('inference_address'),
self.config.get('model_name'),
self.config.get('model_type'),
self.config.get('model_version'),
self.config.get('model_signature'),
self.config.get('uses_predict_api'),
self.config.get('predict_input_tensor'),
self.config.get('predict_output_tensor'),
self.estimator_and_spec.get('estimator'),
self.estimator_and_spec.get('feature_spec'),
self.custom_predict_fn))
if ('inference_address_2' in self.config or
self.compare_estimator_and_spec.get('estimator') or
self.compare_custom_predict_fn):
serving_bundles.append(inference_utils.ServingBundle(
self.config.get('inference_address_2'),
self.config.get('model_name_2'),
self.config.get('model_type'),
self.config.get('model_version_2'),
self.config.get('model_signature_2'),
self.config.get('uses_predict_api'),
self.config.get('predict_input_tensor'),
self.config.get('predict_output_tensor'),
self.compare_estimator_and_spec.get('estimator'),
self.compare_estimator_and_spec.get('feature_spec'),
self.compare_custom_predict_fn))
viz_params = inference_utils.VizParams(
info['x_min'], info['x_max'],
scan_examples, 10,
info['feature_index_pattern'])
json_mapping = inference_utils.mutant_charts_for_feature(
examples, feature_name, serving_bundles, viz_params)
json_mapping['counter'] = self.mutant_charts_counter
self.mutant_charts_counter += 1
self.mutant_charts = json_mapping
def infer_mutants(self, info):
example_index = int(info['example_index'])
feature_name = info['feature_name']
examples = (self.examples if example_index == -1
else [self.examples[example_index]])
examples = [self.json_to_proto(ex) for ex in examples]
scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]]
serving_bundles = []
serving_bundles.append(inference_utils.ServingBundle(
self.config.get('inference_address'),
self.config.get('model_name'),
self.config.get('model_type'),
self.config.get('model_version'),
self.config.get('model_signature'),
self.config.get('uses_predict_api'),
self.config.get('predict_input_tensor'),
self.config.get('predict_output_tensor'),
self.estimator_and_spec.get('estimator'),
self.estimator_and_spec.get('feature_spec'),
self.custom_predict_fn))
if ('inference_address_2' in self.config or
self.compare_estimator_and_spec.get('estimator') or
self.compare_custom_predict_fn):
serving_bundles.append(inference_utils.ServingBundle(
self.config.get('inference_address_2'),
self.config.get('model_name_2'),
self.config.get('model_type'),
self.config.get('model_version_2'),
self.config.get('model_signature_2'),
self.config.get('uses_predict_api'),
self.config.get('predict_input_tensor'),
self.config.get('predict_output_tensor'),
self.compare_estimator_and_spec.get('estimator'),
self.compare_estimator_and_spec.get('feature_spec'),
self.compare_custom_predict_fn))
viz_params = inference_utils.VizParams(
info['x_min'], info['x_max'],
scan_examples, 10,
info['feature_index_pattern'])
json_mapping = inference_utils.mutant_charts_for_feature(
examples, feature_name, serving_bundles, viz_params)
output.eval_js("""inferMutantsCallback('{json_mapping}')""".format(
json_mapping=json.dumps(json_mapping)))
def _infer_mutants_impl(
self,
feature_name,
example_index,
inference_addresses,
model_names,
model_type,
model_versions,
model_signatures,
use_predict,
predict_input_tensor,
predict_output_tensor,
x_min,
x_max,
feature_index_pattern,
):
"""Helper for generating PD plots for a feature."""
examples = (self.examples
if example_index == -1 else [self.examples[example_index]])
serving_bundles = []
for model_num in xrange(len(inference_addresses)):
serving_bundles.append(
inference_utils.ServingBundle(
inference_addresses[model_num],
model_names[model_num],
model_type,
model_versions[model_num],
model_signatures[model_num],
use_predict,
predict_input_tensor,
predict_output_tensor,
))
viz_params = inference_utils.VizParams(
x_min,
x_max,
self.examples[0:NUM_EXAMPLES_TO_SCAN],
NUM_MUTANTS,
feature_index_pattern,
)
return inference_utils.mutant_charts_for_feature(
examples, feature_name, serving_bundles, viz_params)
def test_mutant_charts_for_feature(
self, mock_call_servo, mock_make_json_formatted_for_single_chart):
example = self.make_and_write_fake_example()
serving_bundles = [
inference_utils.ServingBundle("", "", "classification", "", "",
False, "", "")
]
num_mutants = 10
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[example],
num_mutants=num_mutants,
feature_index_pattern=None,
)
mock_call_servo = lambda _, __: None
mock_make_json_formatted_for_single_chart = lambda _, __: {}
charts = inference_utils.mutant_charts_for_feature([example],
"repeated_float",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(4, len(charts["data"]))
charts = inference_utils.mutant_charts_for_feature([example],
"repeated_int",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(2, len(charts["data"]))
charts = inference_utils.mutant_charts_for_feature([example],
"single_int",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(1, len(charts["data"]))
charts = inference_utils.mutant_charts_for_feature([example],
"non_numeric",
serving_bundles,
viz_params)
self.assertEqual("categorical", charts["chartType"])
self.assertEqual(3, len(charts["data"]))
def test_mutant_charts_for_feature_with_feature_index_pattern(
self, mock_call_servo, mock_make_json_formatted_for_single_chart):
example = self.make_and_write_fake_example()
serving_bundles = [
inference_utils.ServingBundle("", "", "classification", "", "",
False, "", "")
]
num_mutants = 10
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[example],
num_mutants=num_mutants,
feature_index_pattern="0 , 2-3",
)
mock_call_servo = lambda _, __: None
mock_make_json_formatted_for_single_chart = lambda _, __: {}
charts = inference_utils.mutant_charts_for_feature([example],
"repeated_float",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(3, len(charts["data"]))
# These should return 3 charts even though all fields from the index
# pattern don't exist for the example.
charts = inference_utils.mutant_charts_for_feature([example],
"repeated_int",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(3, len(charts["data"]))
charts = inference_utils.mutant_charts_for_feature([example],
"single_int",
serving_bundles,
viz_params)
self.assertEqual("numeric", charts["chartType"])
self.assertEqual(3, len(charts["data"]))
def test_vizparams_pattern_parser(self):
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern=None,
)
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="1-3",
)
self.assertEqual([1, 2, 3], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="3-1",
)
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="1-1",
)
self.assertEqual([1], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="3, 1",
)
self.assertEqual([1, 3], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="0-",
)
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="0-error",
)
self.assertEqual([], viz_params.feature_indices)
viz_params = inference_utils.VizParams(
x_min=1,
x_max=10,
examples=[],
num_mutants=0,
feature_index_pattern="0-3-5",
)
self.assertEqual([], viz_params.feature_indices)
