def test_mutant_charts_for_feature_with_feature_index_pattern( self, mock_call_servo, mock_make_json_formatted_for_single_chart): example = self.make_and_write_fake_example() serving_bundle = inference_utils.ServingBundle('', '', 'classification', '', '') num_mutants = 10 viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[example], num_mutants=num_mutants, feature_index_pattern='0 , 2-3') mock_call_servo = lambda _, __: None mock_make_json_formatted_for_single_chart = lambda _, __: {} charts = inference_utils.mutant_charts_for_feature( example, 'repeated_float', serving_bundle, viz_params) self.assertEqual('numeric', charts['chartType']) self.assertEqual(3, len(charts['data'])) # These should error out because they don't have 4 fields. with self.assertRaises(common_utils.InvalidUserInputError): charts = inference_utils.mutant_charts_for_feature( example, 'repeated_int', serving_bundle, viz_params) with self.assertRaises(common_utils.InvalidUserInputError): charts = inference_utils.mutant_charts_for_feature( example, 'single_int', serving_bundle, viz_params)
def _infer_mutants_handler(self, request): """Returns JSON for the `vz-line-chart`s for a feature. Args: request: A request that should contain 'feature_name', 'example_index', 'inference_address', 'model_name', 'model_type', 'model_version', and 'model_signature'. Returns: A list of JSON objects, one for each chart. """ try: if request.method != 'GET': tf.logging.error('%s requests are forbidden.', request.method) return http_util.Respond(request, {'error': 'invalid non-GET request'}, 'application/json', code=405) example_index = int(request.args.get('example_index', '0')) feature_name = request.args.get('feature_name') example = self.examples[example_index] serving_bundle = inference_utils.ServingBundle( request.args.get('inference_address'), request.args.get('model_name'), request.args.get('model_type'), request.args.get('model_version'), request.args.get('model_signature')) viz_params = inference_utils.VizParams( request.args.get('x_min'), request.args.get('x_max'), self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS, request.args.get('feature_index_pattern')) json_mapping = inference_utils.mutant_charts_for_feature( example, feature_name, serving_bundle, viz_params) return http_util.Respond(request, json_mapping, 'application/json') except common_utils.InvalidUserInputError as e: return http_util.Respond(request, {'error': e.message}, 'application/json', code=400)
def test_mutant_charts_for_feature(self, mock_call_servo, mock_make_json_formatted_for_single_chart): example = self.make_and_write_fake_example() serving_bundle = inference_utils.ServingBundle('', '', 'classification', '', '') num_mutants = 10 viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[example], num_mutants=num_mutants, feature_index_pattern=None) mock_call_servo = lambda _, __: None mock_make_json_formatted_for_single_chart = lambda _, __: {} charts = inference_utils.mutant_charts_for_feature( example, 'repeated_float', serving_bundle, viz_params) self.assertEqual('numeric', charts['chartType']) self.assertEqual(4, len(charts['data'])) charts = inference_utils.mutant_charts_for_feature( example, 'repeated_int', serving_bundle, viz_params) self.assertEqual('numeric', charts['chartType']) self.assertEqual(2, len(charts['data'])) charts = inference_utils.mutant_charts_for_feature( example, 'single_int', serving_bundle, viz_params) self.assertEqual('numeric', charts['chartType']) self.assertEqual(1, len(charts['data'])) charts = inference_utils.mutant_charts_for_feature( example, 'non_numeric', serving_bundle, viz_params) self.assertEqual('categorical', charts['chartType']) self.assertEqual(3, len(charts['data']))
def infer_mutants_impl(self, info): """Performs mutant inference on specified examples.""" example_index = int(info["example_index"]) feature_name = info["feature_name"] examples = (self.examples if example_index == -1 else [self.examples[example_index]]) examples = [self.json_to_proto(ex) for ex in examples] scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]] serving_bundles = [] serving_bundles.append( inference_utils.ServingBundle( self.config.get("inference_address"), self.config.get("model_name"), self.config.get("model_type"), self.config.get("model_version"), self.config.get("model_signature"), self.config.get("uses_predict_api"), self.config.get("predict_input_tensor"), self.config.get("predict_output_tensor"), self.estimator_and_spec.get("estimator"), self.estimator_and_spec.get("feature_spec"), self.custom_predict_fn, )) if ("inference_address_2" in self.config or self.compare_estimator_and_spec.get("estimator") or self.compare_custom_predict_fn): serving_bundles.append( inference_utils.ServingBundle( self.config.get("inference_address_2"), self.config.get("model_name_2"), self.config.get("model_type"), self.config.get("model_version_2"), self.config.get("model_signature_2"), self.config.get("uses_predict_api"), self.config.get("predict_input_tensor"), self.config.get("predict_output_tensor"), self.compare_estimator_and_spec.get("estimator"), self.compare_estimator_and_spec.get("feature_spec"), self.compare_custom_predict_fn, )) viz_params = inference_utils.VizParams( info["x_min"], info["x_max"], scan_examples, 10, info["feature_index_pattern"], ) self.running_mutant_infer = True charts = inference_utils.mutant_charts_for_feature( examples, feature_name, serving_bundles, viz_params) self.running_mutant_infer = False return charts
def test_vizparams_pattern_parser(self): viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern=None) self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='1-3') self.assertEqual([1, 2, 3], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='3-1') self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='1-1') self.assertEqual([1], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='3, 1') self.assertEqual([1, 3], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='0-') self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='0-error') self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams(x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern='0-3-5') self.assertEqual([], viz_params.feature_indices)
def _infer_mutants_handler(self, request): """Returns JSON for the `vz-line-chart`s for a feature. Args: request: A request that should contain 'feature_name', 'example_index', 'inference_address', 'model_name', 'model_type', 'model_version', and 'model_signature'. Returns: A list of JSON objects, one for each chart. """ try: if request.method != 'GET': logger.error('%s requests are forbidden.', request.method) return http_util.Respond(request, {'error': 'invalid non-GET request'}, 'application/json', code=405) example_index = int(request.args.get('example_index', '0')) feature_name = request.args.get('feature_name') examples = (self.examples if example_index == -1 else [self.examples[example_index]]) (inference_addresses, model_names, model_versions, model_signatures) = self._parse_request_arguments(request) serving_bundles = [] for model_num in xrange(len(inference_addresses)): serving_bundles.append( inference_utils.ServingBundle( inference_addresses[model_num], model_names[model_num], request.args.get('model_type'), model_versions[model_num], model_signatures[model_num], request.args.get('use_predict') == 'true', request.args.get('predict_input_tensor'), request.args.get('predict_output_tensor'))) viz_params = inference_utils.VizParams( request.args.get('x_min'), request.args.get('x_max'), self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS, request.args.get('feature_index_pattern')) json_mapping = inference_utils.mutant_charts_for_feature( examples, feature_name, serving_bundles, viz_params) return http_util.Respond(request, json_mapping, 'application/json') except common_utils.InvalidUserInputError as e: return http_util.Respond(request, {'error': e.message}, 'application/json', code=400)
def test_make_mutant_tuples_float_list(self): example = self.make_and_write_fake_example() index_to_mutate = 1 num_mutants = 10 viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[example], num_mutants=num_mutants, feature_index_pattern=None, ) original_feature = inference_utils.parse_original_feature_from_example( example, "repeated_float") mutant_features, mutant_examples = inference_utils.make_mutant_tuples( [example], original_feature, index_to_mutate=index_to_mutate, viz_params=viz_params, ) # Check that values in mutant_features and mutant_examples are as expected. expected_values = np.linspace(1, 10, num_mutants) np.testing.assert_almost_equal( expected_values, [ mutant_feature.mutant_value for mutant_feature in mutant_features ], ) np.testing.assert_almost_equal( expected_values, [ mutant_example.features.feature["repeated_float"].float_list. value[index_to_mutate] for mutant_example in mutant_examples ], ) # Check that the example (other than the mutant value) is the same. for expected_value, mutant_example in zip(expected_values, mutant_examples): mutant_values = test_utils.value_from_example( mutant_example, "repeated_float") original_values = test_utils.value_from_example( example, "repeated_float") original_values[index_to_mutate] = expected_value self.assertEqual(original_values, mutant_values)
def _infer_mutants(self, change): info = self.infer_mutants example_index = int(info['example_index']) feature_name = info['feature_name'] examples = (self.examples if example_index == -1 else [self.examples[example_index]]) examples = [self.json_to_proto(ex) for ex in examples] scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]] serving_bundles = [] serving_bundles.append(inference_utils.ServingBundle( self.config.get('inference_address'), self.config.get('model_name'), self.config.get('model_type'), self.config.get('model_version'), self.config.get('model_signature'), self.config.get('uses_predict_api'), self.config.get('predict_input_tensor'), self.config.get('predict_output_tensor'), self.estimator_and_spec.get('estimator'), self.estimator_and_spec.get('feature_spec'), self.custom_predict_fn)) if ('inference_address_2' in self.config or self.compare_estimator_and_spec.get('estimator') or self.compare_custom_predict_fn): serving_bundles.append(inference_utils.ServingBundle( self.config.get('inference_address_2'), self.config.get('model_name_2'), self.config.get('model_type'), self.config.get('model_version_2'), self.config.get('model_signature_2'), self.config.get('uses_predict_api'), self.config.get('predict_input_tensor'), self.config.get('predict_output_tensor'), self.compare_estimator_and_spec.get('estimator'), self.compare_estimator_and_spec.get('feature_spec'), self.compare_custom_predict_fn)) viz_params = inference_utils.VizParams( info['x_min'], info['x_max'], scan_examples, 10, info['feature_index_pattern']) json_mapping = inference_utils.mutant_charts_for_feature( examples, feature_name, serving_bundles, viz_params) json_mapping['counter'] = self.mutant_charts_counter self.mutant_charts_counter += 1 self.mutant_charts = json_mapping
def infer_mutants(self, info): example_index = int(info['example_index']) feature_name = info['feature_name'] examples = (self.examples if example_index == -1 else [self.examples[example_index]]) examples = [self.json_to_proto(ex) for ex in examples] scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]] serving_bundles = [] serving_bundles.append(inference_utils.ServingBundle( self.config.get('inference_address'), self.config.get('model_name'), self.config.get('model_type'), self.config.get('model_version'), self.config.get('model_signature'), self.config.get('uses_predict_api'), self.config.get('predict_input_tensor'), self.config.get('predict_output_tensor'), self.estimator_and_spec.get('estimator'), self.estimator_and_spec.get('feature_spec'), self.custom_predict_fn)) if ('inference_address_2' in self.config or self.compare_estimator_and_spec.get('estimator') or self.compare_custom_predict_fn): serving_bundles.append(inference_utils.ServingBundle( self.config.get('inference_address_2'), self.config.get('model_name_2'), self.config.get('model_type'), self.config.get('model_version_2'), self.config.get('model_signature_2'), self.config.get('uses_predict_api'), self.config.get('predict_input_tensor'), self.config.get('predict_output_tensor'), self.compare_estimator_and_spec.get('estimator'), self.compare_estimator_and_spec.get('feature_spec'), self.compare_custom_predict_fn)) viz_params = inference_utils.VizParams( info['x_min'], info['x_max'], scan_examples, 10, info['feature_index_pattern']) json_mapping = inference_utils.mutant_charts_for_feature( examples, feature_name, serving_bundles, viz_params) output.eval_js("""inferMutantsCallback('{json_mapping}')""".format( json_mapping=json.dumps(json_mapping)))
def _infer_mutants_impl( self, feature_name, example_index, inference_addresses, model_names, model_type, model_versions, model_signatures, use_predict, predict_input_tensor, predict_output_tensor, x_min, x_max, feature_index_pattern, ): """Helper for generating PD plots for a feature.""" examples = (self.examples if example_index == -1 else [self.examples[example_index]]) serving_bundles = [] for model_num in xrange(len(inference_addresses)): serving_bundles.append( inference_utils.ServingBundle( inference_addresses[model_num], model_names[model_num], model_type, model_versions[model_num], model_signatures[model_num], use_predict, predict_input_tensor, predict_output_tensor, )) viz_params = inference_utils.VizParams( x_min, x_max, self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS, feature_index_pattern, ) return inference_utils.mutant_charts_for_feature( examples, feature_name, serving_bundles, viz_params)
def test_mutant_charts_for_feature( self, mock_call_servo, mock_make_json_formatted_for_single_chart): example = self.make_and_write_fake_example() serving_bundles = [ inference_utils.ServingBundle("", "", "classification", "", "", False, "", "") ] num_mutants = 10 viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[example], num_mutants=num_mutants, feature_index_pattern=None, ) mock_call_servo = lambda _, __: None mock_make_json_formatted_for_single_chart = lambda _, __: {} charts = inference_utils.mutant_charts_for_feature([example], "repeated_float", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(4, len(charts["data"])) charts = inference_utils.mutant_charts_for_feature([example], "repeated_int", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(2, len(charts["data"])) charts = inference_utils.mutant_charts_for_feature([example], "single_int", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(1, len(charts["data"])) charts = inference_utils.mutant_charts_for_feature([example], "non_numeric", serving_bundles, viz_params) self.assertEqual("categorical", charts["chartType"]) self.assertEqual(3, len(charts["data"]))
def test_mutant_charts_for_feature_with_feature_index_pattern( self, mock_call_servo, mock_make_json_formatted_for_single_chart): example = self.make_and_write_fake_example() serving_bundles = [ inference_utils.ServingBundle("", "", "classification", "", "", False, "", "") ] num_mutants = 10 viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[example], num_mutants=num_mutants, feature_index_pattern="0 , 2-3", ) mock_call_servo = lambda _, __: None mock_make_json_formatted_for_single_chart = lambda _, __: {} charts = inference_utils.mutant_charts_for_feature([example], "repeated_float", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(3, len(charts["data"])) # These should return 3 charts even though all fields from the index # pattern don't exist for the example. charts = inference_utils.mutant_charts_for_feature([example], "repeated_int", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(3, len(charts["data"])) charts = inference_utils.mutant_charts_for_feature([example], "single_int", serving_bundles, viz_params) self.assertEqual("numeric", charts["chartType"]) self.assertEqual(3, len(charts["data"]))
def test_vizparams_pattern_parser(self): viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern=None, ) self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="1-3", ) self.assertEqual([1, 2, 3], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="3-1", ) self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="1-1", ) self.assertEqual([1], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="3, 1", ) self.assertEqual([1, 3], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="0-", ) self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="0-error", ) self.assertEqual([], viz_params.feature_indices) viz_params = inference_utils.VizParams( x_min=1, x_max=10, examples=[], num_mutants=0, feature_index_pattern="0-3-5", ) self.assertEqual([], viz_params.feature_indices)