def test_build_all_signature_defs(self): input_features = constant_op.constant(["10"]) input_example = constant_op.constant(["11"]) input_ops = input_fn_utils.InputFnOps({"features": input_features}, None, {"default input": input_example}) input_alternatives, _ = ( saved_model_export_utils.get_input_alternatives(input_ops)) output_1 = constant_op.constant(["1"]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) provided_output_alternatives = { "head-1": (constants.ProblemType.LINEAR_REGRESSION, { "some_output_1": output_1 }), "head-2": (constants.ProblemType.CLASSIFICATION, { "some_output_2": output_2 }), "head-3": (constants.ProblemType.UNSPECIFIED, { "some_output_3": output_3 }), } model_fn_ops = model_fn.ModelFnOps( model_fn.ModeKeys.INFER, predictions={"some_output": constant_op.constant(["4"])}, output_alternatives=provided_output_alternatives) output_alternatives, _ = ( saved_model_export_utils.get_output_alternatives( model_fn_ops, "head-1")) signature_defs = saved_model_export_utils.build_all_signature_defs( input_alternatives, output_alternatives, "head-1") expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( input_example, output_1), "default_input_alternative:head-1": signature_def_utils.regression_signature_def( input_example, output_1), "default_input_alternative:head-2": signature_def_utils.classification_signature_def( input_example, output_2, None), "default_input_alternative:head-3": signature_def_utils.predict_signature_def({"input": input_example}, {"output": output_3}), # "features_input_alternative:head-1": # signature_def_utils.regression_signature_def(input_features, # output_1), # "features_input_alternative:head-2": # signature_def_utils.classification_signature_def(input_features, # output_2, None), # "features_input_alternative:head-3": # signature_def_utils.predict_signature_def({ # "input": input_features # }, {"output": output_3}), } self.assertDictEqual(expected_signature_defs, signature_defs)
def test_build_all_signature_defs(self): input_features = constant_op.constant(["10"]) input_example = constant_op.constant(["11"]) input_ops = input_fn_utils.InputFnOps({ "features": input_features }, None, {"default input": input_example}) input_alternatives, _ = ( saved_model_export_utils.get_input_alternatives(input_ops)) output_1 = constant_op.constant(["1"]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) provided_output_alternatives = { "head-1": (constants.ProblemType.LINEAR_REGRESSION, { "some_output_1": output_1 }), "head-2": (constants.ProblemType.CLASSIFICATION, { "some_output_2": output_2 }), "head-3": (constants.ProblemType.UNSPECIFIED, { "some_output_3": output_3 }), } model_fn_ops = model_fn.ModelFnOps( model_fn.ModeKeys.INFER, predictions={"some_output": constant_op.constant(["4"])}, output_alternatives=provided_output_alternatives) output_alternatives, _ = (saved_model_export_utils.get_output_alternatives( model_fn_ops, "head-1")) signature_defs = saved_model_export_utils.build_all_signature_defs( input_alternatives, output_alternatives, "head-1") expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def(input_example, output_1), "default_input_alternative:head-1": signature_def_utils.regression_signature_def(input_example, output_1), "default_input_alternative:head-2": signature_def_utils.classification_signature_def(input_example, output_2, None), "default_input_alternative:head-3": signature_def_utils.predict_signature_def({ "input": input_example }, {"output": output_3}), # "features_input_alternative:head-1": # signature_def_utils.regression_signature_def(input_features, # output_1), # "features_input_alternative:head-2": # signature_def_utils.classification_signature_def(input_features, # output_2, None), # "features_input_alternative:head-3": # signature_def_utils.predict_signature_def({ # "input": input_features # }, {"output": output_3}), } self.assertDictEqual(expected_signature_defs, signature_defs)
def testGetSignatureDefByKeyRegression(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant("b", name="output-1") meta_graph_def = meta_graph_pb2.MetaGraphDef() self._add_to_signature_def_map( meta_graph_def, { "my_regression": signature_def_utils.regression_signature_def(input1, output1) }) # Look up the regression signature with the key used while saving. signature_def = signature_def_contrib_utils.get_signature_def_by_key( meta_graph_def, "my_regression") # Check the method name to match the constants regression method name. self.assertEqual(signature_constants.REGRESS_METHOD_NAME, signature_def.method_name) # Check inputs in signature def. self.assertEqual(1, len(signature_def.inputs)) self._check_tensor_info(signature_def.inputs, signature_constants.REGRESS_INPUTS, "input-1:0") # Check outputs in signature def. self.assertEqual(1, len(signature_def.outputs)) self._check_tensor_info(signature_def.outputs, signature_constants.REGRESS_OUTPUTS, "output-1:0")
def test_build_all_signature_defs_without_receiver_alternatives(self): # Force the test to run in graph mode. # This tests a deprecated v1 API that depends on graph-only functions such # as build_tensor_info. with ops.Graph().as_default(): receiver_tensor = array_ops.placeholder(dtypes.string) output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput( outputs={"some_output_3": output_3}), } signature_defs = export_utils.build_all_signature_defs( receiver_tensor, export_outputs) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}) } self.assertDictEqual(expected_signature_defs, signature_defs)
def test_build_all_signature_defs_with_single_alternatives(self): # Force the test to run in graph mode. # This tests a deprecated v1 API that depends on graph-only functions such # as build_tensor_info. with ops.Graph().as_default(): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = array_ops.placeholder( dtypes.int64) receiver_tensors_alternative_2 = array_ops.sparse_placeholder( dtypes.float32) # Note we are passing single Tensors as values of # receiver_tensors_alternatives, where normally that is a dict. # In this case a dict will be created using the default receiver tensor # name "input". receiver_tensors_alternatives = { "other1": receiver_tensors_alternative_1, "other2": receiver_tensors_alternative_2 } output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput( outputs={"some_output_3": output_3}), } signature_defs = export_utils.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other1:head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensors_alternative_1}, {"some_output_3": output_3}), "other2:head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensors_alternative_2}, {"some_output_3": output_3}) # Note that the alternatives 'other:serving_default' and # 'other:head-2' are invalid, because regression and classification # signatures must take a single string input. Here we verify that # these invalid signatures are not included in the export_utils. } self.assertDictEqual(expected_signature_defs, signature_defs)
def testGetSignatureDefByKeyRegression(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant(7.2, name="output-1") meta_graph_def = meta_graph_pb2.MetaGraphDef() self._add_to_signature_def_map(meta_graph_def, { "my_regression": signature_def_utils.regression_signature_def(input1, output1) }) # Look up the regression signature with the key used while saving. signature_def = signature_def_contrib_utils.get_signature_def_by_key( meta_graph_def, "my_regression") # Check the method name to match the constants regression method name. self.assertEqual(signature_constants.REGRESS_METHOD_NAME, signature_def.method_name) # Check inputs in signature def. self.assertEqual(1, len(signature_def.inputs)) self._check_tensor_info(signature_def.inputs, signature_constants.REGRESS_INPUTS, "input-1:0") # Check outputs in signature def. self.assertEqual(1, len(signature_def.outputs)) self._check_tensor_info(signature_def.outputs, signature_constants.REGRESS_OUTPUTS, "output-1:0")
def test_build_all_signature_defs_without_receiver_alternatives(self): receiver_tensor = array_ops.placeholder(dtypes.string) output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={ "some_output_3": output_3 }), } signature_defs = export.build_all_signature_defs( receiver_tensor, export_outputs) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def(receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def(receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def({ "input": receiver_tensor }, {"some_output_3": output_3}) } self.assertDictEqual(expected_signature_defs, signature_defs)
def as_signature_def(self, receiver_tensors): if len(receiver_tensors) != 1: raise ValueError('Regression input must be a single string Tensor; ' 'got {}'.format(receiver_tensors)) (_, examples), = receiver_tensors.items() if dtypes.as_dtype(examples.dtype) != dtypes.string: raise ValueError('Regression input must be a single string Tensor; ' 'got {}'.format(receiver_tensors)) return signature_def_utils.regression_signature_def(examples, self.value)
def as_signature_def(self, receiver_tensors): if len(receiver_tensors) != 2: raise ValueError('Survival input must be dict with two entry; ' 'got {}'.format(receiver_tensors)) examples = receiver_tensors['input_examples'] if dtypes.as_dtype(examples.dtype) != dtypes.string: raise ValueError('Survival input must be a single string Tensor; ' 'got {}'.format(receiver_tensors)) return signature_def_utils.regression_signature_def(examples, self.value)
def test_build_all_signature_defs_with_single_alternatives(self): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = array_ops.placeholder(dtypes.int64) receiver_tensors_alternative_2 = array_ops.sparse_placeholder( dtypes.float32) # Note we are passing single Tensors as values of # receiver_tensors_alternatives, where normally that is a dict. # In this case a dict will be created using the default receiver tensor # name "input". receiver_tensors_alternatives = {"other1": receiver_tensors_alternative_1, "other2": receiver_tensors_alternative_2} output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={ "some_output_3": output_3 }), } signature_defs = export.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other1:head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensors_alternative_1}, {"some_output_3": output_3}), "other2:head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensors_alternative_2}, {"some_output_3": output_3}) # Note that the alternatives 'other:serving_default' and 'other:head-2' # are invalid, because regession and classification signatures must take # a single string input. Here we verify that these invalid signatures # are not included in the export. } self.assertDictEqual(expected_signature_defs, signature_defs)
def build_standardized_signature_def(input_tensors, output_tensors, problem_type): """Build a SignatureDef using problem type and input and output Tensors. Note that this delegates the actual creation of the signatures to methods in //third_party/tensorflow/python/saved_model/signature_def_utils.py, which may assign names to the input and output tensors (depending on the problem type) that are standardized in the context of SavedModel. Args: input_tensors: a dict of string key to `Tensor` output_tensors: a dict of string key to `Tensor` problem_type: an instance of constants.ProblemType, specifying classification, regression, etc. Returns: A SignatureDef using SavedModel standard keys where possible. Raises: ValueError: if input_tensors or output_tensors is None or empty. """ if not input_tensors: raise ValueError('input_tensors must be provided.') if not output_tensors: raise ValueError('output_tensors must be provided.') # Per-method signature_def functions will standardize the keys if possible if _is_classification_problem(problem_type, input_tensors, output_tensors): (_, examples), = input_tensors.items() classes = _get_classification_classes(output_tensors) scores = _get_classification_scores(output_tensors) if classes is None and scores is None: items = list(output_tensors.items()) if items[0][1].dtype == dtypes.string: (_, classes), = items else: (_, scores), = items return signature_def_utils.classification_signature_def( examples, classes, scores) elif _is_regression_problem(problem_type, input_tensors, output_tensors): (_, examples), = input_tensors.items() (_, predictions), = output_tensors.items() return signature_def_utils.regression_signature_def( examples, predictions) else: return signature_def_utils.predict_signature_def( input_tensors, output_tensors)
def test_build_all_signature_defs_with_dict_alternatives(self): with context.graph_mode(): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = { "foo": array_ops.placeholder(dtypes.int64), "bar": array_ops.sparse_placeholder(dtypes.float32)} receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1} output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={ "some_output_3": output_3 }), } signature_defs = export_utils.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other:head-3": signature_def_utils.predict_signature_def( receiver_tensors_alternative_1, {"some_output_3": output_3}) # Note that the alternatives 'other:serving_default' and # 'other:head-2' are invalid, because regession and classification # signatures must take a single string input. Here we verify that # these invalid signatures are not included in the export_utils. } self.assertDictEqual(expected_signature_defs, signature_defs)
def build_standardized_signature_def(input_tensors, output_tensors, problem_type): """Build a SignatureDef using problem type and input and output Tensors. Note that this delegates the actual creation of the signatures to methods in //third_party/tensorflow/python/saved_model/signature_def_utils.py, which may assign names to the input and output tensors (depending on the problem type) that are standardized in the context of SavedModel. Args: input_tensors: a dict of string key to `Tensor` output_tensors: a dict of string key to `Tensor` problem_type: an instance of constants.ProblemType, specifying classification, regression, etc. Returns: A SignatureDef using SavedModel standard keys where possible. Raises: ValueError: if input_tensors or output_tensors is None or empty. """ if not input_tensors: raise ValueError('input_tensors must be provided.') if not output_tensors: raise ValueError('output_tensors must be provided.') # Per-method signature_def functions will standardize the keys if possible if _is_classification_problem(problem_type, input_tensors, output_tensors): (_, examples), = input_tensors.items() classes = _get_classification_classes(output_tensors) scores = _get_classification_scores(output_tensors) if classes is None and scores is None: items = list(output_tensors.items()) if items[0][1].dtype == dtypes.string: (_, classes), = items else: (_, scores), = items return signature_def_utils.classification_signature_def( examples, classes, scores) elif _is_regression_problem(problem_type, input_tensors, output_tensors): (_, examples), = input_tensors.items() (_, predictions), = output_tensors.items() return signature_def_utils.regression_signature_def(examples, predictions) else: return signature_def_utils.predict_signature_def(input_tensors, output_tensors)
def test_build_all_signature_defs_with_dict_alternatives(self): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = { "foo": array_ops.placeholder(dtypes.int64), "bar": array_ops.sparse_placeholder(dtypes.float32)} receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1} output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={ "some_output_3": output_3 }), } signature_defs = export_utils.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other:head-3": signature_def_utils.predict_signature_def( receiver_tensors_alternative_1, {"some_output_3": output_3}) # Note that the alternatives 'other:serving_default' and # 'other:head-2' are invalid, because regession and classification # signatures must take a single string input. Here we verify that # these invalid signatures are not included in the export_utils. } self.assertDictEqual(expected_signature_defs, signature_defs)
def as_signature_def(self, receiver_tensors): if len(receiver_tensors) != 1: raise ValueError( 'Regression signatures can only accept a single tensor input of ' 'type tf.string. Please check to make sure that you have structured ' 'the serving_input_receiver_fn so that it creates a single string ' 'placeholder. If your model function expects multiple inputs, then ' 'use `tf.io.parse_example()` to parse the string into multiple ' f'tensors.\n Received: {receiver_tensors}') (_, examples), = receiver_tensors.items() if dtypes.as_dtype(examples.dtype) != dtypes.string: raise ValueError( 'Regression signatures can only accept a single tensor input of ' 'type tf.string. Please check to make sure that you have structured ' 'the serving_input_receiver_fn so that it creates a single string ' 'placeholder. If your model function expects multiple inputs, then ' 'use `tf.io.parse_example()` to parse the string into multiple ' f'tensors.\n Received: {receiver_tensors}') return signature_def_utils.regression_signature_def(examples, self.value)
def testRegressionSignatureDef(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant("b", name="output-1") signature_def = signature_def_utils.regression_signature_def( input1, output1) self.assertEqual(signature_constants.REGRESS_METHOD_NAME, signature_def.method_name) # Check inputs in signature def. self.assertEqual(1, len(signature_def.inputs)) x_tensor_info_actual = ( signature_def.inputs[signature_constants.REGRESS_INPUTS]) self.assertEqual("input-1:0", x_tensor_info_actual.name) self.assertEqual(types_pb2.DT_STRING, x_tensor_info_actual.dtype) self.assertEqual(0, len(x_tensor_info_actual.tensor_shape.dim)) # Check outputs in signature def. self.assertEqual(1, len(signature_def.outputs)) y_tensor_info_actual = ( signature_def.outputs[signature_constants.REGRESS_OUTPUTS]) self.assertEqual("output-1:0", y_tensor_info_actual.name) self.assertEqual(types_pb2.DT_STRING, y_tensor_info_actual.dtype) self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim))
def testRegressionSignatureDef(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant("b", name="output-1") signature_def = signature_def_utils.regression_signature_def(input1, output1) self.assertEqual(signature_constants.REGRESS_METHOD_NAME, signature_def.method_name) # Check inputs in signature def. self.assertEqual(1, len(signature_def.inputs)) x_tensor_info_actual = ( signature_def.inputs[signature_constants.REGRESS_INPUTS]) self.assertEqual("input-1:0", x_tensor_info_actual.name) self.assertEqual(types_pb2.DT_STRING, x_tensor_info_actual.dtype) self.assertEqual(0, len(x_tensor_info_actual.tensor_shape.dim)) # Check outputs in signature def. self.assertEqual(1, len(signature_def.outputs)) y_tensor_info_actual = ( signature_def.outputs[signature_constants.REGRESS_OUTPUTS]) self.assertEqual("output-1:0", y_tensor_info_actual.name) self.assertEqual(types_pb2.DT_STRING, y_tensor_info_actual.dtype) self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim))
def test_build_all_signature_defs_with_dict_alternatives(self): # Force the test to run in graph mode. # This tests a deprecated v1 API that depends on graph-only functions such # as build_tensor_info. with ops.Graph().as_default(): receiver_tensor = array_ops.placeholder(dtypes.string) receiver_tensors_alternative_1 = { "foo": array_ops.placeholder(dtypes.int64), "bar": array_ops.sparse_placeholder(dtypes.float32) } unfed_input = array_ops.placeholder(dtypes.bool) receiver_tensors_alternative_2 = {"unfed": unfed_input} receiver_tensors_alternatives = { "other": receiver_tensors_alternative_1, "with_unfed_input": receiver_tensors_alternative_2 } output_1 = constant_op.constant([1.]) output_2 = constant_op.constant(["2"]) output_3 = constant_op.constant(["3"]) output_4 = unfed_input output_5 = math_ops.logical_not(unfed_input) export_outputs = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: export_output.RegressionOutput(value=output_1), "head-2": export_output.ClassificationOutput(classes=output_2), "head-3": export_output.PredictOutput(outputs={"some_output_3": output_3}), "head-4": export_output.PredictOutput(outputs={"some_output_4": output_4}), "head-5": export_output.PredictOutput(outputs={"some_output_5": output_5}), } signature_defs = export_utils.build_all_signature_defs( receiver_tensor, export_outputs, receiver_tensors_alternatives) expected_signature_defs = { "serving_default": signature_def_utils.regression_signature_def( receiver_tensor, output_1), "head-2": signature_def_utils.classification_signature_def( receiver_tensor, output_2, None), "head-3": signature_def_utils.predict_signature_def( {"input": receiver_tensor}, {"some_output_3": output_3}), "other:head-3": signature_def_utils.predict_signature_def( receiver_tensors_alternative_1, {"some_output_3": output_3}), # Note that the alternatives 'other:serving_default' and # 'other:head-2' are invalid, because regression and classification # signatures must take a single string input. Here we verify that # these invalid signatures are not included in the export_utils. # Similarly, we verify that 'head-4' and 'head-5', which depend on an # input that is not being fed as a receiver tensor, are also omitted. # All the three heads are present when that input is fed, however: "with_unfed_input:head-3": signature_def_utils.predict_signature_def( receiver_tensors_alternative_2, {"some_output_3": output_3}), "with_unfed_input:head-4": signature_def_utils.predict_signature_def( receiver_tensors_alternative_2, {"some_output_4": output_4}), "with_unfed_input:head-5": signature_def_utils.predict_signature_def( receiver_tensors_alternative_2, {"some_output_5": output_5}) } self.assertDictEqual(expected_signature_defs, signature_defs)