def update_schema(schema: schema_pb2.Schema, statistics: statistics_pb2.DatasetFeatureStatisticsList, infer_feature_shape: Optional[bool] = True, max_string_domain_size: Optional[int] = 100 ) -> schema_pb2.Schema: """Updates input schema to conform to the input statistics. Args: schema: A Schema protocol buffer. statistics: A DatasetFeatureStatisticsList protocol buffer. Schema inference is currently supported only for lists with a single DatasetFeatureStatistics proto or lists with multiple DatasetFeatureStatistics protos corresponding to data slices that include the default slice (i.e., the slice with all examples). If a list with multiple DatasetFeatureStatistics protos is used, this function will update the schema to conform to the statistics corresponding to the default slice. infer_feature_shape: A boolean to indicate if shape of the features need to be inferred from the statistics. max_string_domain_size: Maximum size of the domain of a string feature in order to be interpreted as a categorical feature. Returns: A Schema protocol buffer. Raises: TypeError: If the input argument is not of the expected type. ValueError: If the input statistics proto contains multiple datasets, none of which corresponds to the default slice. """ if not isinstance(schema, schema_pb2.Schema): raise TypeError('schema is of type %s, should be a Schema proto.' % type(schema).__name__) if not isinstance(statistics, statistics_pb2.DatasetFeatureStatisticsList): raise TypeError( 'statistics is of type %s, should be ' 'a DatasetFeatureStatisticsList proto.' % type(statistics).__name__) # This will raise an exception if there are multiple datasets, none of which # corresponds to the default slice. dataset_statistics = _get_default_dataset_statistics(statistics) _check_for_unsupported_stats_fields(dataset_statistics, 'statistics') schema_proto_string = pywrap_tensorflow_data_validation.UpdateSchema( tf.compat.as_bytes(schema.SerializeToString()), tf.compat.as_bytes(dataset_statistics.SerializeToString()), max_string_domain_size) # Parse the serialized Schema proto. result = schema_pb2.Schema() result.ParseFromString(schema_proto_string) # TODO(b/113605666): Push this shape inference logic into example validation # code. if infer_feature_shape: _infer_shape(result) return result
def update_schema( schema: schema_pb2.Schema, statistics: statistics_pb2.DatasetFeatureStatisticsList, infer_feature_shape: Optional[bool] = True, max_string_domain_size: Optional[int] = 100) -> schema_pb2.Schema: """Updates input schema to conform to the input statistics. Args: schema: A Schema protocol buffer. statistics: A DatasetFeatureStatisticsList protocol buffer. Schema inference is currently only supported for lists with a single DatasetFeatureStatistics proto. infer_feature_shape: A boolean to indicate if shape of the features need to be inferred from the statistics. max_string_domain_size: Maximum size of the domain of a string feature in order to be interpreted as a categorical feature. Returns: A Schema protocol buffer. Raises: TypeError: If the input argument is not of the expected type. ValueError: If the input statistics proto does not have only one dataset. """ if not isinstance(schema, schema_pb2.Schema): raise TypeError('schema is of type %s, should be a Schema proto.' % type(schema).__name__) if not isinstance(statistics, statistics_pb2.DatasetFeatureStatisticsList): raise TypeError('statistics is of type %s, should be ' 'a DatasetFeatureStatisticsList proto.' % type(statistics).__name__) if len(statistics.datasets) != 1: raise ValueError('Only statistics proto with one dataset is currently ' 'supported for inferring schema.') _check_for_unsupported_stats_fields(statistics.datasets[0], 'statistics') schema_proto_string = pywrap_tensorflow_data_validation.UpdateSchema( tf.compat.as_bytes(schema.SerializeToString()), tf.compat.as_bytes(statistics.datasets[0].SerializeToString()), max_string_domain_size) # Parse the serialized Schema proto. result = schema_pb2.Schema() result.ParseFromString(schema_proto_string) # TODO(b/113605666): Push this shape inference logic into example validation # code. if infer_feature_shape: _infer_shape(result) return result