def preprocessing_fn(inputs: tf.Tensor) -> tf.Tensor:
"""tf.transform's callback function for preprocessing inputs.
Args:
inputs: map from feature keys to raw not-yet-transformed features.
Returns:
Map from string feature key to transformed feature operations.
"""
outputs = {}
for key in features.ONE_HOT_FEATURES.keys():
dim = features.ONE_HOT_FEATURES[key]
int_value = tft.compute_and_apply_vocabulary(fill_in_missing(
inputs[key]),
top_k=dim + 1)
outputs[features.transformed_name(key)] = convert_num_to_one_hot(
int_value, num_labels=dim + 1)
for key, bucket_count in features.BUCKET_FEATURES.items():
temp_feature = tft.bucketize(
convert_zip_code(fill_in_missing(inputs[key])),
bucket_count,
always_return_num_quantiles=False,
)
outputs[features.transformed_name(key)] = convert_num_to_one_hot(
temp_feature, num_labels=bucket_count + 1)
for key in features.TEXT_FEATURES.keys():
outputs[features.transformed_name(key)] = fill_in_missing(inputs[key])
outputs[features.transformed_name(features.LABEL_KEY)] = fill_in_missing(
inputs[features.LABEL_KEY])
return outputs
def _build_keras_model(tf_transform_output, hidden_units, learning_rate):
"""Creates a DNN Keras model for classifying taxi data.
Args:
hidden_units: [int], the layer sizes of the DNN (input layer first).
Returns:
A keras Model.
"""
numeric_columns = [
tf.feature_column.numeric_column(key=features.transformed_name(key),
shape=())
for key in features.NUMERIC_FEATURE_KEYS
]
categorical_columns = [
tf.feature_column.categorical_column_with_identity(
key=features.transformed_name(key),
num_buckets=tf_transform_output.
num_buckets_for_transformed_feature(
features.transformed_name(key)),
default_value=0) for key in features.CATEGORICAL_FEATURE_KEYS
]
indicator_columns = [
tf.feature_column.indicator_column(categorical_column)
for categorical_column in categorical_columns
]
model = _wide_and_deep_classifier(
# TODO(b/139668410) replace with premade wide_and_deep keras model
wide_columns=indicator_columns,
deep_columns=numeric_columns,
dnn_hidden_units=hidden_units,
learning_rate=learning_rate)
return model
def _build_keras_model(hparams: kerastuner.HyperParameters,
tf_transform_output: tft.TFTransformOutput) -> tf.keras.Model:
"""Creates a Keras WideDeep Classifier model.
Args:
hparams: Holds HyperParameters for tuning.
tf_transform_output: A TFTransformOutput.
Returns:
A keras Model.
"""
# Defines deep feature columns and input layers.
deep_columns = [
tf.feature_column.numeric_column(
key=features.transformed_name(key),
shape=())
for key in features.NUMERIC_FEATURE_KEYS
]
input_layers = {
column.key: tf.keras.layers.Input(name=column.key, shape=(), dtype=tf.float32)
for column in deep_columns
}
# Defines wide feature columns and input layers.
categorical_columns = [
tf.feature_column.categorical_column_with_identity(
key=features.transformed_name(key),
num_buckets=tf_transform_output.num_buckets_for_transformed_feature(features.transformed_name(key)),
default_value=0)
for key in features.CATEGORICAL_FEATURE_KEYS
]
wide_columns = [
tf.feature_column.indicator_column(categorical_column)
for categorical_column in categorical_columns
]
input_layers.update({
column.categorical_column.key: tf.keras.layers.Input(name=column.categorical_column.key, shape=(), dtype=tf.int32)
for column in wide_columns
})
# Build Keras model using hparams.
deep = tf.keras.layers.DenseFeatures(deep_columns)(input_layers)
for n in range(int(hparams.get('n_layers'))):
deep = tf.keras.layers.Dense(units=hparams.get('n_units_' + str(n + 1)))(deep)
wide = tf.keras.layers.DenseFeatures(wide_columns)(input_layers)
output = tf.keras.layers.Dense(features.NUM_CLASSES, activation='softmax')(
tf.keras.layers.concatenate([deep, wide]))
model = tf.keras.Model(input_layers, output)
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(lr=hparams.get('learning_rate')),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
model.summary(print_fn=absl.logging.info)
return model
def _input_fn(file_pattern, tf_transform_output, batch_size=200):
"""Generates features and label for tuning/training.
Args:
file_pattern: input tfrecord file pattern.
tf_transform_output: A TFTransformOutput.
batch_size: representing the number of consecutive elements of returned
dataset to combine in a single batch
Returns:
A dataset that contains (features, indices) tuple where features is a
dictionary of Tensors, and indices is a single Tensor of label indices.
"""
transformed_feature_spec = (
tf_transform_output.transformed_feature_spec().copy()
)
dataset = tf.data.experimental.make_batched_features_dataset(
file_pattern=file_pattern,
batch_size=batch_size,
features=transformed_feature_spec,
reader=_gzip_reader_fn,
label_key=features.transformed_name(features.LABEL_KEY),
)
return dataset
def _input_fn(filenames, tf_transform_output, batch_size=200):
"""Generates features and labels for training or evaluation.
Args:
filenames: [str] list of CSV files to read data from.
tf_transform_output: A TFTransformOutput.
batch_size: int First dimension size of the Tensors returned by input_fn
Returns:
A (features, indices) tuple where features is a dictionary of
Tensors, and indices is a single Tensor of label indices.
"""
transformed_feature_spec = (
tf_transform_output.transformed_feature_spec().copy())
dataset = tf.data.experimental.make_batched_features_dataset(
filenames,
batch_size,
transformed_feature_spec,
reader=_gzip_reader_fn)
transformed_features = tf.compat.v1.data.make_one_shot_iterator(
dataset).get_next()
# We pop the label because we do not want to use it as a feature while we're
# training.
return transformed_features, transformed_features.pop(
features.transformed_name(features.LABEL_KEY))
def preprocessing_fn(inputs):
"""tf.transform's callback function for preprocessing inputs.
Args:
inputs: map from feature keys to raw not-yet-transformed features.
Returns:
Map from string feature key to transformed feature operations.
"""
outputs = {}
for key in outputs.keys():
# Preserve this feature as a dense float, setting nan's to the mean.
outputs[features.transformed_name(key)] = tft.scale_to_z_score(
_fill_in_missing(inputs[key]))
outputs[features.transformed_name(
features.LABEL_KEY)] = inputs[features.LABEL_KEY]
return outputs
def preprocessing_fn(inputs):
"""Preprocesses Covertype Dataset."""
outputs = {}
# Scale numerical features
for key in features.NUMERIC_FEATURE_KEYS:
outputs[features.transformed_name(key)] = tft.scale_to_z_score(
_fill_in_missing(inputs[key]))
# Generate vocabularies and maps categorical features
for key in features.CATEGORICAL_FEATURE_KEYS:
outputs[features.transformed_name(key)] = tft.compute_and_apply_vocabulary(
x=_fill_in_missing(inputs[key]), num_oov_buckets=1, vocab_filename=key)
# Convert Cover_Type to dense tensor
outputs[features.transformed_name(features.LABEL_KEY)] = _fill_in_missing(
inputs[features.LABEL_KEY])
return outputs
def serve_tf_examples_fn(serialized_tf_examples):
"""Returns the output to be used in the serving signature."""
feature_spec = tf_transform_output.raw_feature_spec()
feature_spec.pop(features.LABEL_KEY)
parsed_features = tf.io.parse_example(serialized_tf_examples, feature_spec)
transformed_features = model.tft_layer(parsed_features)
transformed_features.pop(
features.transformed_name(features.LABEL_KEY), None)
return model(transformed_features)
def preprocessing_fn(inputs):
"""tf.transform's callback function for preprocessing inputs.
Args:
inputs: map from feature keys to raw not-yet-transformed features.
Returns:
Map from string feature key to transformed feature operations.
"""
outputs = {}
for key in features.DENSE_FLOAT_FEATURE_KEYS:
# Preserve this feature as a dense float, setting nan's to the mean.
outputs[features.transformed_name(key)] = tft.scale_to_z_score(
_fill_in_missing(inputs[key]))
for key in features.VOCAB_FEATURE_KEYS:
# Build a vocabulary for this feature.
outputs[features.transformed_name(
key)] = tft.compute_and_apply_vocabulary(
_fill_in_missing(inputs[key]),
top_k=features.VOCAB_SIZE,
num_oov_buckets=features.OOV_SIZE)
for key, num_buckets in zip(features.BUCKET_FEATURE_KEYS,
features.BUCKET_FEATURE_BUCKET_COUNT):
outputs[features.transformed_name(key)] = tft.bucketize(
_fill_in_missing(inputs[key]), num_buckets)
for key in features.CATEGORICAL_FEATURE_KEYS:
outputs[features.transformed_name(key)] = _fill_in_missing(inputs[key])
# TODO(b/157064428): Support label transformation for Keras.
# Do not apply label transformation as it will result in wrong evaluation.
outputs[features.transformed_name(
features.LABEL_KEY)] = inputs[features.LABEL_KEY]
return outputs
def _eval_input_receiver_fn(tf_transform_output, schema):
"""Build everything needed for the tf-model-analysis to run the model.
Args:
tf_transform_output: A TFTransformOutput.
schema: the schema of the input data.
Returns:
EvalInputReceiver function, which contains:
- Tensorflow graph which parses raw untransformed features, applies the
tf-transform preprocessing operators.
- Set of raw, untransformed features.
- Label against which predictions will be compared.
"""
# Notice that the inputs are raw features, not transformed features here.
raw_feature_spec = _get_raw_feature_spec(schema)
serialized_tf_example = tf.compat.v1.placeholder(
dtype=tf.string, shape=[None], name='input_example_tensor')
# Add a parse_example operator to the tensorflow graph, which will parse
# raw, untransformed, tf examples.
raw_features = tf.io.parse_example(
serialized=serialized_tf_example, features=raw_feature_spec)
# Now that we have our raw examples, process them through the tf-transform
# function computed during the preprocessing step.
transformed_features = tf_transform_output.transform_raw_features(
raw_features)
# The key name MUST be 'examples'.
receiver_tensors = {'examples': serialized_tf_example}
# NOTE: Model is driven by transformed features (since training works on the
# materialized output of TFT, but slicing will happen on raw features.
raw_features.update(transformed_features)
return tfma.export.EvalInputReceiver(
features=raw_features,
receiver_tensors=receiver_tensors,
labels=transformed_features[features.transformed_name(
features.LABEL_KEY)])
def _input_fn(file_pattern, data_accessor, tf_transform_output, batch_size=200):
"""Generates features and label for tuning/training.
Args:
file_pattern: List of paths or patterns of input tfrecord files.
data_accessor: DataAccessor for converting input to RecordBatch.
tf_transform_output: A TFTransformOutput.
batch_size: representing the number of consecutive elements of returned
dataset to combine in a single batch
Returns:
A dataset that contains (features, indices) tuple where features is a
dictionary of Tensors, and indices is a single Tensor of label indices.
"""
return data_accessor.tf_dataset_factory(
file_pattern,
dataset_options.TensorFlowDatasetOptions(
batch_size=batch_size,
label_key=features.transformed_name(features.LABEL_KEY)),
tf_transform_output.transformed_metadata.schema)
def get_model(show_summary: bool = True) -> tf.keras.models.Model:
"""
This function defines a Keras model and returns the model as a Keras object.
"""
# one-hot categorical features
input_features = []
for key, dim in features.ONE_HOT_FEATURES.items():
input_features.append(
tf.keras.Input(
shape=(dim + 1,), name=features.transformed_name(key)
)
)
# adding bucketized features
for key, dim in features.BUCKET_FEATURES.items():
input_features.append(
tf.keras.Input(
shape=(dim + 1,), name=features.transformed_name(key)
)
)
# adding text input features
input_texts = []
for key in features.TEXT_FEATURES.keys():
input_texts.append(
tf.keras.Input(
shape=(1,), name=features.transformed_name(key), dtype=tf.string
)
)
# embed text features
MODULE_URL = "https://tfhub.dev/google/universal-sentence-encoder/4"
embed = hub.KerasLayer(MODULE_URL)
reshaped_narrative = tf.reshape(input_texts[0], [-1])
embed_narrative = embed(reshaped_narrative)
deep_ff = tf.keras.layers.Reshape((512,), input_shape=(1, 512))(
embed_narrative
)
deep = tf.keras.layers.Dense(256, activation="relu")(deep_ff)
deep = tf.keras.layers.Dense(64, activation="relu")(deep)
deep = tf.keras.layers.Dense(16, activation="relu")(deep)
wide_ff = tf.keras.layers.concatenate(input_features)
wide = tf.keras.layers.Dense(16, activation="relu")(wide_ff)
both = tf.keras.layers.concatenate([deep, wide])
output = tf.keras.layers.Dense(1, activation="sigmoid")(both)
inputs = input_features + input_texts
keras_model = tf.keras.models.Model(inputs, output)
keras_model.compile(
optimizer=tf.keras.optimizers.Adam(
learning_rate=constants.LEARNING_RATE
),
loss="binary_crossentropy",
metrics=[
tf.keras.metrics.BinaryAccuracy(),
tf.keras.metrics.TruePositives(),
],
)
if show_summary:
keras_model.summary()
return keras_model
def testTransformedName(self):
name = "a_name"
expected = "a_name_xf"
self.assertEqual(expected, features.transformed_name(name))
