def __init__(self,
embedding_dim,
projection_dim,
vocab_size=1000,
**kwargs):
super(EmbeddingTabNet, self).__init__(**kwargs)
# For demonstration purposes, assume we have a "data_col" column.
cat_col = tf.feature_column.categorical_column_with_identity(
"data_col", num_buckets=vocab_size + 1, default_value=0)
embed_col = tf.feature_column.embedding_column(cat_col, embedding_dim)
self.embed_layer = tf.keras.layers.DenseFeatures([embed_col])
self.projection = tf.keras.layers.Dense(projection_dim,
activation='linear',
use_bias=False)
# Assume we have `d` classes, mapped as y = (data_col % d), where data col can take 1000 values.
# Note: `num_features` *must* be the length of the projection dim.
self.tabnet_model = tabnet.TabNetClassifier(
None,
num_classes=num_classes,
num_features=projection_dim,
feature_dim=4,
output_dim=4,
num_decision_steps=2,
relaxation_factor=1.0,
sparsity_coefficient=1e-5,
batch_momentum=0.98,
virtual_batch_size=None,
norm_type='group',
num_groups=2)
ds_train = ds_full.take(train_size)
ds_train = ds_train.map(transform)
ds_train = ds_train.batch(BATCH_SIZE)
ds_test = ds_full.skip(train_size)
ds_test = ds_test.map(transform)
ds_test = ds_test.batch(BATCH_SIZE)
feature_columns = []
for col_name in col_names:
feature_columns.append(tf.feature_column.numeric_column(col_name))
# Group Norm does better for small datasets
model = tabnet.TabNetClassifier(feature_columns, num_classes=3,
feature_dim=8, output_dim=4,
num_decision_steps=4, relaxation_factor=1.0,
sparsity_coefficient=1e-5, batch_momentum=0.98,
virtual_batch_size=None, norm_type='group',
num_groups=1)
lr = tf.keras.optimizers.schedules.ExponentialDecay(0.01, decay_steps=100, decay_rate=0.9, staircase=False)
optimizer = tf.keras.optimizers.Adam(lr)
model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(ds_train, epochs=100, validation_data=ds_test, verbose=2)
model.summary()
print()
if os.path.exists('logs/'):
shutil.rmtree('logs/')
data_dir='mnist',
shuffle_files=False)
ds_train = ds_train.shuffle(60000)
ds_train = ds_train.map(transform)
ds_train = ds_train.batch(BATCH_SIZE)
ds_test = ds_test.map(transform)
ds_test = ds_test.batch(BATCH_SIZE)
# Use Group Normalization for small batch sizes
model = tabnet.TabNetClassifier(feature_columns=None,
num_classes=10,
num_features=784,
feature_dim=16,
output_dim=16,
num_decision_steps=5,
relaxation_factor=1.5,
sparsity_coefficient=0.,
batch_momentum=0.98,
virtual_batch_size=None,
norm_type='group',
num_groups=-1)
lr = tf.keras.optimizers.schedules.ExponentialDecay(0.001,
decay_steps=500,
decay_rate=0.9,
staircase=False)
optimizer = tf.keras.optimizers.Adam(lr)
model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(ds_train, epochs=5, validation_data=ds_test, verbose=2)