def __init__(self, embedding_dim, projection_dim, vocab_size=1000, **kwargs): super(EmbeddingTabNet, self).__init__(**kwargs) # For demonstration purposes, assume we have a "data_col" column. cat_col = tf.feature_column.categorical_column_with_identity( "data_col", num_buckets=vocab_size + 1, default_value=0) embed_col = tf.feature_column.embedding_column(cat_col, embedding_dim) self.embed_layer = tf.keras.layers.DenseFeatures([embed_col]) self.projection = tf.keras.layers.Dense(projection_dim, activation='linear', use_bias=False) # Assume we have `d` classes, mapped as y = (data_col % d), where data col can take 1000 values. # Note: `num_features` *must* be the length of the projection dim. self.tabnet_model = tabnet.TabNetClassifier( None, num_classes=num_classes, num_features=projection_dim, feature_dim=4, output_dim=4, num_decision_steps=2, relaxation_factor=1.0, sparsity_coefficient=1e-5, batch_momentum=0.98, virtual_batch_size=None, norm_type='group', num_groups=2)
ds_train = ds_full.take(train_size) ds_train = ds_train.map(transform) ds_train = ds_train.batch(BATCH_SIZE) ds_test = ds_full.skip(train_size) ds_test = ds_test.map(transform) ds_test = ds_test.batch(BATCH_SIZE) feature_columns = [] for col_name in col_names: feature_columns.append(tf.feature_column.numeric_column(col_name)) # Group Norm does better for small datasets model = tabnet.TabNetClassifier(feature_columns, num_classes=3, feature_dim=8, output_dim=4, num_decision_steps=4, relaxation_factor=1.0, sparsity_coefficient=1e-5, batch_momentum=0.98, virtual_batch_size=None, norm_type='group', num_groups=1) lr = tf.keras.optimizers.schedules.ExponentialDecay(0.01, decay_steps=100, decay_rate=0.9, staircase=False) optimizer = tf.keras.optimizers.Adam(lr) model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(ds_train, epochs=100, validation_data=ds_test, verbose=2) model.summary() print() if os.path.exists('logs/'): shutil.rmtree('logs/')
data_dir='mnist', shuffle_files=False) ds_train = ds_train.shuffle(60000) ds_train = ds_train.map(transform) ds_train = ds_train.batch(BATCH_SIZE) ds_test = ds_test.map(transform) ds_test = ds_test.batch(BATCH_SIZE) # Use Group Normalization for small batch sizes model = tabnet.TabNetClassifier(feature_columns=None, num_classes=10, num_features=784, feature_dim=16, output_dim=16, num_decision_steps=5, relaxation_factor=1.5, sparsity_coefficient=0., batch_momentum=0.98, virtual_batch_size=None, norm_type='group', num_groups=-1) lr = tf.keras.optimizers.schedules.ExponentialDecay(0.001, decay_steps=500, decay_rate=0.9, staircase=False) optimizer = tf.keras.optimizers.Adam(lr) model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(ds_train, epochs=5, validation_data=ds_test, verbose=2)