def __init__(self, config={}):
super(DistilBert, self).__init__()
self.masking = tf.keras.layers.Masking()
self.fc1 = Dense(config['base_config']['dim'], activation='relu')
self.model_config = DistilBertConfig.from_dict(config['base_config'])
self.base = TFDistilBertModel(self.model_config)
self.head = HEADS[config['head']['name']](config['head'])
def __init__(self, config={}):
super(DistilBertNorm, self).__init__()
self.masking = tf.keras.layers.Masking()
self.fc1 = Dense(config['base_config']['dim'])
self.norm1 = tf.keras.layers.LayerNormalization()
self.gelu1 = tf.keras.layers.ReLU()
self.model_config = DistilBertConfig.from_dict(config['base_config'])
self.base = TFDistilBertModel(self.model_config)
self.fc2 = Dense(1024)
self.norm2 = tf.keras.layers.LayerNormalization()
self.gelu2 = tf.keras.layers.ReLU()
self.head = HEADS[config['head']['name']](config['head'])
