def __init__(self, config={}): super(DistilBert, self).__init__() self.masking = tf.keras.layers.Masking() self.fc1 = Dense(config['base_config']['dim'], activation='relu') self.model_config = DistilBertConfig.from_dict(config['base_config']) self.base = TFDistilBertModel(self.model_config) self.head = HEADS[config['head']['name']](config['head'])
def __init__(self, config={}): super(DistilBertNorm, self).__init__() self.masking = tf.keras.layers.Masking() self.fc1 = Dense(config['base_config']['dim']) self.norm1 = tf.keras.layers.LayerNormalization() self.gelu1 = tf.keras.layers.ReLU() self.model_config = DistilBertConfig.from_dict(config['base_config']) self.base = TFDistilBertModel(self.model_config) self.fc2 = Dense(1024) self.norm2 = tf.keras.layers.LayerNormalization() self.gelu2 = tf.keras.layers.ReLU() self.head = HEADS[config['head']['name']](config['head'])