def create_and_check_for_sequence_classification(
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
):
config.num_labels = self.num_labels
model = TFLongformerForSequenceClassification(config=config)
output = model(
input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
).logits
self.parent.assertListEqual(shape_list(output), [self.batch_size, self.num_labels])
def build(self):
# Handle the Meta Data
ids = tf.keras.Input((self.config.max_length, ),
dtype=tf.int32,
name='input_ids')
# Handle the Transformer
self.base_transformer_model = TFLongformerForSequenceClassification.from_pretrained(
"allenai/longformer-base-4096")
x = self.base_transformer_model.longformer(ids) # Get the main Layer
hidden_state = x[0][:, 0, :]
dl_model = tf.keras.Model(inputs={"input_ids": ids},
outputs=[hidden_state])
self.deep_legis_model = dl_model
def compile(self, lr, freeze_pretrained):
loss = 'binary_crossentropy'
# Wrap up model + Compile with optimizer and loss function
# self.model = Model(inputs=word_inputs, outputs=[outputs])
'''self.model = TFBertForSequenceClassification.from_pretrained(Configuration['model']['uri'], num_labels=4271)'''
# Import the needed model(Bert, Roberta or DistilBert) with output_hidden_states=True
if "longformer" in Configuration["model"]["uri"]:
transformer_model = TFLongformerForSequenceClassification.from_pretrained(
Configuration['model']['uri'], num_labels=4271)
else:
transformer_model = TFAutoModelForSequenceClassification.from_pretrained(
Configuration['model']['uri'], num_labels=4271)
if freeze_pretrained:
"""
[ < transformers.modeling_tf_bert.TFBertMainLayer
at
0x169a23e10 >,
< tensorflow.python.keras.layers.core.Dropout
at
0x169abde90 >,
< tensorflow.python.keras.layers.core.Dense
at
0x169ac31d0 >]"""
transformer_model.layers[0].trainable = False
input_ids = tf.keras.Input(
shape=(Configuration['sampling']['max_sequence_size'], ),
dtype='int32')
attention_mask = tf.keras.Input(
shape=(Configuration['sampling']['max_sequence_size'], ),
dtype='int32')
transformer = transformer_model([input_ids, attention_mask],
training=True)
hidden_states = transformer[0] # get output_hidden_states
output = tf.keras.activations.sigmoid(hidden_states)
self.model = tf.keras.models.Model(inputs=[input_ids, attention_mask],
outputs=output)
self.model.compile(optimizer=Adam(lr=lr), loss=loss)