def test_output_embeds_base_model(self):
model = TFFlaubertModel.from_pretrained("jplu/tf-flaubert-small-cased")
input_ids = tf.convert_to_tensor(
[[0, 158, 735, 2592, 1424, 6727, 82, 1]],
dtype=tf.int32,
) # "J'aime flaubert !"
output = model(input_ids)[0]
expected_shape = tf.TensorShape((1, 8, 512))
self.assertEqual(output.shape, expected_shape)
# compare the actual values for a slice.
expected_slice = tf.convert_to_tensor(
[[
[-1.8768773, -1.566555, 0.27072418],
[-1.6920038, -0.5873505, 1.9329599],
[-2.9563985, -1.6993835, 1.7972052],
]],
dtype=tf.float32,
)
self.assertTrue(
np.allclose(output[:, :3, :3].numpy(),
expected_slice.numpy(),
atol=1e-4))
def create_and_check_flaubert_model(
self,
config,
input_ids,
token_type_ids,
input_lengths,
sequence_labels,
token_labels,
is_impossible_labels,
choice_labels,
input_mask,
):
model = TFFlaubertModel(config=config)
inputs = {
"input_ids": input_ids,
"lengths": input_lengths,
"langs": token_type_ids
}
result = model(inputs)
inputs = [input_ids, input_mask]
result = model(inputs)
self.parent.assertEqual(
result.last_hidden_state.shape,
(self.batch_size, self.seq_length, self.hidden_size))
def create_and_check_flaubert_model(
self,
config,
input_ids,
token_type_ids,
input_lengths,
sequence_labels,
token_labels,
is_impossible_labels,
choice_labels,
input_mask,
):
model = TFFlaubertModel(config=config)
inputs = {
"input_ids": input_ids,
"lengths": input_lengths,
"langs": token_type_ids
}
outputs = model(inputs)
inputs = [input_ids, input_mask]
outputs = model(inputs)
sequence_output = outputs[0]
result = {
"sequence_output": sequence_output.numpy(),
}
self.parent.assertListEqual(
list(result["sequence_output"].shape),
[self.batch_size, self.seq_length, self.hidden_size])
def __init__(
self,
pretrained_model_name_or_path='jplu/tf-flaubert-small-cased',
reduce_output='sum',
trainable=True,
num_tokens=None,
**kwargs
):
super(FlauBERTEncoder, self).__init__()
try:
from transformers import TFFlaubertModel
except ModuleNotFoundError:
logger.error(
' transformers is not installed. '
'In order to install all text feature dependencies run '
'pip install ludwig[text]'
)
sys.exit(-1)
self.transformer = TFFlaubertModel.from_pretrained(
pretrained_model_name_or_path
)
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
self.transformer.trainable = trainable
self.transformer.resize_token_embeddings(num_tokens)
def test_model_from_pretrained(self):
for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
model = TFFlaubertModel.from_pretrained(model_name)
self.assertIsNotNone(model)
def bert_model(
ntargets=18,
seq_max=100,
nb_meta=134,
loss="categorical_crossentropy",
activation="softmax",
bert_model="jplu/tf-camembert-base",
):
"""Pre-defined architecture of a pre-trained Bert model.
Parameters
----------
ntargets : int, optional
Dimension of model output.
Default value, 18.
seq_max : int, optional
Maximum input length.
Default value, 100.
nb_meta : int, optional
Dimension of meta data input.
Default value, 252.
loss : str, optional
Loss function for training.
Default value, 'categorical_crossentropy'.
activation : str, optional
Activation function.
Default value, 'softmax'.
bert_model : str, optional
Model name from HuggingFace library or path to local model
Only Camembert and Flaubert supported
Default value, 'camembert-base'
Returns
-------
Model instance
"""
# Prevent the HuggingFace dependency
try:
from transformers import TFCamembertModel, TFFlaubertModel
except ModuleNotFoundError:
raise (
"""Please install transformers 3.4.0 (only version currently supported)
pip install melusine[transformers]""")
text_input = Input(shape=(seq_max, ), dtype="int32")
attention_input = Input(shape=(seq_max, ), dtype="int32")
if "camembert" in bert_model.lower():
x = TFCamembertModel.from_pretrained(bert_model)(
inputs=text_input, attention_mask=attention_input)[1]
elif "flaubert" in bert_model.lower():
x = TFFlaubertModel.from_pretrained(bert_model)(
inputs=text_input, attention_mask=attention_input)[0][:, 0, :]
else:
raise NotImplementedError(
"Bert model {} is not implemented.".format(bert_model))
if nb_meta == 0:
inputs = [text_input, attention_input]
concatenate_2 = x
else:
Meta_input = Input(shape=(nb_meta, ), dtype="float32")
inputs = [text_input, attention_input, Meta_input]
concatenate_1 = Meta_input
y = Dense(150, activation="linear")(concatenate_1)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
y = Dense(100, activation="linear")(y)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
y = Dense(80, activation="linear")(y)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
concatenate_2 = Concatenate(axis=1)([x, y])
z = Dense(200, activation="linear")(concatenate_2)
z = Dropout(0.2)(z)
z = LeakyReLU(alpha=0.05)(z)
z = Dense(100, activation="linear")(z)
z = Dropout(0.2)(z)
z = LeakyReLU(alpha=0.05)(z)
output = Dense(ntargets, activation=activation)(z)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer=Adam(learning_rate=5e-5),
loss=loss,
metrics=["accuracy"])
return model