def test_output_embeds_base_model(self):
model = TFCamembertModel.from_pretrained("jplu/tf-camembert-base")
input_ids = tf.convert_to_tensor(
[[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]],
dtype=tf.int32,
) # J'aime le camembert !"
output = model(input_ids)["last_hidden_state"]
expected_shape = tf.TensorShape((1, 10, 768))
self.assertEqual(output.shape, expected_shape)
# compare the actual values for a slice.
expected_slice = tf.convert_to_tensor(
[[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418],
[-0.1561, -0.1127, 0.2687]]],
dtype=tf.float32,
)
# camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
# camembert.eval()
# expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
self.assertTrue(
np.allclose(output[:, :3, :3].numpy(),
expected_slice.numpy(),
atol=1e-4))
def __init__(
self,
pretrained_model_name_or_path='jplu/tf-camembert-base',
reduce_output='cls_pooled',
trainable=True,
num_tokens=None,
**kwargs
):
super(CamemBERTEncoder, self).__init__()
try:
from transformers import TFCamembertModel
except ModuleNotFoundError:
logger.error(
' transformers is not installed. '
'In order to install all text feature dependencies run '
'pip install ludwig[text]'
)
sys.exit(-1)
self.transformer = TFCamembertModel.from_pretrained(
pretrained_model_name_or_path
)
self.reduce_output = reduce_output
if not self.reduce_output == 'cls_pooled':
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
self.transformer.trainable = trainable
self.transformer.resize_token_embeddings(num_tokens)
def bert_model(
ntargets=18,
seq_max=100,
nb_meta=134,
loss="categorical_crossentropy",
activation="softmax",
bert_model="jplu/tf-camembert-base",
):
"""Pre-defined architecture of a pre-trained Bert model.
Parameters
----------
ntargets : int, optional
Dimension of model output.
Default value, 18.
seq_max : int, optional
Maximum input length.
Default value, 100.
nb_meta : int, optional
Dimension of meta data input.
Default value, 252.
loss : str, optional
Loss function for training.
Default value, 'categorical_crossentropy'.
activation : str, optional
Activation function.
Default value, 'softmax'.
bert_model : str, optional
Model name from HuggingFace library or path to local model
Only Camembert and Flaubert supported
Default value, 'camembert-base'
Returns
-------
Model instance
"""
# Prevent the HuggingFace dependency
try:
from transformers import TFCamembertModel, TFFlaubertModel
except ModuleNotFoundError:
raise (
"""Please install transformers 3.4.0 (only version currently supported)
pip install melusine[transformers]""")
text_input = Input(shape=(seq_max, ), dtype="int32")
attention_input = Input(shape=(seq_max, ), dtype="int32")
if "camembert" in bert_model.lower():
x = TFCamembertModel.from_pretrained(bert_model)(
inputs=text_input, attention_mask=attention_input)[1]
elif "flaubert" in bert_model.lower():
x = TFFlaubertModel.from_pretrained(bert_model)(
inputs=text_input, attention_mask=attention_input)[0][:, 0, :]
else:
raise NotImplementedError(
"Bert model {} is not implemented.".format(bert_model))
if nb_meta == 0:
inputs = [text_input, attention_input]
concatenate_2 = x
else:
Meta_input = Input(shape=(nb_meta, ), dtype="float32")
inputs = [text_input, attention_input, Meta_input]
concatenate_1 = Meta_input
y = Dense(150, activation="linear")(concatenate_1)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
y = Dense(100, activation="linear")(y)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
y = Dense(80, activation="linear")(y)
y = Dropout(0.2)(y)
y = LeakyReLU(alpha=0.05)(y)
concatenate_2 = Concatenate(axis=1)([x, y])
z = Dense(200, activation="linear")(concatenate_2)
z = Dropout(0.2)(z)
z = LeakyReLU(alpha=0.05)(z)
z = Dense(100, activation="linear")(z)
z = Dropout(0.2)(z)
z = LeakyReLU(alpha=0.05)(z)
output = Dense(ntargets, activation=activation)(z)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer=Adam(learning_rate=5e-5),
loss=loss,
metrics=["accuracy"])
return model