def test_label(nlp: BertPipeline):
# We add the pattern recognizer to the pipeline.
pattern_recognizer = AttentionPatternRecognizer()
nlp.pattern_recognizer = pattern_recognizer
text_1 = ("We are great fans of Slack, but we wish the subscriptions "
"were more accessible to small startups.")
text_2 = "The Slack often has bugs."
text_3 = "best of all is the warm vibe"
aspect = "slack"
examples = [
Example(text_1, aspect),
Example(text_2, aspect),
Example(text_3, aspect)
]
tokenized_examples = nlp.tokenize(examples)
input_batch = nlp.encode(tokenized_examples)
output_batch = nlp.predict(input_batch)
labeled_examples = nlp.label(tokenized_examples, output_batch)
labeled_examples = list(labeled_examples)
labeled_1, labeled_2, labeled_3 = labeled_examples
assert labeled_1.sentiment == Sentiment.positive
assert labeled_2.sentiment == Sentiment.negative
assert isinstance(labeled_1.scores, list)
assert np.argmax(labeled_1.aspect_representation.look_at) == 5
assert np.argmax(labeled_2.aspect_representation.look_at) == 1
# We need to calibrate the model. The prediction should be neutral.
# In fact, the model does not recognize the aspect correctly.
assert labeled_3.sentiment == Sentiment.positive
assert np.allclose(labeled_3.aspect_representation.look_at,
[1.0, 0.16, 0.50, 0.54, 0.34, 0.39, 0.12],
atol=0.01)
def test_evaluate(nlp: BertPipeline):
examples = load_examples(dataset='semeval', domain='restaurant', test=True)
metric = tf.metrics.Accuracy()
result = nlp.evaluate(examples[:10], metric, batch_size=10)
result = result.numpy()
# The model predicts the first 10 labels perfectly.
assert result == 1
result = nlp.evaluate(examples[10:20], metric, batch_size=10)
assert np.isclose(result, 0.95)
def test_predict(nlp: BertPipeline):
text_1 = ("We are great fans of Slack, but we wish the subscriptions "
"were more accessible to small startups.")
text_2 = "We are great fans of Slack"
aspect = "Slack"
examples = [Example(text_1, aspect), Example(text_2, aspect)]
tokenized_examples = nlp.tokenize(examples)
input_batch = nlp.encode(tokenized_examples)
output_batch = nlp.predict(input_batch)
assert output_batch.scores.shape == [2, 3]
assert output_batch.hidden_states.shape == [2, 13, 23, 768]
assert output_batch.attentions.shape == [2, 12, 12, 23, 23]
assert output_batch.attention_grads.shape == [2, 12, 12, 23, 23]
scores = output_batch.scores.numpy()
assert np.argmax(scores, axis=-1).tolist() == [2, 2]
def test_preprocess(nlp: BertPipeline):
# We split a document into spans (in this case, into sentences).
nlp.text_splitter = lambda text: text.split('\n')
raw_document = ("This is the test sentence 1.\n"
"This is the test sentence 2.\n"
"This is the test sentence 3.")
task = nlp.preprocess(text=raw_document, aspects=['aspect_1', 'aspect_2'])
assert len(task.subtasks) == 2
assert list(task.subtasks) == ['aspect_1', 'aspect_2']
assert len(task.batch) == 6
assert task.indices == [(0, 3), (3, 6)]
subtask_1, subtask_2 = task
assert subtask_1.text == subtask_2.text == raw_document
assert subtask_1.aspect == 'aspect_1'
assert len(subtask_1.examples) == 3
def nlp() -> BertPipeline:
# Here, we do more integration like tests rather than
# mocked unit tests. We show up how the pipeline works,
# and it's why we use this well-defined pipeline fixture.
name = 'absa/classifier-rest-0.1'
tokenizer = transformers.BertTokenizer.from_pretrained(name)
model = BertABSClassifier.from_pretrained(name)
nlp = BertPipeline(model, tokenizer)
return nlp
def nlp() -> BertPipeline:
# Here, we do more integration like tests rather than
# mocked unit tests. We show up how the pipeline works,
# and it's why we use this well-defined pipeline fixture.
name = 'absa/classifier-rest-0.1'
tokenizer = transformers.BertTokenizer.from_pretrained(name)
# We pass a config explicitly (however, it can be downloaded automatically)
config = BertABSCConfig.from_pretrained(name)
model = BertABSClassifier.from_pretrained(name, config=config)
nlp = BertPipeline(model, tokenizer)
return nlp
def test_encode(nlp: BertPipeline):
text_1 = ("We are great fans of Slack, but we wish the subscriptions "
"were more accessible to small startups.")
text_2 = "We are great fans of Slack"
aspect = "Slack"
examples = [Example(text_1, aspect), Example(text_2, aspect)]
tokenized_examples = nlp.tokenize(examples)
input_batch = nlp.encode(tokenized_examples)
assert isinstance(input_batch.token_ids, tf.Tensor)
# 101 the CLS token, 102 the SEP tokens.
token_ids = input_batch.token_ids.numpy()
values = [101, 2057, 2024, 2307, 4599, 1997, 19840, 102, 19840, 102]
assert token_ids[1, :10].tolist() == values
assert token_ids[0, :7].tolist() == values[:7]
# The second sequence should be padded (shorter),
# and attention mask should be set.
assert np.allclose(token_ids[1, 10:], 0)
attention_mask = input_batch.attention_mask.numpy()
assert np.allclose(attention_mask[1, 10:], 0)
# Check how the tokenizer marked the segments.
token_type_ids = input_batch.token_type_ids.numpy()
assert token_type_ids[0, -2:].tolist() == [1, 1]
assert np.allclose(token_type_ids[0, :-2], 0)
def test_get_completed_task(nlp: BertPipeline):
text = ("We are great fans of Slack.\n"
"The Slack often has bugs.\n"
"best of all is the warm vibe")
# Make sure we have defined a text_splitter, even naive.
nlp.text_splitter = lambda text: text.split('\n')
task = nlp.preprocess(text, aspects=['slack', 'price'])
tokenized_examples = task.batch
input_batch = nlp.encode(tokenized_examples)
output_batch = nlp.predict(input_batch)
aspect_span_labeled = nlp.label(tokenized_examples, output_batch)
completed_task = nlp.get_completed_task(task, aspect_span_labeled)
assert len(completed_task.batch) == 6
assert completed_task.indices == [(0, 3), (3, 6)]
slack, price = completed_task
assert slack.text == price.text == text
# The sentiment among fragments are different. We normalize scores.
assert np.allclose(slack.scores, [0.06, 0.46, 0.48], atol=0.01)
# Please note once gain that there is a problem
# with the neutral sentiment, model is over-fitted.
assert np.allclose(price.scores, [0.06, 0.42, 0.52], atol=0.01)
def test_sanity_classifier():
np.random.seed(1)
tf.random.set_seed(1)
# This sanity test verifies and presents how train a classifier. To
# build our model, we have to define a config, which contains all required
# information needed to build the `BertABSClassifier` model (including
# the BERT language model). In this example, we use default parameters
# (which are set up for our best performance), but of course, you can pass
# your own parameters (maybe you would be interested to change the number
# of polarities to classify, or properties of the BERT itself).
base_model_name = 'bert-base-uncased'
strategy = tf.distribute.OneDeviceStrategy('CPU')
with strategy.scope():
config = BertABSCConfig.from_pretrained(base_model_name)
model = BertABSClassifier.from_pretrained(base_model_name,
config=config)
tokenizer = transformers.BertTokenizer.from_pretrained(base_model_name)
optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-8)
# The first step to train the model is to define a dataset. The dataset
# can be understood as a non-differential part of the training pipeline
# The dataset knows how to transform human-understandable example into
# model understandable batches. You are not obligated to use datasets,
# you can create your own iterable, which transforms classifier example
# to the classifier train batches.
example = LabeledExample(text='The breakfast was delicious, really great.',
aspect='breakfast',
sentiment=Sentiment.positive)
dataset = ClassifierDataset(examples=[example, example],
tokenizer=tokenizer,
batch_size=2)
# To easily adjust optimization process to our needs, we define custom
# training loops called routines (in contrast to use built-in methods as
# the `fit`). Each routine has its own optimization step wherein we can
# control which and how parameters are updated (according to the custom
# training paradigm presented in the TensorFlow 2.0). We iterate over a
# dataset, perform train/test optimization steps, and collect results
# using callbacks (which have a similar interface as the tf.keras.Callback).
# Please take a look at the `train_classifier` function for more details.
logger, loss_value = Logger(), LossHistory()
train_classifier(model,
optimizer,
dataset,
epochs=10,
callbacks=[logger, loss_value],
strategy=strategy)
# Our model should easily overfit in just 10 iterations.
assert 1 < loss_value.train[1] < 2
assert loss_value.train[10] < 2e-2
# In the end, we would like to save the model. Our implementation
# gentle extend the *transformers* lib capabilities, in consequences,
# `BertABSClassifier` inherits from the `TFBertPreTrainedModel`, and
# we can do a serialization easily.
model.save_pretrained('.')
# To make sure that the model serving works fine, we initialize the model
# and the config once again. We perform the check on a single example.
del model, config
config = BertABSCConfig.from_pretrained('.')
model = BertABSClassifier.from_pretrained('.', config=config)
batch = next(iter(dataset))
model_outputs = model.call(batch.token_ids,
attention_mask=batch.attention_mask,
token_type_ids=batch.token_type_ids)
logits, *details = model_outputs
loss_fn = tf.nn.softmax_cross_entropy_with_logits
loss_value = loss_fn(batch.target_labels, logits, axis=-1, name='Loss')
loss_value = loss_value.numpy().mean()
assert loss_value < 2e-2
# The training procedure is roughly verified. Now, using our tuned model,
# we can build the `BertPipeline`. The pipeline is the high level interface
# to perform predictions. The model should be highly confident that this is
# the positive example (verify the softmax scores).
nlp = BertPipeline(model, tokenizer)
breakfast, = nlp(example.text, aspects=['breakfast'])
assert breakfast.sentiment == Sentiment.positive
assert np.allclose(breakfast.scores, [0.0, 0.0, 0.99], atol=0.01)
# That's all, clean up the configuration, and the temporary saved model.
os.remove('config.json')
os.remove('tf_model.h5')