def _evaluate(nlp: Pipeline, domain: str, name: str) -> np.ndarray:
partial_results = []
dataset = absa.load_examples('semeval', domain, test=True)
batches = absa.utils.batches(dataset, batch_size=32)
for batch in batches:
predictions = nlp.transform(batch)
predictions = list(predictions) # Keep in memory to append at the end.
new_batch = [mask_tokens(nlp, prediction, k=2) for prediction in predictions]
new_predictions = nlp.transform(new_batch)
y_ref = [e.sentiment.value for e in predictions]
y_new = [e.sentiment.value for e in new_predictions]
partial_results.extend(zip(y_ref, y_new))
# It's not a generator because we cache function results.
return np.array(partial_results)
def _retrieve_labels(nlp: Pipeline, domain: str, is_test: bool) -> np.ndarray:
partial_results = []
examples = mask_examples(nlp, domain, is_test)
batches = absa.utils.batches(examples, batch_size=32)
for batch in batches:
indices, mask_index, batch_examples = zip(*batch)
predictions = nlp.transform(batch_examples)
y_hat = [e.sentiment.value for e in predictions]
partial_results.extend(zip(indices, mask_index, y_hat))
return np.array(partial_results)
def _retrieve_labels(
nlp: Pipeline,
domain: str,
part_parts: Tuple[int, int]
) -> np.ndarray:
partial_results = []
examples = mask_examples(nlp, domain, part_parts)
batches = absa.utils.batches(examples, batch_size=32)
for batch in batches:
indices, *masked_tokens_ij, batch_examples = zip(*batch)
predictions = nlp.transform(batch_examples)
y_hat = [e.sentiment.value for e in predictions]
partial_results.extend(zip(indices, *masked_tokens_ij, y_hat))
return np.array(partial_results)
def test_postprocess(nlp: Pipeline):
text = ("We are great fans of Slack.\n"
"The Slack often has bugs.\n"
"best of all is the warm vibe")
# Define a naive text_splitter.
nlp.text_splitter = lambda text: text.split('\n')
task = nlp.preprocess(text, aspects=['slack', 'price'])
predictions = nlp.transform(task.examples)
completed_task = nlp.postprocess(task, predictions)
assert len(completed_task.examples) == 6
assert completed_task.indices == [(0, 3), (3, 6)]
slack, price = completed_task
assert slack.text == price.text == text
# The sentiment among fragments are different. We normalize scores.
assert np.allclose(slack.scores, [0.03, 0.48, 0.48], atol=0.01)
# Please note that there is a problem with the neutral sentiment.
assert np.allclose(price.scores, [0.02, 0.49, 0.49], atol=0.01)