def run_model(train_fp: Path,
val_fp: Path,
test_fp: Path,
config_fp: Path,
number_runs: int,
prediction_key: str,
save_dir: Optional[Path] = None,
only_produce_model: bool = False) -> None:
'''
:param train_fp: Path to file that contains JSON formatted training data
:param val_fp: Path to file that contains JSON formatted validation data
:param test_fp: Path to file that contains JSON formatted testing data
:param config_fp: Path to file that contains the models configuration
:param number_runs: Number of times to run the model
:param prediction_key: The key to save the predictions to within the
validation and test data
:param save_dir: Path to save the model to.
:param only_produce_model: Whether or not to run the model after all the
predictions have been made to save a model to
the `save_dir`
'''
# Test if all the predictions have already been made
temp_test_data = TargetTextCollection.load_json(test_fp)
temp_test_value = next(temp_test_data.dict_iterator())
predictions_left = number_runs
if prediction_key in temp_test_value:
number_runs_done = len(temp_test_value[prediction_key])
predictions_left = predictions_left - number_runs_done
if number_runs_done >= number_runs and not only_produce_model:
print('Predictions have already been made')
return
train_data = TargetTextCollection.load_json(train_fp)
val_data = TargetTextCollection.load_json(val_fp)
test_data = TargetTextCollection.load_json(test_fp)
key_mappings = {'sentiments': prediction_key}
if only_produce_model:
model = AllenNLPModel('model',
config_fp,
save_dir=save_dir,
predictor_name='target-sentiment')
model.fit(train_data, val_data, test_data)
for run in range(predictions_left):
print(f'Run number {run}')
if run == 0 and predictions_left == number_runs:
model = AllenNLPModel('model',
config_fp,
save_dir=save_dir,
predictor_name='target-sentiment')
else:
model = AllenNLPModel('model',
config_fp,
predictor_name='target-sentiment')
model.fit(train_data, val_data, test_data)
model.predict_into_collection(val_data, key_mapping=key_mappings)
model.predict_into_collection(test_data, key_mapping=key_mappings)
val_data.to_json_file(val_fp, include_metadata=True)
test_data.to_json_file(test_fp, include_metadata=True)
def test_eq(self):
examples = self._regular_examples()
example_0 = examples[0]
example_1 = examples[1]
example_2 = examples[2]
assert example_0 != example_1
assert example_2 == example_2
assert example_0 == example_0
del example_2['2']
assert example_2 == example_1
# Ensure that it only relies on the text_id and not the content of the
# target text
example_2['2'] = TargetText('hello how', '2')
example_1['2'] = TargetText('another test', '2')
assert example_2 == example_1
# Testing instance, if not TargetTextCollection should return False
text = 'today was good'
text_id = '1'
dict_version = [{'text_id': text_id, 'text': text}]
collection_version = TargetTextCollection(
[TargetText(**dict_version[0])])
assert dict_version != collection_version
# Should return False as they have different text_id's but have the
# content
alt_collection_version = TargetTextCollection(
[TargetText(text_id='2', text=text)])
assert collection_version != alt_collection_version
assert len(collection_version) == len(alt_collection_version)
def get_target_sentiment_distribution(dataset: TargetTextCollection) -> Dict[str, float]:
target_sentiment_distribution = Counter()
for target_text in dataset.values():
target_sentiment_distribution.update(target_text['target_sentiments'])
for key, value in target_sentiment_distribution.items():
target_sentiment_distribution[key] = round((value / dataset.number_targets()), 2) * 100
return dict(target_sentiment_distribution)
def test_add(self):
new_collection = TargetTextCollection()
assert len(new_collection) == 0
new_collection.add(self._target_text_example())
assert len(new_collection) == 1
assert '2' in new_collection
def test_predict_into_collection(self, batch_size: Optional[int],
append_if_exists: bool):
# Test that it raises an Error when the model attribute is not None
model_dir = self.TARGET_EXTRACTION_MODEL
model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger',
model_dir)
model.load()
# Test the normal case
train_data = TargetTextCollection.load_json(
self.TARGET_EXTRACTION_TRAIN_DATA)
key_mappings = {'tags': 'predicted_tags', 'words': 'predicted_tokens'}
train_data = model.predict_into_collection(train_data, key_mappings,
batch_size,
append_if_exists)
for target_data in train_data.values():
assert 'predicted_tags' in target_data
assert 'tags' not in target_data
assert 'predicted_tokens' in target_data
assert 'tokens' not in target_data
target_tokens = target_data['tokenized_text']
assert len(target_tokens) == len(target_data['predicted_tags'][0])
assert len(target_tokens) == len(
target_data['predicted_tokens'][0])
assert target_tokens == target_data['predicted_tokens'][0]
# This should be fine when append_if_exists is True and KeyError other
# wise.
if append_if_exists:
train_data = model.predict_into_collection(train_data,
key_mappings,
batch_size,
append_if_exists)
for target_data in train_data.values():
target_tokens = target_data['tokenized_text']
assert 2 == len(target_data['predicted_tags'])
assert target_data['predicted_tags'][0] == target_data[
'predicted_tags'][1]
assert target_tokens == target_data['predicted_tokens'][0]
assert target_tokens == target_data['predicted_tokens'][1]
else:
with pytest.raises(KeyError):
train_data = model.predict_into_collection(
train_data, key_mappings, batch_size, append_if_exists)
# Raise a KeyError when the `key_mappings` values are not within the
# TargetText
from collections import OrderedDict
key_mappings = OrderedDict([('tags', 'predicted_tags'),
('wordss', 'predicted_tokens')])
train_data = TargetTextCollection.load_json(
self.TARGET_EXTRACTION_TRAIN_DATA)
with pytest.raises(KeyError):
train_data = model.predict_into_collection(train_data,
key_mappings,
batch_size,
append_if_exists)
for target_data in train_data.values():
assert 'predicted_tags' not in target_data
assert 'predicted_tokens' not in target_data
def remove_multi_sentiment_targets(dataset: TargetTextCollection) -> TargetTextCollection:
dataset.one_sentiment_text('target_sentiments', False)
ids_to_remove = []
for text_id, target_text in dataset.items():
if 'text_sentiment' not in target_text:
ids_to_remove.append(text_id)
for id_to_remove in ids_to_remove:
del dataset[id_to_remove]
return dataset
def test__statistics_to_dataframe():
# Test with just one collection
target_stats = dataset_target_extraction_statistics([TRAIN_COLLECTION])
tl_1 = round((17 / 19.0) * 100, 2)
tl_2 = round((2 / 19.0) * 100, 2)
true_stats = {
'Name': 'train',
'No. Sentences': 6,
'No. Sentences(t)': 5,
'No. Targets': 19,
'No. Uniq Targets': 13,
'ATS': round(19 / 6.0, 2),
'ATS(t)': round(19 / 5.0, 2),
'TL 1 %': tl_1,
'TL 2 %': tl_2,
'TL 3+ %': 0,
'Mean Sentence Length': 15.33,
'Mean Sentence Length(t)': 16.6
}
true_stats_list = {key: [value] for key, value in true_stats.items()}
true_stats_df = pd.DataFrame(true_stats_list)
test_stats_df = _statistics_to_dataframe(target_stats)
pd.testing.assert_frame_equal(true_stats_df,
test_stats_df,
check_less_precise=2)
# Test with two collections
subcollection = TargetTextCollection(name='sub')
subcollection.add(TRAIN_COLLECTION["81207500773427072"])
subcollection.add(TRAIN_COLLECTION["78522643479064576"])
target_stats = dataset_target_extraction_statistics(
[subcollection, TRAIN_COLLECTION])
tl_1 = round((6 / 7.0) * 100, 2)
tl_2 = round((1 / 7.0) * 100, 2)
sub_stats = {
'Name': 'sub',
'No. Sentences': 2,
'No. Sentences(t)': 2,
'No. Targets': 7,
'No. Uniq Targets': 7,
'ATS': round(7 / 2.0, 2),
'ATS(t)': round(7 / 2.0, 2),
'TL 1 %': tl_1,
'TL 2 %': tl_2,
'TL 3+ %': 0,
'Mean Sentence Length': 13,
'Mean Sentence Length(t)': 13
}
true_stats_list = {
key: [value, true_stats[key]]
for key, value in sub_stats.items()
}
true_stats_df = pd.DataFrame(true_stats_list)
test_stats_df = _statistics_to_dataframe(target_stats)
pd.testing.assert_frame_equal(true_stats_df,
test_stats_df,
check_less_precise=2)
def test_to_json(self):
# no target text instances in the collection (empty collection)
new_collection = TargetTextCollection()
assert new_collection.to_json() == ''
# One target text in the collection
new_collection = TargetTextCollection([self._target_text_example()])
true_json_version = (
'{"text": "The laptop case was great and cover '
'was rubbish", "text_id": "2", "targets": ["laptop '
'case", "cover"], "spans": [[4, 15], [30, 35]], '
'"target_sentiments": [0, 1], "categories": '
'["LAPTOP#CASE", "LAPTOP"], "category_sentiments": null}')
assert new_collection.to_json() == true_json_version
# Multiple target text in the collection
new_collection = TargetTextCollection(self._target_text_examples()[:2])
true_json_version = (
'{"text": "The laptop case was great and cover '
'was rubbish", "text_id": "0", "targets": '
'["laptop case"], "spans": [[4, 15]], '
'"target_sentiments": [0], "categories": '
'["LAPTOP#CASE"], "category_sentiments": null}'
'\n{"text": "The laptop case was '
'great and cover was rubbish", "text_id": '
'"another_id", "targets": ["cover"], "spans": '
'[[30, 35]], "target_sentiments": [1], "categories": '
'["LAPTOP"], "category_sentiments": null}')
assert new_collection.to_json() == true_json_version
def test_metric_df(metric_function_name: str, metric_name: str,
include_run_number: bool):
model_1_collection = passable_example_multiple_preds('true_sentiments', 'model_1')
model_2_collection = passable_example_multiple_preds('true_sentiments', 'model_2')
combined_collection = TargetTextCollection()
for key, value in model_1_collection.items():
combined_collection.add(value)
combined_collection[key]['model_2'] = model_2_collection[key]['model_2']
metric_function = getattr(sentiment_metrics, metric_function_name)
# Test the array score version first
model_1_scores = metric_function(model_1_collection, 'true_sentiments', 'model_1',
average=False, array_scores=True)
model_2_scores = metric_function(model_2_collection, 'true_sentiments', 'model_2',
average=False, array_scores=True)
test_df = util.metric_df(combined_collection, metric_function, 'true_sentiments',
predicted_sentiment_keys=['model_1', 'model_2'],
average=False, array_scores=True, metric_name=metric_name,
include_run_number=include_run_number)
get_metric_name = 'metric' if None else metric_name
if include_run_number:
assert (4, 3) == test_df.shape
else:
assert (4, 2) == test_df.shape
for model_name, true_model_scores in [('model_1', model_1_scores),
('model_2', model_2_scores)]:
test_model_scores = test_df.loc[test_df['prediction key']==f'{model_name}'][f'{get_metric_name}']
assert true_model_scores == test_model_scores.to_list()
if include_run_number:
test_run_numbers = test_df.loc[test_df['prediction key']==f'{model_name}']['run number']
test_run_numbers = test_run_numbers.to_list()
assert [0, 1] == test_run_numbers
# Test the average version
model_1_scores = metric_function(model_1_collection, 'true_sentiments', 'model_1',
average=True, array_scores=False)
model_2_scores = metric_function(model_2_collection, 'true_sentiments', 'model_2',
average=True, array_scores=False)
if include_run_number:
with pytest.raises(ValueError):
util.metric_df(combined_collection, metric_function, 'true_sentiments',
predicted_sentiment_keys=['model_1', 'model_2'],
average=True, array_scores=False, metric_name=metric_name,
include_run_number=include_run_number)
else:
test_df = util.metric_df(combined_collection, metric_function, 'true_sentiments',
predicted_sentiment_keys=['model_1', 'model_2'],
average=True, array_scores=False, metric_name=metric_name,
include_run_number=include_run_number)
get_metric_name = 'metric' if None else metric_name
assert (2,2) == test_df.shape
for model_name, true_model_scores in [('model_1', model_1_scores),
('model_2', model_2_scores)]:
test_model_scores = test_df.loc[test_df['prediction key']==f'{model_name}'][f'{get_metric_name}']
test_model_scores = test_model_scores.to_list()
assert 1 == len(test_model_scores)
assert true_model_scores == test_model_scores[0]
def test_construction(self):
new_collection = TargetTextCollection(name='new_name')
assert new_collection.name == 'new_name'
new_collection = TargetTextCollection()
assert new_collection.name is ''
new_collection = TargetTextCollection(
target_texts=self._target_text_examples())
assert len(new_collection) == 3
assert '2' in new_collection
def test_force_targets(self):
text = 'The laptop casewas great and cover was rubbish'
spans = [Span(4, 15), Span(29, 34)]
targets = ['laptop case', 'cover']
target_text = TargetText(text=text,
text_id='0',
targets=targets,
spans=spans)
text_1 = 'The laptop casewas great andcover was rubbish'
spans_1 = [Span(4, 15), Span(28, 33)]
target_text_1 = TargetText(text=text_1,
text_id='1',
targets=targets,
spans=spans_1)
perfect_text = 'The laptop case was great and cover was rubbish'
perfect_spans = [Span(4, 15), Span(30, 35)]
# Test the single case
test_collection = TargetTextCollection([target_text])
test_collection.force_targets()
assert test_collection['0']['text'] == perfect_text
assert test_collection['0']['spans'] == perfect_spans
# Test the multiple case
test_collection = TargetTextCollection([target_text, target_text_1])
test_collection.force_targets()
for target_key in ['0', '1']:
assert test_collection[target_key]['text'] == perfect_text
assert test_collection[target_key]['spans'] == perfect_spans
def wrapper(target_collection: TargetTextCollection,
true_sentiment_key: str,
predicted_sentiment_key: str,
average: bool,
array_scores: bool,
assert_number_labels: Optional[int] = None,
ignore_label_differences: bool = True,
**kwargs) -> Union[float, np.ndarray]:
# Check that the TargetTextCollection contains both the true and
# predicted sentiment keys
unique_label_set = set()
total_number_model_predictions = 0
for target_object in target_collection.values():
target_object: TargetText
target_object._key_error(true_sentiment_key)
target_object._key_error(predicted_sentiment_key)
for true_label in target_object[true_sentiment_key]:
unique_label_set.add(true_label)
# Cannot have inconsistent number of model predictions
number_model_predictions = len(
target_object[predicted_sentiment_key])
if total_number_model_predictions == 0:
total_number_model_predictions = number_model_predictions
else:
if total_number_model_predictions != number_model_predictions:
raise ValueError(
'The number of predictions made per '
'Target within the collection is different'
f'. This TargetText could have no targets'
' within the collection thus this error '
'will be raise. TargetText that has an error'
f' {target_object}\nThe number of predcitions'
f' that this object should have: {total_number_model_predictions}'
)
# Cannot have zero predictions
if total_number_model_predictions == 0:
raise ValueError(
'The number of predictions made per target are zero')
# Perform the LabelError check
if assert_number_labels is not None:
number_unique_labels = len(unique_label_set)
if number_unique_labels != assert_number_labels:
raise LabelError(number_unique_labels, assert_number_labels)
# If the dataset has one model prediction per target then average and
# array_scores should be False
if number_model_predictions == 1:
if average or array_scores:
raise ValueError('When only one set of predictions per target'
' then `average` and `array_scores` have to '
'be both False')
else:
if average == array_scores:
raise ValueError(
'As the number of model predictions is > 1 '
'then either `average` or `array_scores` have '
'to be True but not both.')
return func(target_collection, true_sentiment_key,
predicted_sentiment_key, average, array_scores,
assert_number_labels, ignore_label_differences, **kwargs)
def test_strict_text_accuracy(true_sentiment_key: str,
predicted_sentiment_key: str):
# Test macro F1 works as should on one set of predictions
example, _, _ = passable_example(true_sentiment_key, predicted_sentiment_key)
score = strict_text_accuracy(example, true_sentiment_key,
predicted_sentiment_key, False, False, None)
assert 0.0 == score
# Test it works on multiple predictions
example, _, _ = passable_example_multiple_preds(true_sentiment_key,
predicted_sentiment_key)
score = strict_text_accuracy(example, true_sentiment_key,
predicted_sentiment_key, True, False, None)
assert 0.25 == score
# Test it works on multiple predictions
example, _, _ = passable_example_multiple_preds(true_sentiment_key,
predicted_sentiment_key)
score = strict_text_accuracy(example, true_sentiment_key,
predicted_sentiment_key, False, True, None)
assert [(0.0), (0.5)] == score
# Test the case where the TargetCollection has a sentence/text with no
# Targets/Predictions. This should raise a ValueError.
no_target = TargetText(text='hello how are you', text_id='10',
target_sentiments=[], targets=[], spans=[])
no_target[true_sentiment_key] = []
no_target[predicted_sentiment_key] = []
target_examples, _, _ = passable_example_multiple_preds(true_sentiment_key,
predicted_sentiment_key)
all_targets = list(target_examples.values())
all_targets.append(no_target)
test_collection = TargetTextCollection(all_targets)
assert 3 == len(test_collection)
with pytest.raises(ValueError):
strict_text_accuracy(test_collection, true_sentiment_key,
predicted_sentiment_key, True, False, None)
def get_data(
tweet_id_file: Path, all_tweet_data: Dict[str, Dict[str, Any]],
all_annotation_data: Dict[str, Dict[str,
Any]]) -> TargetTextCollection:
'''
:params tweet_id_file: File Path containing a Tweet id on each new line
:params all_tweet_data: Dictionary containing data about the Tweet where
the keys are Tweet ID's and values a Dict of
information about the Tweet.
:param all_annotation_data: Dictionary containing annotation data about
the Tweet where the keys are Tweet ID's
and values are the annotation data about
the Tweet in a form of a Dict.
:returns: The Twitter data into a
:class:`target_extraction.data_types.TargetTextCollection`
object.
'''
targets = []
with tweet_id_file.open('r') as tweet_id_data:
for tweet_id in tweet_id_data:
tweet_id = tweet_id.strip()
tweet_data = all_tweet_data[tweet_id]
anno_data = all_annotation_data[tweet_id]
targets.append(parse_tweet(tweet_data, anno_data, tweet_id))
return TargetTextCollection(targets)
def fit(self,
train_data: TargetTextCollection,
val_data: TargetTextCollection,
test_data: Optional[TargetTextCollection] = None) -> None:
'''
Given the training, validation, and optionally the test data it will
train the model that is defined in the model params file provided as
argument to the constructor of the class. Once trained the model can
be accessed through the `model` attribute.
NOTE: If the test data is given the model only uses it to fit to the
vocabularly that is within the test data, the model NEVER trains on
the test data.
:param train_data: Training data.
:param val_data: Validation data.
:param test_data: Optional, test data.
'''
model_params = self._preprocess_and_load_param_file(self._param_fp)
# Ensures that a different random seed is used each time
self._set_random_seeds(model_params)
with tempfile.TemporaryDirectory() as temp_dir:
train_fp = Path(temp_dir, 'train_data.json')
val_fp = Path(temp_dir, 'val_data.json')
# Write the training and validation data to json Optionally test as
# well
train_data.to_json_file(train_fp)
val_data.to_json_file(val_fp)
if test_data:
test_fp = Path(temp_dir, 'test_data.json')
test_data.to_json_file(test_fp)
self._add_dataset_paths(model_params, train_fp, val_fp,
test_fp)
model_params["evaluate_on_test"] = True
else:
self._add_dataset_paths(model_params, train_fp, val_fp)
save_dir = self.save_dir
if save_dir is None:
save_dir = Path(temp_dir, 'temp_save_dir')
temp_param_fp = Path(temp_dir, 'temp_param_file.json')
model_params.to_file(temp_param_fp.resolve())
trained_model = train_model_from_file(temp_param_fp, save_dir)
self.model = trained_model
def predict_into_collection(
self,
collection: TargetTextCollection,
key_mapping: Dict[str, str],
batch_size: Optional[int] = None,
append_if_exists: bool = True) -> TargetTextCollection:
'''
:param collection: The TargetTextCollection that is to be predicted on
and to be the store of the predicted data.
:param key_mapping: Dictionary mapping the prediction keys that contain
the prediction values to the keys that will store
those prediction values within the collection that
has been predicted on.
:param batch_size: Specify the batch size to predict on. If left None
defaults to 64 unless it is specified in the
`model_param_fp` within the constructor then
the batch size from the param file is used.
:param append_if_exists: If False and a TargetText within the collection
already has a prediction within the given key
based on the `key_mapping` then KeyError is
raised.
:returns: The collection that was predict on with the new predictions
within the collection stored in keys that are the values of
the `key_mapping` argument. Note that all predictions are
sotred within Lists within their respective keys in the
collection.
:raises KeyError: If the keys from `key_mapping` is not within the
prediction dictionary.
:raises KeyError: If `append_if_exists` is False and the a TargetText
within the collection already has a prediction within
the given key based on the `key_mapping` then this
is raised.
'''
for prediction, original_target in self._predict_iter(
collection.dict_iterator(),
batch_size=batch_size,
yield_original_target=True):
text_id = original_target['text_id']
# This happens first as we want an error to be raised before any
# data is added to the TargetTextCollection.
for prediction_key, collection_key in key_mapping.items():
if prediction_key not in prediction:
raise KeyError(
f'The key {prediction_key} from `key_mapping`'
f' {key_mapping} is not within the prediction'
f' {prediction} for the follwoing TargeText'
f' {original_target}')
for prediction_key, collection_key in key_mapping.items():
if collection_key not in collection[text_id]:
collection[text_id][collection_key] = []
elif not append_if_exists:
raise KeyError(
f'The key {collection_key} from `key_mapping`'
f' {key_mapping} already exists within the'
f' follwoing TargeText {original_target}')
collection[text_id][collection_key].append(
prediction[prediction_key])
return collection
def get_text_sentiment_distribution(dataset: TargetTextCollection) -> Dict[str, float]:
text_sentiment_distribution = Counter()
data_size = len(dataset)
for target_text in dataset.values():
text_sentiment_distribution.update([target_text['text_sentiment']])
for key, value in text_sentiment_distribution.items():
text_sentiment_distribution[key] = round((value / data_size), 2) * 100
return dict(text_sentiment_distribution)
def average_target_per_sentences(collection: TargetTextCollection,
sentence_must_contain_targets: bool) -> float:
'''
:param collection: Collection to calculate average target per sentence (ATS)
on.
:param sentence_must_contain_targets: Whether or not the sentences within the
collection must contains at least one
target. This filtering would affect
the value of the dominator stated in
the returns.
:returns: The ATS for the given collection. Which is:
Number of targets / number of sentences
'''
number_targets = float(collection.number_targets())
if sentence_must_contain_targets:
number_sentences = len(collection.samples_with_targets())
else:
number_sentences = len(collection)
return number_targets / float(number_sentences)
def test_from_json(self, name):
# no text given
test_collection = TargetTextCollection.from_json('', name=name)
assert TargetTextCollection() == test_collection
assert test_collection.name == name
# One target text instance in the text
new_collection = TargetTextCollection([self._target_text_example()],
name=name)
json_one_collection = new_collection.to_json()
assert new_collection == TargetTextCollection.from_json(
json_one_collection)
assert new_collection.name == name
# Multiple target text instances in the text
new_collection = TargetTextCollection(self._target_text_examples()[:2])
json_multi_collection = new_collection.to_json()
assert new_collection == TargetTextCollection.from_json(
json_multi_collection)
def run_model(train_fp: Path,
val_fp: Path,
test_fp: Path,
config_fp: Path,
save_test_fp: Path,
number_runs: int,
model_save_dir: Optional[Path] = None,
save_val_fp: Optional[Path] = None) -> None:
'''
:param train_fp: Path to file that contains JSON formatted training data
:param val_fp: Path to file that contains JSON formatted validation data
:param test_fp: Path to file that contains JSON formatted testing data
:param config_fp: Path to file that contains the models configuration
:param save_test_fp: Path to save the test data results
:param number_runs: Number of times to run the model
:param model_save_dir: Path to save the first trained model (optional)
:param save_val_fp: Path to save the validation data results (optional)
'''
train_data = TargetTextCollection.load_json(train_fp)
val_data = TargetTextCollection.load_json(val_fp)
test_data = TargetTextCollection.load_json(test_fp)
test_prediction_data = list(test_data.dict_iterator())
if save_val_fp:
val_prediction_data = list(val_data.dict_iterator())
for run in range(number_runs):
if run == 0 and model_save_dir:
model = AllenNLPModel('model',
config_fp,
predictor_name='target-sentiment',
save_dir=model_save_dir)
else:
model = AllenNLPModel('model',
config_fp,
predictor_name='target-sentiment')
model.fit(train_data, val_data, test_data)
predict_on(model, test_prediction_data, test_data)
if save_val_fp:
predict_on(model, val_prediction_data, val_data)
test_data.to_json_file(save_test_fp)
if save_val_fp:
val_data.to_json_file(save_val_fp)
def test_target_length_plot():
# standard/normal case
ax = target_length_plot([TRAIN_COLLECTION], 'targets', whitespace())
del ax
# cumulative percentage True
ax = target_length_plot([TRAIN_COLLECTION],
'targets',
whitespace(),
cumulative_percentage=True)
del ax
# Max target length
ax = target_length_plot([TRAIN_COLLECTION],
'targets',
whitespace(),
cumulative_percentage=True,
max_target_length=1)
del ax
# Can take consume an axes
fig, alt_ax = plt.subplots(1, 1)
ax = target_length_plot([TRAIN_COLLECTION],
'targets',
whitespace(),
cumulative_percentage=True,
max_target_length=1,
ax=alt_ax)
assert alt_ax == ax
del ax
plt.close(fig)
# Can take more than one collection
alt_collection = copy.deepcopy(list(TRAIN_COLLECTION.dict_iterator()))
alt_collection = [TargetText(**v) for v in alt_collection]
alt_collection = TargetTextCollection(alt_collection)
alt_collection.name = 'Another'
ax = target_length_plot([TRAIN_COLLECTION, alt_collection],
'targets',
whitespace(),
cumulative_percentage=True,
max_target_length=1)
assert alt_ax != ax
del alt_ax
del ax
def test_train_and_test_dataset():
with tempfile.TemporaryDirectory() as temp_dir:
# Test both the normal cahce_dir and the given cache dir
for data_dir in [None, Path(temp_dir, 'twitter data')]:
train_data = wang_2017_election_twitter_train(data_dir)
test_data = wang_2017_election_twitter_test(data_dir)
assert len(train_data) > len(test_data)
combined_data = TargetTextCollection.combine(train_data, test_data)
assert 11899 == combined_data.number_targets()
def test_pos_text(self):
# Test the normal case with one TargetText Instance in the collection
test_collection = TargetTextCollection([self._target_text_example()])
test_collection.pos_text(spacy_tagger())
pos_answer = [
'DET', 'NOUN', 'NOUN', 'VERB', 'ADJ', 'CCONJ', 'NOUN', 'VERB',
'ADJ'
]
assert test_collection['2']['pos_tags'] == pos_answer
# Test the normal case with multiple TargetText Instance in the
# collection
test_collection = TargetTextCollection(self._target_text_examples())
test_collection.pos_text(spacy_tagger())
assert test_collection['2']['pos_tags'] == pos_answer
# Ensure that at least one error is raised but all of these tests are
# covered in the TargetText tests.
with pytest.raises(TypeError):
test_collection.pos_text(str.strip)
def _regular_examples(self) -> List[TargetTextCollection]:
target_text_examples = self._target_text_examples()
collection_examples = []
for i in range(1, 4):
collection = TargetTextCollection()
for example in target_text_examples[:i]:
collection[example['text_id']] = example
collection_examples.append(collection)
return collection_examples
def test_target_extraction_fit(self, test_data: bool):
model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger')
assert model.model is None
train_data = TargetTextCollection.load_json(
self.TARGET_EXTRACTION_TRAIN_DATA)
val_data = TargetTextCollection.load_json(
self.TARGET_EXTRACTION_TRAIN_DATA)
tokens_in_vocab = [
'at', 'case', 'was', 'the', 'day', 'great', 'cover', 'office',
'another', 'and', 'rubbish', 'laptop', '@@PADDING@@', '@@UNKNOWN@@'
]
if test_data:
tokens_in_vocab = tokens_in_vocab + ['better']
test_data = TargetTextCollection.load_json(
self.TARGET_EXTRACTION_TEST_DATA)
model.fit(train_data, val_data, test_data)
else:
model.fit(train_data, val_data)
token_index = model.model.vocab.get_token_to_index_vocabulary('tokens')
assert len(token_index) == len(tokens_in_vocab)
for token in tokens_in_vocab:
assert token in token_index
# Check attributes have changed.
assert model.model is not None
assert isinstance(model.model, Model)
# Check that it will save to a directory of our choosing
with tempfile.TemporaryDirectory() as save_dir:
saved_model_fp = Path(save_dir, 'model.tar.gz')
assert not saved_model_fp.exists()
model = AllenNLPModel('TE',
self.CONFIG_FILE,
'target-tagger',
save_dir=save_dir)
model.fit(train_data, val_data)
assert saved_model_fp.exists()
def no_pred_values(true_sentiment_key: str, predicted_sentiment_key: str
) -> Tuple[TargetTextCollection, List[str], List[List[str]]]:
example_1 = TargetText(text_id='1', text='some text')
example_1[true_sentiment_key] = []
example_1[predicted_sentiment_key] = []
example_2 = TargetText(text_id='2', text='some text')
example_2[true_sentiment_key] = []
example_2[predicted_sentiment_key] = []
true_labels = []
pred_labels = []
return TargetTextCollection([example_1, example_2]), true_labels, pred_labels
def test_load_json(self, name):
empty_json_fp = Path(self._json_data_dir(),
'empty_target_instance.json')
empty_collection = TargetTextCollection.load_json(empty_json_fp,
name=name)
assert TargetTextCollection() == empty_collection
assert empty_collection.name == name
# Ensure that it raises an error when loading a bad json file
wrong_json_fp = Path(self._json_data_dir(),
'wrong_target_instance.json')
with pytest.raises(ValueError):
TargetTextCollection.load_json(wrong_json_fp, name=name)
# Ensure that it can load a single target text instance correctly
one_target_json_fp = Path(self._json_data_dir(),
'one_target_instance.json')
one_target_collection = TargetTextCollection.load_json(
one_target_json_fp)
assert len(one_target_collection) == 1
assert one_target_collection['0'][
'text'] == 'The laptop case was great and cover was rubbish'
assert one_target_collection['0']['target_sentiments'] == [0]
assert one_target_collection['0']['category_sentiments'] == ['pos']
assert one_target_collection['0']['categories'] == ['LAPTOP#CASE']
assert one_target_collection['0']['spans'] == [Span(4, 15)]
assert one_target_collection['0']['targets'] == ['laptop case']
# Ensure that it can load multiple target text instances
two_target_json_fp = Path(self._json_data_dir(),
'one_target_one_empty_instance.json')
two_target_collection = TargetTextCollection.load_json(
two_target_json_fp)
assert len(two_target_collection) == 2
def test_one_sample_per_span(self, remove_empty: bool):
# Case where nothing should change with respect to the number of spans
# but will change the values target_sentiments to None etc
target_text = TargetText(
text_id='0',
spans=[Span(4, 15)],
text='The laptop case was great and cover was rubbish',
target_sentiments=[0],
targets=['laptop case'])
collection = TargetTextCollection([target_text])
new_collection = collection.one_sample_per_span(
remove_empty=remove_empty)
assert new_collection == collection
assert new_collection['0']['spans'] == [Span(4, 15)]
assert new_collection['0']['target_sentiments'] == None
assert collection['0']['target_sentiments'] == [0]
# Should change the number of Spans.
assert target_text['target_sentiments'] == [0]
target_text._storage['spans'] = [Span(4, 15), Span(4, 15)]
target_text._storage['targets'] = ['laptop case', 'laptop case']
target_text._storage['target_sentiments'] = [0, 1]
diff_collection = TargetTextCollection([target_text])
new_collection = diff_collection.one_sample_per_span(
remove_empty=remove_empty)
assert new_collection == collection
assert new_collection['0']['spans'] == [Span(4, 15)]
assert new_collection['0']['target_sentiments'] == None
assert diff_collection['0']['target_sentiments'] == [0, 1]
assert diff_collection['0']['spans'] == [Span(4, 15), Span(4, 15)]
def get_samples_per_sentiment(
collection: TargetTextCollection) -> Dict[str, int]:
'''
:param collection: The collection being tested
:returns: A dictionary where keys are the sentiment value names and the
the value is the number of samples that have that sentiment
value name e.g. {'pos': 500, 'neg': 400}
'''
sentiment_samples = defaultdict(lambda: 0)
for value in collection.values():
for sentiment in value['category_sentiments']:
sentiment_samples[sentiment] += 1
return sentiment_samples
def passable_diff_num_labels(true_sentiment_key: str,
predicted_sentiment_key: str
) -> Tuple[TargetTextCollection, List[str], List[List[str]]]:
example_1 = TargetText(text_id='1', text='some text')
example_1[true_sentiment_key] = ['pos', 'neg']
example_1[predicted_sentiment_key] = [['neg', 'pos']]
example_2 = TargetText(text_id='2', text='some text')
example_2[true_sentiment_key] = ['pos', 'neg', 'neu']
example_2[predicted_sentiment_key] = [['neg', 'neg', 'pos']]
true_labels = ['pos', 'neg', 'pos', 'neg', 'neu']
pred_labels = [['neg', 'pos', 'neg', 'neg', 'pos']]
return TargetTextCollection([example_1, example_2]), true_labels, pred_labels
