def run_model(train_fp: Path, val_fp: Path, test_fp: Path, config_fp: Path, number_runs: int, prediction_key: str, save_dir: Optional[Path] = None, only_produce_model: bool = False) -> None: ''' :param train_fp: Path to file that contains JSON formatted training data :param val_fp: Path to file that contains JSON formatted validation data :param test_fp: Path to file that contains JSON formatted testing data :param config_fp: Path to file that contains the models configuration :param number_runs: Number of times to run the model :param prediction_key: The key to save the predictions to within the validation and test data :param save_dir: Path to save the model to. :param only_produce_model: Whether or not to run the model after all the predictions have been made to save a model to the `save_dir` ''' # Test if all the predictions have already been made temp_test_data = TargetTextCollection.load_json(test_fp) temp_test_value = next(temp_test_data.dict_iterator()) predictions_left = number_runs if prediction_key in temp_test_value: number_runs_done = len(temp_test_value[prediction_key]) predictions_left = predictions_left - number_runs_done if number_runs_done >= number_runs and not only_produce_model: print('Predictions have already been made') return train_data = TargetTextCollection.load_json(train_fp) val_data = TargetTextCollection.load_json(val_fp) test_data = TargetTextCollection.load_json(test_fp) key_mappings = {'sentiments': prediction_key} if only_produce_model: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) for run in range(predictions_left): print(f'Run number {run}') if run == 0 and predictions_left == number_runs: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') else: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) model.predict_into_collection(val_data, key_mapping=key_mappings) model.predict_into_collection(test_data, key_mapping=key_mappings) val_data.to_json_file(val_fp, include_metadata=True) test_data.to_json_file(test_fp, include_metadata=True)
def test_eq(self): examples = self._regular_examples() example_0 = examples[0] example_1 = examples[1] example_2 = examples[2] assert example_0 != example_1 assert example_2 == example_2 assert example_0 == example_0 del example_2['2'] assert example_2 == example_1 # Ensure that it only relies on the text_id and not the content of the # target text example_2['2'] = TargetText('hello how', '2') example_1['2'] = TargetText('another test', '2') assert example_2 == example_1 # Testing instance, if not TargetTextCollection should return False text = 'today was good' text_id = '1' dict_version = [{'text_id': text_id, 'text': text}] collection_version = TargetTextCollection( [TargetText(**dict_version[0])]) assert dict_version != collection_version # Should return False as they have different text_id's but have the # content alt_collection_version = TargetTextCollection( [TargetText(text_id='2', text=text)]) assert collection_version != alt_collection_version assert len(collection_version) == len(alt_collection_version)
def get_target_sentiment_distribution(dataset: TargetTextCollection) -> Dict[str, float]: target_sentiment_distribution = Counter() for target_text in dataset.values(): target_sentiment_distribution.update(target_text['target_sentiments']) for key, value in target_sentiment_distribution.items(): target_sentiment_distribution[key] = round((value / dataset.number_targets()), 2) * 100 return dict(target_sentiment_distribution)
def test_add(self): new_collection = TargetTextCollection() assert len(new_collection) == 0 new_collection.add(self._target_text_example()) assert len(new_collection) == 1 assert '2' in new_collection
def test_predict_into_collection(self, batch_size: Optional[int], append_if_exists: bool): # Test that it raises an Error when the model attribute is not None model_dir = self.TARGET_EXTRACTION_MODEL model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger', model_dir) model.load() # Test the normal case train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) key_mappings = {'tags': 'predicted_tags', 'words': 'predicted_tokens'} train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): assert 'predicted_tags' in target_data assert 'tags' not in target_data assert 'predicted_tokens' in target_data assert 'tokens' not in target_data target_tokens = target_data['tokenized_text'] assert len(target_tokens) == len(target_data['predicted_tags'][0]) assert len(target_tokens) == len( target_data['predicted_tokens'][0]) assert target_tokens == target_data['predicted_tokens'][0] # This should be fine when append_if_exists is True and KeyError other # wise. if append_if_exists: train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): target_tokens = target_data['tokenized_text'] assert 2 == len(target_data['predicted_tags']) assert target_data['predicted_tags'][0] == target_data[ 'predicted_tags'][1] assert target_tokens == target_data['predicted_tokens'][0] assert target_tokens == target_data['predicted_tokens'][1] else: with pytest.raises(KeyError): train_data = model.predict_into_collection( train_data, key_mappings, batch_size, append_if_exists) # Raise a KeyError when the `key_mappings` values are not within the # TargetText from collections import OrderedDict key_mappings = OrderedDict([('tags', 'predicted_tags'), ('wordss', 'predicted_tokens')]) train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) with pytest.raises(KeyError): train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): assert 'predicted_tags' not in target_data assert 'predicted_tokens' not in target_data
def remove_multi_sentiment_targets(dataset: TargetTextCollection) -> TargetTextCollection: dataset.one_sentiment_text('target_sentiments', False) ids_to_remove = [] for text_id, target_text in dataset.items(): if 'text_sentiment' not in target_text: ids_to_remove.append(text_id) for id_to_remove in ids_to_remove: del dataset[id_to_remove] return dataset
def test__statistics_to_dataframe(): # Test with just one collection target_stats = dataset_target_extraction_statistics([TRAIN_COLLECTION]) tl_1 = round((17 / 19.0) * 100, 2) tl_2 = round((2 / 19.0) * 100, 2) true_stats = { 'Name': 'train', 'No. Sentences': 6, 'No. Sentences(t)': 5, 'No. Targets': 19, 'No. Uniq Targets': 13, 'ATS': round(19 / 6.0, 2), 'ATS(t)': round(19 / 5.0, 2), 'TL 1 %': tl_1, 'TL 2 %': tl_2, 'TL 3+ %': 0, 'Mean Sentence Length': 15.33, 'Mean Sentence Length(t)': 16.6 } true_stats_list = {key: [value] for key, value in true_stats.items()} true_stats_df = pd.DataFrame(true_stats_list) test_stats_df = _statistics_to_dataframe(target_stats) pd.testing.assert_frame_equal(true_stats_df, test_stats_df, check_less_precise=2) # Test with two collections subcollection = TargetTextCollection(name='sub') subcollection.add(TRAIN_COLLECTION["81207500773427072"]) subcollection.add(TRAIN_COLLECTION["78522643479064576"]) target_stats = dataset_target_extraction_statistics( [subcollection, TRAIN_COLLECTION]) tl_1 = round((6 / 7.0) * 100, 2) tl_2 = round((1 / 7.0) * 100, 2) sub_stats = { 'Name': 'sub', 'No. Sentences': 2, 'No. Sentences(t)': 2, 'No. Targets': 7, 'No. Uniq Targets': 7, 'ATS': round(7 / 2.0, 2), 'ATS(t)': round(7 / 2.0, 2), 'TL 1 %': tl_1, 'TL 2 %': tl_2, 'TL 3+ %': 0, 'Mean Sentence Length': 13, 'Mean Sentence Length(t)': 13 } true_stats_list = { key: [value, true_stats[key]] for key, value in sub_stats.items() } true_stats_df = pd.DataFrame(true_stats_list) test_stats_df = _statistics_to_dataframe(target_stats) pd.testing.assert_frame_equal(true_stats_df, test_stats_df, check_less_precise=2)
def test_to_json(self): # no target text instances in the collection (empty collection) new_collection = TargetTextCollection() assert new_collection.to_json() == '' # One target text in the collection new_collection = TargetTextCollection([self._target_text_example()]) true_json_version = ( '{"text": "The laptop case was great and cover ' 'was rubbish", "text_id": "2", "targets": ["laptop ' 'case", "cover"], "spans": [[4, 15], [30, 35]], ' '"target_sentiments": [0, 1], "categories": ' '["LAPTOP#CASE", "LAPTOP"], "category_sentiments": null}') assert new_collection.to_json() == true_json_version # Multiple target text in the collection new_collection = TargetTextCollection(self._target_text_examples()[:2]) true_json_version = ( '{"text": "The laptop case was great and cover ' 'was rubbish", "text_id": "0", "targets": ' '["laptop case"], "spans": [[4, 15]], ' '"target_sentiments": [0], "categories": ' '["LAPTOP#CASE"], "category_sentiments": null}' '\n{"text": "The laptop case was ' 'great and cover was rubbish", "text_id": ' '"another_id", "targets": ["cover"], "spans": ' '[[30, 35]], "target_sentiments": [1], "categories": ' '["LAPTOP"], "category_sentiments": null}') assert new_collection.to_json() == true_json_version
def test_metric_df(metric_function_name: str, metric_name: str, include_run_number: bool): model_1_collection = passable_example_multiple_preds('true_sentiments', 'model_1') model_2_collection = passable_example_multiple_preds('true_sentiments', 'model_2') combined_collection = TargetTextCollection() for key, value in model_1_collection.items(): combined_collection.add(value) combined_collection[key]['model_2'] = model_2_collection[key]['model_2'] metric_function = getattr(sentiment_metrics, metric_function_name) # Test the array score version first model_1_scores = metric_function(model_1_collection, 'true_sentiments', 'model_1', average=False, array_scores=True) model_2_scores = metric_function(model_2_collection, 'true_sentiments', 'model_2', average=False, array_scores=True) test_df = util.metric_df(combined_collection, metric_function, 'true_sentiments', predicted_sentiment_keys=['model_1', 'model_2'], average=False, array_scores=True, metric_name=metric_name, include_run_number=include_run_number) get_metric_name = 'metric' if None else metric_name if include_run_number: assert (4, 3) == test_df.shape else: assert (4, 2) == test_df.shape for model_name, true_model_scores in [('model_1', model_1_scores), ('model_2', model_2_scores)]: test_model_scores = test_df.loc[test_df['prediction key']==f'{model_name}'][f'{get_metric_name}'] assert true_model_scores == test_model_scores.to_list() if include_run_number: test_run_numbers = test_df.loc[test_df['prediction key']==f'{model_name}']['run number'] test_run_numbers = test_run_numbers.to_list() assert [0, 1] == test_run_numbers # Test the average version model_1_scores = metric_function(model_1_collection, 'true_sentiments', 'model_1', average=True, array_scores=False) model_2_scores = metric_function(model_2_collection, 'true_sentiments', 'model_2', average=True, array_scores=False) if include_run_number: with pytest.raises(ValueError): util.metric_df(combined_collection, metric_function, 'true_sentiments', predicted_sentiment_keys=['model_1', 'model_2'], average=True, array_scores=False, metric_name=metric_name, include_run_number=include_run_number) else: test_df = util.metric_df(combined_collection, metric_function, 'true_sentiments', predicted_sentiment_keys=['model_1', 'model_2'], average=True, array_scores=False, metric_name=metric_name, include_run_number=include_run_number) get_metric_name = 'metric' if None else metric_name assert (2,2) == test_df.shape for model_name, true_model_scores in [('model_1', model_1_scores), ('model_2', model_2_scores)]: test_model_scores = test_df.loc[test_df['prediction key']==f'{model_name}'][f'{get_metric_name}'] test_model_scores = test_model_scores.to_list() assert 1 == len(test_model_scores) assert true_model_scores == test_model_scores[0]
def test_construction(self): new_collection = TargetTextCollection(name='new_name') assert new_collection.name == 'new_name' new_collection = TargetTextCollection() assert new_collection.name is '' new_collection = TargetTextCollection( target_texts=self._target_text_examples()) assert len(new_collection) == 3 assert '2' in new_collection
def test_force_targets(self): text = 'The laptop casewas great and cover was rubbish' spans = [Span(4, 15), Span(29, 34)] targets = ['laptop case', 'cover'] target_text = TargetText(text=text, text_id='0', targets=targets, spans=spans) text_1 = 'The laptop casewas great andcover was rubbish' spans_1 = [Span(4, 15), Span(28, 33)] target_text_1 = TargetText(text=text_1, text_id='1', targets=targets, spans=spans_1) perfect_text = 'The laptop case was great and cover was rubbish' perfect_spans = [Span(4, 15), Span(30, 35)] # Test the single case test_collection = TargetTextCollection([target_text]) test_collection.force_targets() assert test_collection['0']['text'] == perfect_text assert test_collection['0']['spans'] == perfect_spans # Test the multiple case test_collection = TargetTextCollection([target_text, target_text_1]) test_collection.force_targets() for target_key in ['0', '1']: assert test_collection[target_key]['text'] == perfect_text assert test_collection[target_key]['spans'] == perfect_spans
def wrapper(target_collection: TargetTextCollection, true_sentiment_key: str, predicted_sentiment_key: str, average: bool, array_scores: bool, assert_number_labels: Optional[int] = None, ignore_label_differences: bool = True, **kwargs) -> Union[float, np.ndarray]: # Check that the TargetTextCollection contains both the true and # predicted sentiment keys unique_label_set = set() total_number_model_predictions = 0 for target_object in target_collection.values(): target_object: TargetText target_object._key_error(true_sentiment_key) target_object._key_error(predicted_sentiment_key) for true_label in target_object[true_sentiment_key]: unique_label_set.add(true_label) # Cannot have inconsistent number of model predictions number_model_predictions = len( target_object[predicted_sentiment_key]) if total_number_model_predictions == 0: total_number_model_predictions = number_model_predictions else: if total_number_model_predictions != number_model_predictions: raise ValueError( 'The number of predictions made per ' 'Target within the collection is different' f'. This TargetText could have no targets' ' within the collection thus this error ' 'will be raise. TargetText that has an error' f' {target_object}\nThe number of predcitions' f' that this object should have: {total_number_model_predictions}' ) # Cannot have zero predictions if total_number_model_predictions == 0: raise ValueError( 'The number of predictions made per target are zero') # Perform the LabelError check if assert_number_labels is not None: number_unique_labels = len(unique_label_set) if number_unique_labels != assert_number_labels: raise LabelError(number_unique_labels, assert_number_labels) # If the dataset has one model prediction per target then average and # array_scores should be False if number_model_predictions == 1: if average or array_scores: raise ValueError('When only one set of predictions per target' ' then `average` and `array_scores` have to ' 'be both False') else: if average == array_scores: raise ValueError( 'As the number of model predictions is > 1 ' 'then either `average` or `array_scores` have ' 'to be True but not both.') return func(target_collection, true_sentiment_key, predicted_sentiment_key, average, array_scores, assert_number_labels, ignore_label_differences, **kwargs)
def test_strict_text_accuracy(true_sentiment_key: str, predicted_sentiment_key: str): # Test macro F1 works as should on one set of predictions example, _, _ = passable_example(true_sentiment_key, predicted_sentiment_key) score = strict_text_accuracy(example, true_sentiment_key, predicted_sentiment_key, False, False, None) assert 0.0 == score # Test it works on multiple predictions example, _, _ = passable_example_multiple_preds(true_sentiment_key, predicted_sentiment_key) score = strict_text_accuracy(example, true_sentiment_key, predicted_sentiment_key, True, False, None) assert 0.25 == score # Test it works on multiple predictions example, _, _ = passable_example_multiple_preds(true_sentiment_key, predicted_sentiment_key) score = strict_text_accuracy(example, true_sentiment_key, predicted_sentiment_key, False, True, None) assert [(0.0), (0.5)] == score # Test the case where the TargetCollection has a sentence/text with no # Targets/Predictions. This should raise a ValueError. no_target = TargetText(text='hello how are you', text_id='10', target_sentiments=[], targets=[], spans=[]) no_target[true_sentiment_key] = [] no_target[predicted_sentiment_key] = [] target_examples, _, _ = passable_example_multiple_preds(true_sentiment_key, predicted_sentiment_key) all_targets = list(target_examples.values()) all_targets.append(no_target) test_collection = TargetTextCollection(all_targets) assert 3 == len(test_collection) with pytest.raises(ValueError): strict_text_accuracy(test_collection, true_sentiment_key, predicted_sentiment_key, True, False, None)
def get_data( tweet_id_file: Path, all_tweet_data: Dict[str, Dict[str, Any]], all_annotation_data: Dict[str, Dict[str, Any]]) -> TargetTextCollection: ''' :params tweet_id_file: File Path containing a Tweet id on each new line :params all_tweet_data: Dictionary containing data about the Tweet where the keys are Tweet ID's and values a Dict of information about the Tweet. :param all_annotation_data: Dictionary containing annotation data about the Tweet where the keys are Tweet ID's and values are the annotation data about the Tweet in a form of a Dict. :returns: The Twitter data into a :class:`target_extraction.data_types.TargetTextCollection` object. ''' targets = [] with tweet_id_file.open('r') as tweet_id_data: for tweet_id in tweet_id_data: tweet_id = tweet_id.strip() tweet_data = all_tweet_data[tweet_id] anno_data = all_annotation_data[tweet_id] targets.append(parse_tweet(tweet_data, anno_data, tweet_id)) return TargetTextCollection(targets)
def fit(self, train_data: TargetTextCollection, val_data: TargetTextCollection, test_data: Optional[TargetTextCollection] = None) -> None: ''' Given the training, validation, and optionally the test data it will train the model that is defined in the model params file provided as argument to the constructor of the class. Once trained the model can be accessed through the `model` attribute. NOTE: If the test data is given the model only uses it to fit to the vocabularly that is within the test data, the model NEVER trains on the test data. :param train_data: Training data. :param val_data: Validation data. :param test_data: Optional, test data. ''' model_params = self._preprocess_and_load_param_file(self._param_fp) # Ensures that a different random seed is used each time self._set_random_seeds(model_params) with tempfile.TemporaryDirectory() as temp_dir: train_fp = Path(temp_dir, 'train_data.json') val_fp = Path(temp_dir, 'val_data.json') # Write the training and validation data to json Optionally test as # well train_data.to_json_file(train_fp) val_data.to_json_file(val_fp) if test_data: test_fp = Path(temp_dir, 'test_data.json') test_data.to_json_file(test_fp) self._add_dataset_paths(model_params, train_fp, val_fp, test_fp) model_params["evaluate_on_test"] = True else: self._add_dataset_paths(model_params, train_fp, val_fp) save_dir = self.save_dir if save_dir is None: save_dir = Path(temp_dir, 'temp_save_dir') temp_param_fp = Path(temp_dir, 'temp_param_file.json') model_params.to_file(temp_param_fp.resolve()) trained_model = train_model_from_file(temp_param_fp, save_dir) self.model = trained_model
def predict_into_collection( self, collection: TargetTextCollection, key_mapping: Dict[str, str], batch_size: Optional[int] = None, append_if_exists: bool = True) -> TargetTextCollection: ''' :param collection: The TargetTextCollection that is to be predicted on and to be the store of the predicted data. :param key_mapping: Dictionary mapping the prediction keys that contain the prediction values to the keys that will store those prediction values within the collection that has been predicted on. :param batch_size: Specify the batch size to predict on. If left None defaults to 64 unless it is specified in the `model_param_fp` within the constructor then the batch size from the param file is used. :param append_if_exists: If False and a TargetText within the collection already has a prediction within the given key based on the `key_mapping` then KeyError is raised. :returns: The collection that was predict on with the new predictions within the collection stored in keys that are the values of the `key_mapping` argument. Note that all predictions are sotred within Lists within their respective keys in the collection. :raises KeyError: If the keys from `key_mapping` is not within the prediction dictionary. :raises KeyError: If `append_if_exists` is False and the a TargetText within the collection already has a prediction within the given key based on the `key_mapping` then this is raised. ''' for prediction, original_target in self._predict_iter( collection.dict_iterator(), batch_size=batch_size, yield_original_target=True): text_id = original_target['text_id'] # This happens first as we want an error to be raised before any # data is added to the TargetTextCollection. for prediction_key, collection_key in key_mapping.items(): if prediction_key not in prediction: raise KeyError( f'The key {prediction_key} from `key_mapping`' f' {key_mapping} is not within the prediction' f' {prediction} for the follwoing TargeText' f' {original_target}') for prediction_key, collection_key in key_mapping.items(): if collection_key not in collection[text_id]: collection[text_id][collection_key] = [] elif not append_if_exists: raise KeyError( f'The key {collection_key} from `key_mapping`' f' {key_mapping} already exists within the' f' follwoing TargeText {original_target}') collection[text_id][collection_key].append( prediction[prediction_key]) return collection
def get_text_sentiment_distribution(dataset: TargetTextCollection) -> Dict[str, float]: text_sentiment_distribution = Counter() data_size = len(dataset) for target_text in dataset.values(): text_sentiment_distribution.update([target_text['text_sentiment']]) for key, value in text_sentiment_distribution.items(): text_sentiment_distribution[key] = round((value / data_size), 2) * 100 return dict(text_sentiment_distribution)
def average_target_per_sentences(collection: TargetTextCollection, sentence_must_contain_targets: bool) -> float: ''' :param collection: Collection to calculate average target per sentence (ATS) on. :param sentence_must_contain_targets: Whether or not the sentences within the collection must contains at least one target. This filtering would affect the value of the dominator stated in the returns. :returns: The ATS for the given collection. Which is: Number of targets / number of sentences ''' number_targets = float(collection.number_targets()) if sentence_must_contain_targets: number_sentences = len(collection.samples_with_targets()) else: number_sentences = len(collection) return number_targets / float(number_sentences)
def test_from_json(self, name): # no text given test_collection = TargetTextCollection.from_json('', name=name) assert TargetTextCollection() == test_collection assert test_collection.name == name # One target text instance in the text new_collection = TargetTextCollection([self._target_text_example()], name=name) json_one_collection = new_collection.to_json() assert new_collection == TargetTextCollection.from_json( json_one_collection) assert new_collection.name == name # Multiple target text instances in the text new_collection = TargetTextCollection(self._target_text_examples()[:2]) json_multi_collection = new_collection.to_json() assert new_collection == TargetTextCollection.from_json( json_multi_collection)
def run_model(train_fp: Path, val_fp: Path, test_fp: Path, config_fp: Path, save_test_fp: Path, number_runs: int, model_save_dir: Optional[Path] = None, save_val_fp: Optional[Path] = None) -> None: ''' :param train_fp: Path to file that contains JSON formatted training data :param val_fp: Path to file that contains JSON formatted validation data :param test_fp: Path to file that contains JSON formatted testing data :param config_fp: Path to file that contains the models configuration :param save_test_fp: Path to save the test data results :param number_runs: Number of times to run the model :param model_save_dir: Path to save the first trained model (optional) :param save_val_fp: Path to save the validation data results (optional) ''' train_data = TargetTextCollection.load_json(train_fp) val_data = TargetTextCollection.load_json(val_fp) test_data = TargetTextCollection.load_json(test_fp) test_prediction_data = list(test_data.dict_iterator()) if save_val_fp: val_prediction_data = list(val_data.dict_iterator()) for run in range(number_runs): if run == 0 and model_save_dir: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment', save_dir=model_save_dir) else: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) predict_on(model, test_prediction_data, test_data) if save_val_fp: predict_on(model, val_prediction_data, val_data) test_data.to_json_file(save_test_fp) if save_val_fp: val_data.to_json_file(save_val_fp)
def test_target_length_plot(): # standard/normal case ax = target_length_plot([TRAIN_COLLECTION], 'targets', whitespace()) del ax # cumulative percentage True ax = target_length_plot([TRAIN_COLLECTION], 'targets', whitespace(), cumulative_percentage=True) del ax # Max target length ax = target_length_plot([TRAIN_COLLECTION], 'targets', whitespace(), cumulative_percentage=True, max_target_length=1) del ax # Can take consume an axes fig, alt_ax = plt.subplots(1, 1) ax = target_length_plot([TRAIN_COLLECTION], 'targets', whitespace(), cumulative_percentage=True, max_target_length=1, ax=alt_ax) assert alt_ax == ax del ax plt.close(fig) # Can take more than one collection alt_collection = copy.deepcopy(list(TRAIN_COLLECTION.dict_iterator())) alt_collection = [TargetText(**v) for v in alt_collection] alt_collection = TargetTextCollection(alt_collection) alt_collection.name = 'Another' ax = target_length_plot([TRAIN_COLLECTION, alt_collection], 'targets', whitespace(), cumulative_percentage=True, max_target_length=1) assert alt_ax != ax del alt_ax del ax
def test_train_and_test_dataset(): with tempfile.TemporaryDirectory() as temp_dir: # Test both the normal cahce_dir and the given cache dir for data_dir in [None, Path(temp_dir, 'twitter data')]: train_data = wang_2017_election_twitter_train(data_dir) test_data = wang_2017_election_twitter_test(data_dir) assert len(train_data) > len(test_data) combined_data = TargetTextCollection.combine(train_data, test_data) assert 11899 == combined_data.number_targets()
def test_pos_text(self): # Test the normal case with one TargetText Instance in the collection test_collection = TargetTextCollection([self._target_text_example()]) test_collection.pos_text(spacy_tagger()) pos_answer = [ 'DET', 'NOUN', 'NOUN', 'VERB', 'ADJ', 'CCONJ', 'NOUN', 'VERB', 'ADJ' ] assert test_collection['2']['pos_tags'] == pos_answer # Test the normal case with multiple TargetText Instance in the # collection test_collection = TargetTextCollection(self._target_text_examples()) test_collection.pos_text(spacy_tagger()) assert test_collection['2']['pos_tags'] == pos_answer # Ensure that at least one error is raised but all of these tests are # covered in the TargetText tests. with pytest.raises(TypeError): test_collection.pos_text(str.strip)
def _regular_examples(self) -> List[TargetTextCollection]: target_text_examples = self._target_text_examples() collection_examples = [] for i in range(1, 4): collection = TargetTextCollection() for example in target_text_examples[:i]: collection[example['text_id']] = example collection_examples.append(collection) return collection_examples
def test_target_extraction_fit(self, test_data: bool): model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger') assert model.model is None train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) val_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) tokens_in_vocab = [ 'at', 'case', 'was', 'the', 'day', 'great', 'cover', 'office', 'another', 'and', 'rubbish', 'laptop', '@@PADDING@@', '@@UNKNOWN@@' ] if test_data: tokens_in_vocab = tokens_in_vocab + ['better'] test_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TEST_DATA) model.fit(train_data, val_data, test_data) else: model.fit(train_data, val_data) token_index = model.model.vocab.get_token_to_index_vocabulary('tokens') assert len(token_index) == len(tokens_in_vocab) for token in tokens_in_vocab: assert token in token_index # Check attributes have changed. assert model.model is not None assert isinstance(model.model, Model) # Check that it will save to a directory of our choosing with tempfile.TemporaryDirectory() as save_dir: saved_model_fp = Path(save_dir, 'model.tar.gz') assert not saved_model_fp.exists() model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger', save_dir=save_dir) model.fit(train_data, val_data) assert saved_model_fp.exists()
def no_pred_values(true_sentiment_key: str, predicted_sentiment_key: str ) -> Tuple[TargetTextCollection, List[str], List[List[str]]]: example_1 = TargetText(text_id='1', text='some text') example_1[true_sentiment_key] = [] example_1[predicted_sentiment_key] = [] example_2 = TargetText(text_id='2', text='some text') example_2[true_sentiment_key] = [] example_2[predicted_sentiment_key] = [] true_labels = [] pred_labels = [] return TargetTextCollection([example_1, example_2]), true_labels, pred_labels
def test_load_json(self, name): empty_json_fp = Path(self._json_data_dir(), 'empty_target_instance.json') empty_collection = TargetTextCollection.load_json(empty_json_fp, name=name) assert TargetTextCollection() == empty_collection assert empty_collection.name == name # Ensure that it raises an error when loading a bad json file wrong_json_fp = Path(self._json_data_dir(), 'wrong_target_instance.json') with pytest.raises(ValueError): TargetTextCollection.load_json(wrong_json_fp, name=name) # Ensure that it can load a single target text instance correctly one_target_json_fp = Path(self._json_data_dir(), 'one_target_instance.json') one_target_collection = TargetTextCollection.load_json( one_target_json_fp) assert len(one_target_collection) == 1 assert one_target_collection['0'][ 'text'] == 'The laptop case was great and cover was rubbish' assert one_target_collection['0']['target_sentiments'] == [0] assert one_target_collection['0']['category_sentiments'] == ['pos'] assert one_target_collection['0']['categories'] == ['LAPTOP#CASE'] assert one_target_collection['0']['spans'] == [Span(4, 15)] assert one_target_collection['0']['targets'] == ['laptop case'] # Ensure that it can load multiple target text instances two_target_json_fp = Path(self._json_data_dir(), 'one_target_one_empty_instance.json') two_target_collection = TargetTextCollection.load_json( two_target_json_fp) assert len(two_target_collection) == 2
def test_one_sample_per_span(self, remove_empty: bool): # Case where nothing should change with respect to the number of spans # but will change the values target_sentiments to None etc target_text = TargetText( text_id='0', spans=[Span(4, 15)], text='The laptop case was great and cover was rubbish', target_sentiments=[0], targets=['laptop case']) collection = TargetTextCollection([target_text]) new_collection = collection.one_sample_per_span( remove_empty=remove_empty) assert new_collection == collection assert new_collection['0']['spans'] == [Span(4, 15)] assert new_collection['0']['target_sentiments'] == None assert collection['0']['target_sentiments'] == [0] # Should change the number of Spans. assert target_text['target_sentiments'] == [0] target_text._storage['spans'] = [Span(4, 15), Span(4, 15)] target_text._storage['targets'] = ['laptop case', 'laptop case'] target_text._storage['target_sentiments'] = [0, 1] diff_collection = TargetTextCollection([target_text]) new_collection = diff_collection.one_sample_per_span( remove_empty=remove_empty) assert new_collection == collection assert new_collection['0']['spans'] == [Span(4, 15)] assert new_collection['0']['target_sentiments'] == None assert diff_collection['0']['target_sentiments'] == [0, 1] assert diff_collection['0']['spans'] == [Span(4, 15), Span(4, 15)]
def get_samples_per_sentiment( collection: TargetTextCollection) -> Dict[str, int]: ''' :param collection: The collection being tested :returns: A dictionary where keys are the sentiment value names and the the value is the number of samples that have that sentiment value name e.g. {'pos': 500, 'neg': 400} ''' sentiment_samples = defaultdict(lambda: 0) for value in collection.values(): for sentiment in value['category_sentiments']: sentiment_samples[sentiment] += 1 return sentiment_samples
def passable_diff_num_labels(true_sentiment_key: str, predicted_sentiment_key: str ) -> Tuple[TargetTextCollection, List[str], List[List[str]]]: example_1 = TargetText(text_id='1', text='some text') example_1[true_sentiment_key] = ['pos', 'neg'] example_1[predicted_sentiment_key] = [['neg', 'pos']] example_2 = TargetText(text_id='2', text='some text') example_2[true_sentiment_key] = ['pos', 'neg', 'neu'] example_2[predicted_sentiment_key] = [['neg', 'neg', 'pos']] true_labels = ['pos', 'neg', 'pos', 'neg', 'neu'] pred_labels = [['neg', 'pos', 'neg', 'neg', 'pos']] return TargetTextCollection([example_1, example_2]), true_labels, pred_labels