def test_load_json(self, name): empty_json_fp = Path(self._json_data_dir(), 'empty_target_instance.json') empty_collection = TargetTextCollection.load_json(empty_json_fp, name=name) assert TargetTextCollection() == empty_collection assert empty_collection.name == name # Ensure that it raises an error when loading a bad json file wrong_json_fp = Path(self._json_data_dir(), 'wrong_target_instance.json') with pytest.raises(ValueError): TargetTextCollection.load_json(wrong_json_fp, name=name) # Ensure that it can load a single target text instance correctly one_target_json_fp = Path(self._json_data_dir(), 'one_target_instance.json') one_target_collection = TargetTextCollection.load_json( one_target_json_fp) assert len(one_target_collection) == 1 assert one_target_collection['0'][ 'text'] == 'The laptop case was great and cover was rubbish' assert one_target_collection['0']['target_sentiments'] == [0] assert one_target_collection['0']['category_sentiments'] == ['pos'] assert one_target_collection['0']['categories'] == ['LAPTOP#CASE'] assert one_target_collection['0']['spans'] == [Span(4, 15)] assert one_target_collection['0']['targets'] == ['laptop case'] # Ensure that it can load multiple target text instances two_target_json_fp = Path(self._json_data_dir(), 'one_target_one_empty_instance.json') two_target_collection = TargetTextCollection.load_json( two_target_json_fp) assert len(two_target_collection) == 2
def test_to_json_file(self): test_collection = TargetTextCollection() with tempfile.NamedTemporaryFile(mode='w+') as temp_fp: temp_path = Path(temp_fp.name) test_collection.to_json_file(temp_path) assert len(TargetTextCollection.load_json(temp_path)) == 0 # Ensure that it can load more than one Target Text examples test_collection = TargetTextCollection( self._target_text_examples()) test_collection.to_json_file(temp_path) assert len(TargetTextCollection.load_json(temp_path)) == 3 # Ensure that if it saves to the same file that it overwrites that # file test_collection = TargetTextCollection( self._target_text_examples()) test_collection.to_json_file(temp_path) assert len(TargetTextCollection.load_json(temp_path)) == 3 # Ensure that it can just load one examples test_collection = TargetTextCollection( [self._target_text_example()]) test_collection.to_json_file(temp_path) assert len(TargetTextCollection.load_json(temp_path)) == 1
def run_model(train_fp: Path, val_fp: Path, test_fp: Path, config_fp: Path, number_runs: int, prediction_key: str, save_dir: Optional[Path] = None, only_produce_model: bool = False) -> None: ''' :param train_fp: Path to file that contains JSON formatted training data :param val_fp: Path to file that contains JSON formatted validation data :param test_fp: Path to file that contains JSON formatted testing data :param config_fp: Path to file that contains the models configuration :param number_runs: Number of times to run the model :param prediction_key: The key to save the predictions to within the validation and test data :param save_dir: Path to save the model to. :param only_produce_model: Whether or not to run the model after all the predictions have been made to save a model to the `save_dir` ''' # Test if all the predictions have already been made temp_test_data = TargetTextCollection.load_json(test_fp) temp_test_value = next(temp_test_data.dict_iterator()) predictions_left = number_runs if prediction_key in temp_test_value: number_runs_done = len(temp_test_value[prediction_key]) predictions_left = predictions_left - number_runs_done if number_runs_done >= number_runs and not only_produce_model: print('Predictions have already been made') return train_data = TargetTextCollection.load_json(train_fp) val_data = TargetTextCollection.load_json(val_fp) test_data = TargetTextCollection.load_json(test_fp) key_mappings = {'sentiments': prediction_key} if only_produce_model: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) for run in range(predictions_left): print(f'Run number {run}') if run == 0 and predictions_left == number_runs: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') else: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) model.predict_into_collection(val_data, key_mapping=key_mappings) model.predict_into_collection(test_data, key_mapping=key_mappings) val_data.to_json_file(val_fp, include_metadata=True) test_data.to_json_file(test_fp, include_metadata=True)
def test_predict_into_collection(self, batch_size: Optional[int], append_if_exists: bool): # Test that it raises an Error when the model attribute is not None model_dir = self.TARGET_EXTRACTION_MODEL model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger', model_dir) model.load() # Test the normal case train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) key_mappings = {'tags': 'predicted_tags', 'words': 'predicted_tokens'} train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): assert 'predicted_tags' in target_data assert 'tags' not in target_data assert 'predicted_tokens' in target_data assert 'tokens' not in target_data target_tokens = target_data['tokenized_text'] assert len(target_tokens) == len(target_data['predicted_tags'][0]) assert len(target_tokens) == len( target_data['predicted_tokens'][0]) assert target_tokens == target_data['predicted_tokens'][0] # This should be fine when append_if_exists is True and KeyError other # wise. if append_if_exists: train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): target_tokens = target_data['tokenized_text'] assert 2 == len(target_data['predicted_tags']) assert target_data['predicted_tags'][0] == target_data[ 'predicted_tags'][1] assert target_tokens == target_data['predicted_tokens'][0] assert target_tokens == target_data['predicted_tokens'][1] else: with pytest.raises(KeyError): train_data = model.predict_into_collection( train_data, key_mappings, batch_size, append_if_exists) # Raise a KeyError when the `key_mappings` values are not within the # TargetText from collections import OrderedDict key_mappings = OrderedDict([('tags', 'predicted_tags'), ('wordss', 'predicted_tokens')]) train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) with pytest.raises(KeyError): train_data = model.predict_into_collection(train_data, key_mappings, batch_size, append_if_exists) for target_data in train_data.values(): assert 'predicted_tags' not in target_data assert 'predicted_tokens' not in target_data
def run_model(train_fp: Path, val_fp: Path, test_fp: Path, config_fp: Path, save_test_fp: Path, number_runs: int, model_save_dir: Optional[Path] = None, save_val_fp: Optional[Path] = None) -> None: ''' :param train_fp: Path to file that contains JSON formatted training data :param val_fp: Path to file that contains JSON formatted validation data :param test_fp: Path to file that contains JSON formatted testing data :param config_fp: Path to file that contains the models configuration :param save_test_fp: Path to save the test data results :param number_runs: Number of times to run the model :param model_save_dir: Path to save the first trained model (optional) :param save_val_fp: Path to save the validation data results (optional) ''' train_data = TargetTextCollection.load_json(train_fp) val_data = TargetTextCollection.load_json(val_fp) test_data = TargetTextCollection.load_json(test_fp) test_prediction_data = list(test_data.dict_iterator()) if save_val_fp: val_prediction_data = list(val_data.dict_iterator()) for run in range(number_runs): if run == 0 and model_save_dir: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment', save_dir=model_save_dir) else: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment') model.fit(train_data, val_data, test_data) predict_on(model, test_prediction_data, test_data) if save_val_fp: predict_on(model, val_prediction_data, val_data) test_data.to_json_file(save_test_fp) if save_val_fp: val_data.to_json_file(save_val_fp)
def test_target_extraction_fit(self, test_data: bool): model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger') assert model.model is None train_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) val_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TRAIN_DATA) tokens_in_vocab = [ 'at', 'case', 'was', 'the', 'day', 'great', 'cover', 'office', 'another', 'and', 'rubbish', 'laptop', '@@PADDING@@', '@@UNKNOWN@@' ] if test_data: tokens_in_vocab = tokens_in_vocab + ['better'] test_data = TargetTextCollection.load_json( self.TARGET_EXTRACTION_TEST_DATA) model.fit(train_data, val_data, test_data) else: model.fit(train_data, val_data) token_index = model.model.vocab.get_token_to_index_vocabulary('tokens') assert len(token_index) == len(tokens_in_vocab) for token in tokens_in_vocab: assert token in token_index # Check attributes have changed. assert model.model is not None assert isinstance(model.model, Model) # Check that it will save to a directory of our choosing with tempfile.TemporaryDirectory() as save_dir: saved_model_fp = Path(save_dir, 'model.tar.gz') assert not saved_model_fp.exists() model = AllenNLPModel('TE', self.CONFIG_FILE, 'target-tagger', save_dir=save_dir) model.fit(train_data, val_data) assert saved_model_fp.exists()
prediction_key_rename = { 'predicted_target_sentiment_average_GloVe': 'predicted_target_sentiment_CNN_GloVe_None_None', 'predicted_target_sentiment_average_CWR': 'predicted_target_sentiment_CNN_CWR_None_None' } for dataset_name in dataset_names: new_dataset_result_folder = Path(new_data_dir, f'{dataset_name}_dataset') existing_dataset_result_folder = Path(existing_data_dir, f'{dataset_name}_dataset') for split_name in split_names: new_split_data = Path(new_dataset_result_folder, f'{split_name}.json') new_target_collection = TargetTextCollection.load_json( new_split_data) exist_split_data = Path(existing_dataset_result_folder, f'{split_name}.json') exist_target_collection = TargetTextCollection.load_json( exist_split_data) id_results = defaultdict(dict) for text_id, target_text in new_target_collection.items(): for prediction_key in prediction_keys_to_copy: cnn_result = target_text[prediction_key] id_results[text_id][prediction_key] = cnn_result len_err = ('The number of keys that the merge predictions are' f' associated with {len(id_results)} are not the same ' 'number of keys as the existing predictions dataset ' f'contains {len(exist_target_collection)}.')
type=str, help='Metric to be used on the error subsets') args = parser.parse_args() results_dir = args.results_dir save_fp = args.save_fp subset_metric_func = getattr(sentiment_metrics, args.subset_metric) # Get the data data_splits = ['test', 'val'] dataset_names = ['election', 'laptop', 'restaurant'] index_keys = ['prediction key', 'run number'] all_dfs: List[pd.DataFrame] = [] training_datasets = {} for dataset_name in dataset_names: data_fp = Path(results_dir, f'{dataset_name}_dataset', 'train.json') training_datasets[dataset_name] = TargetTextCollection.load_json( data_fp) formatted_data_split = {'test': 'Test', 'val': 'Validation'} import time overall_time = time.time() for data_split in data_splits: print(f'Data Split {data_split}') for dataset_name in dataset_names: one_time = time.time() print(f'Dataset {dataset_name}') data_fp = Path(results_dir, f'{dataset_name}_dataset', f'{data_split}.json') test_collection = TargetTextCollection.load_json(data_fp) metric_df = overall_metric_results( test_collection,
nt_error_names = ERROR_SPLIT_SUBSET_NAMES['NT'] ds_error_names = ERROR_SPLIT_SUBSET_NAMES['DS'] tssr_error_names = ERROR_SPLIT_SUBSET_NAMES['TSSR'] reduced_collection_subset_names = ds_error_names + tssr_error_names nt_split_subsets = {'NT': ERROR_SPLIT_SUBSET_NAMES['NT']} import time overall_time = time.time() for dataset_name in dataset_names: print(f'Dataset {dataset_name}') for split in splits: one_time = time.time() print(f'Data Split {split}') data_fp = Path(results_dir, f'{dataset_name.lower()}_dataset', f'{split}.json') dataset = TargetTextCollection.load_json(data_fp) for error_func in all_relevant_error_funcs: error_func(None, dataset, True) for reduced_collection_subset_name in reduced_collection_subset_names: temp_df = error_split_df( None, dataset, relevant_prediction_keys, 'target_sentiments', nt_split_subsets, accuracy, {'ignore_label_differences': True}, include_dataset_size=True, collection_subsetting=[[reduced_collection_subset_name]], table_format_return=False) temp_df = add_metadata_to_df(temp_df, dataset, 'predicted_target_sentiment_key')
print('Not overwriting the data') # Process the training, validation, and test data split_names = ['train', 'val', 'train_val', 'test'] split_name_dataset = [] train_size = 0 val_size = 0 train_text_sentiment_distribution = Counter() val_text_sentiment_distribution = Counter() train_target_sentiment_distribution = Counter() val_target_sentiment_distribution = Counter() for split_name in split_names: data_fp = Path(args.dataset_dir, f'{split_name}.json') if split_name == 'train_val': data_fp = Path(args.dataset_dir, f'val.json') dataset = TargetTextCollection.load_json(data_fp) if split_name == 'train': train_target_sentiment_distribution = get_target_sentiment_distribution(dataset) if split_name == 'train_val': val_target_sentiment_distribution = get_target_sentiment_distribution(dataset) if split_name == 'train' and average_sentiment: dataset.one_sentiment_text('target_sentiments', average_sentiment) elif split_name == 'train' and not average_sentiment: dataset = remove_multi_sentiment_targets(dataset) if split_name == 'train_val' and average_sentiment: dataset.one_sentiment_text('target_sentiments', average_sentiment) elif split_name == 'train_val' and not average_sentiment: dataset = remove_multi_sentiment_targets(dataset)
'election': 'Election', 'laptop': 'Laptop', 'restaurant': 'Restaurant' } split_name_mapper = {'train': 'Train', 'val': 'Validation', 'test': 'Test'} all_dataset_names = [] all_split_names = [] targets_in_split = [] for dataset_name in dataset_names: dataset_dir = data_dir / f'{dataset_name}_dataset' total_targets_datasets = 0 for split_name in split_names: data_fp = dataset_dir / f'{split_name}.json' total_targets_datasets += TargetTextCollection.load_json( data_fp).number_targets() for split_name in split_names: data_fp = dataset_dir / f'{split_name}.json' num_targets = TargetTextCollection.load_json(data_fp).number_targets() num_targets = f'{num_targets} ({(num_targets/total_targets_datasets)*100:.2f}%)' targets_in_split.append(num_targets) all_split_names.append(split_name_mapper[split_name]) all_dataset_names.append(data_name_mapper[dataset_name]) targets_in_split.append(total_targets_datasets) all_split_names.append('Total') all_dataset_names.append(data_name_mapper[dataset_name]) import pandas as pd stats_df = pd.DataFrame({ 'Dataset': all_dataset_names, 'Data Split': all_split_names, 'values': targets_in_split
path_string = Path(path_string).resolve() return path_string if __name__ == '__main__': save_dir_help = ('File Path to directory where the anonymised results ' 'will be saved.') results_dir_help = ('File path to the directory that currently stores all ' 'results') parser = argparse.ArgumentParser() parser.add_argument("results_dir", type=parse_path, help=results_dir_help) parser.add_argument("save_dir", type=parse_path, help=save_dir_help) args = parser.parse_args() save_dir = args.save_dir results_dir = args.results_dir save_dir.mkdir(parents=True, exist_ok=True) dataset_names = ['election', 'laptop', 'restaurant'] split_names = ['train', 'val', 'test'] for dataset_name in dataset_names: dataset_result_folder = Path(results_dir, f'{dataset_name}_dataset') save_dataset_folder = Path(save_dir, f'{dataset_name}_dataset') save_dataset_folder.mkdir(parents=True, exist_ok=True) for split_name in split_names: split_fp = Path(dataset_result_folder, f'{split_name}.json') split_dataset = TargetTextCollection.load_json(split_fp) split_dataset: TargetTextCollection split_dataset.anonymised = True save_fp = Path(save_dataset_folder, f'{split_name}.json') split_dataset.to_json_file(save_fp, include_metadata=True)
import pandas as pd from target_extraction.data_types import TargetTextCollection, TargetText from target_extraction.data_types_util import Span from target_extraction.analysis.dataset_statistics import get_sentiment_counts from target_extraction.analysis.dataset_statistics import average_target_per_sentences from target_extraction.analysis.dataset_statistics import dataset_target_sentiment_statistics from target_extraction.analysis.dataset_statistics import tokens_per_target from target_extraction.analysis.dataset_statistics import dataset_target_extraction_statistics from target_extraction.analysis.dataset_statistics import _statistics_to_dataframe from target_extraction.analysis.dataset_statistics import tokens_per_sentence from target_extraction.tokenizers import whitespace DATA_DIR = Path(__file__, '..', '..', 'data', 'analysis', 'sentiment_error_analysis').resolve() TRAIN_COLLECTION = TargetTextCollection.load_json( Path(DATA_DIR, 'train_with_blank.json')) TRAIN_COLLECTION.name = 'train' SENTIMENT_KEY = 'target_sentiments' def test_get_sentiment_counts(): num_pos = 2 num_neu = 12 num_neg = 5 total = 19.0 true_sentiment_counts = dict([('positive', num_pos), ('neutral', num_neu), ('negative', num_neg)]) sentiment_counts = get_sentiment_counts(TRAIN_COLLECTION, normalised=False, sentiment_key=SENTIMENT_KEY) assert len(true_sentiment_counts) == len(sentiment_counts)
from pathlib import Path from target_extraction.data_types import TargetTextCollection def parse_path(path_string: str) -> Path: path_string = Path(path_string).resolve() return path_string if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("data_dir", type=parse_path) args = parser.parse_args() for split_name in ['train.json', 'val.json', 'test.json']: split_fp = Path(args.data_dir, split_name) collection = TargetTextCollection.load_json(split_fp) for value in collection.values(): try: value.re_order(keys_not_to_order=[ 'pos_tags', 'tokenized_text', 'category_sentiments', 'categories' ]) except: print(value) collection.re_order(keys_not_to_order=[ 'pos_tags', 'tokenized_text', 'category_sentiments', 'categories' ]) assert collection.in_order() collection.to_json_file(split_fp, include_metadata=True)
all_targets.append(aug_target_object) except OverLappingTargetsError: # This needs to be skipped as when targets overlap it is very # difficult to easily calculate all possible span offsets # for all other targets. Furthermore there are only 3 # occasion this happens so it is a very rare occurrence. continue return all_targets if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("augmented_dataset", type=parse_path, help='File path the augmented dataset') parser.add_argument("save_fp", type=parse_path, help='File path to save the new re-formated augmented dataset') args = parser.parse_args() augmented_data_fp = args.augmented_dataset save_fp = args.save_fp augmented_dataset = TargetTextCollection.load_json(augmented_data_fp) new_dataset = [] for target_object in augmented_dataset.values(): augmented_targets = add_augmented_targets(target_object, remove_repeats=True) new_dataset.extend(augmented_targets) new_dataset = TargetTextCollection(new_dataset) number_samples = new_dataset.number_targets() print(f'The number of samples in the dataset {number_samples}') new_dataset.to_json_file(save_fp)
config_params.to_file(temp_file.name) only_produce_model = True if args.run_to_get_saved_model else False run_model(train_fp, train_val_fp, val_fp, test_fp, Path(temp_file.name), args.N, prediction_key, save_dir=model_dir_save_dir, only_produce_model=only_produce_model) # Add metadata and ensure that the datasets have there correct name # associated to them for split_name, data_fp in [('Validation', val_fp), ('Test', test_fp)]: data = TargetTextCollection.load_json(data_fp, name=args.domain) metadata = data.metadata if data.metadata else {} metadata['split'] = split_name sentiment_key_metadata = {} sentiment_key_metadata_key = 'predicted_target_sentiment_key' if 'predicted_target_sentiment_key' in metadata: sentiment_key_metadata = metadata['predicted_target_sentiment_key'] key_metadata = {} key_metadata['CWR'] = True if args.cwr else False key_metadata['Position'] = True if args.position else False key_metadata[ 'Inter-Aspect'] = args.inter_aspect if args.inter_aspect else False key_metadata['Model'] = args.model_name key_metadata['data-trained-on'] = data_trained_on_name sentiment_key_metadata[prediction_key] = key_metadata metadata['predicted_target_sentiment_key'] = sentiment_key_metadata
help='File path JSON training data') parser.add_argument("glove_embedding_fp", type=parse_path, help=glove_fp_help) parser.add_argument("target_only_word2vec_path", type=parse_path, help='File path to save the embedding too.') args = parser.parse_args() save_fp = args.target_only_word2vec_path if save_fp.exists(): print('A file already exists at the location to store ' f'the new Word2Vec model/vector: {save_fp}\n' 'Thus skipping the rest of this script.') else: dataset = TargetTextCollection.load_json(args.json_train_data) all_targets = list(dataset.target_count(lower=True).keys()) tokenizer = spacy_tokenizer() tokenised_targets = [ target for targets in all_targets for target in tokenizer(targets) ] with tempfile.TemporaryDirectory() as temp_dir: shrink_glove_temp_fp = Path(temp_dir, 'temp_glove') shrink_word_vec_temp_fp = Path(temp_dir, 'temp_wordvec') shrink_glove_file(args.glove_embedding_fp, tokenised_targets, shrink_glove_temp_fp) glove2word2vec(shrink_glove_temp_fp, shrink_word_vec_temp_fp) model = KeyedVectors.load_word2vec_format(shrink_word_vec_temp_fp) model.save(str(save_fp)) print(f'Word2Vec shrunk to target model saved to {save_fp}')
def run_model(train_fp: Path, train_val_fp: Path, val_fp: Path, test_fp: Path, config_fp: Path, number_runs: int, prediction_key: str, save_dir: Optional[Path] = None, only_produce_model: bool = False) -> None: ''' :param train_fp: Path to file that contains JSON formatted training data :param train_val_fp: Path to file that contains JSON formatted validation data that will be used to train the model (used for early stopping). :param val_fp: Path to file that contains JSON formatted validation data that will be predicted on and used to evaluate the text classification model on the target sentiment task :param test_fp: Path to file that contains JSON formatted testing data for target sentiment evaluation. :param config_fp: Path to file that contains the models configuration :param number_runs: Number of times to run the model :param prediction_key: The key to save the predictions to within the validation and test data :param save_dir: Path to save the model to. :param only_produce_model: Whether or not to run the model after all the predictions have been made to save a model to the `save_dir` ''' # Test if all the predictions have already been made temp_test_data = TargetTextCollection.load_json(test_fp) temp_test_value = next(temp_test_data.dict_iterator()) predictions_left = number_runs if prediction_key in temp_test_value: number_runs_done = len(temp_test_value[prediction_key]) predictions_left = predictions_left - number_runs_done if number_runs_done >= number_runs and not only_produce_model: print('Predictions have already been made') return train_data = TargetTextCollection.load_json(train_fp) train_val_data = TargetTextCollection.load_json(train_val_fp) val_data = TargetTextCollection.load_json(val_fp) test_data = TargetTextCollection.load_json(test_fp) if only_produce_model: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') model.fit(train_data, train_val_data, test_data) for run in range(predictions_left): print(f'Run number {run}') if run == 0 and predictions_left == number_runs: model = AllenNLPModel('model', config_fp, save_dir=save_dir, predictor_name='target-sentiment') else: model = AllenNLPModel('model', config_fp, predictor_name='target-sentiment') model.fit(train_data, train_val_data, test_data) text_classification_prediction(model, val_data, prediction_key) text_classification_prediction(model, test_data, prediction_key) #for value in model._predict_iter(val_data.dict_iterator(), yield_original_target=True): # prediction_object, target_object = value # predicted_sentiment = prediction_object['label'] # true_sentiment = target_object['target_sentiments'] # number_sentiments = len(true_sentiment) # predicted_sentiment = [predicted_sentiment] * number_sentiments # text_id = target_object['text_id'] # if prediction_key not in val_data[text_id]: # val_data[text_id][prediction_key] = [] # val_data[text_id][prediction_key].append(predicted_sentiment) #model.predict_into_collection(val_data, key_mapping=key_mappings) #model.predict_into_collection(test_data, key_mapping=key_mappings) val_data.to_json_file(val_fp, include_metadata=True) test_data.to_json_file(test_fp, include_metadata=True)
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( "train_augmented_fp", type=parse_path, help='File path to the training only targets augmented dataset') parser.add_argument( "predicted_augmented_fp", type=parse_path, help='File path to the predicted only targets augmented dataset') parser.add_argument("save_fp", type=parse_path, help='File path to save the new combined dataset') args = parser.parse_args() augmented_train_dataset = TargetTextCollection.load_json( args.train_augmented_fp) augmented_pred_dataset = TargetTextCollection.load_json( args.predicted_augmented_fp) save_fp = args.save_fp assert len(augmented_train_dataset) == len(augmented_pred_dataset) assert augmented_train_dataset == augmented_pred_dataset assert augmented_pred_dataset == augmented_train_dataset combined_target_objects = [] for text_id, train_target_text in augmented_train_dataset.items(): pred_target_text = augmented_pred_dataset[text_id] train_targets = train_target_text['targets'] pred_targets = pred_target_text['targets'] target_not_same_err = ('The target lists have to be the same within '
"target based on the model's confidence scores" save_fp_help = 'File to save the predicted targets to, each target will be '\ 'saved on a new line' parser = argparse.ArgumentParser() parser.add_argument("predicted_target_data_fp", type=parse_path, help='File path to the predicted targets') parser.add_argument("train_fp", type=parse_path, help=train_fp_help) parser.add_argument('confidence_score', type=float, help=confidence_score_help) parser.add_argument('save_fp', type=parse_path, help=save_fp_help) args = parser.parse_args() # Setting the data up train_data = TargetTextCollection.load_json(args.train_fp) train_targets = set(list(train_data.target_count(lower=True).keys())) acceptable_confidence = args.confidence_score all_targets: List[TargetText] = [] with args.predicted_target_data_fp.open('r') as predicted_file: for index, line in enumerate(predicted_file): target_data = json.loads(line) target_id = str(index) target_data_dict = { 'text': target_data['text'], 'text_id': target_id, 'confidences': target_data['confidence'], 'sequence_labels': target_data['sequence_labels'], 'tokenized_text': target_data['tokens'] }
# save directory encoder = encoder_name(args.elmo, args.elmo_ds, args.word_embedding, args.word_embedding_ds, args.elmo_contextualised) print(encoder) if 'use_target_sequences' in config_params['model']: print(config_params['model']['use_target_sequences']) save_dir = Path(args.save_dir, args.model_name, args.domain, encoder).resolve() save_dir.mkdir(parents=True, exist_ok=True) test_save_fp = Path(save_dir, 'pred_test.json') val_save_fp = Path(save_dir, 'pred_val.json') dataset_dir = args.dataset_dir train_fp = Path(dataset_dir, 'train.json') if test_save_fp.exists() and val_save_fp.exists(): save_test_data = TargetTextCollection.load_json(test_save_fp) test_value = next(iter(save_test_data.values())) num_predictions = len(test_value['predicted_target_sentiments']) if num_predictions >= args.N: print( f'Predictions have already been made at the following directory {save_dir}' ) else: predictions_left = args.N - num_predictions print(f'Number of predictions left {predictions_left}') with tempfile.NamedTemporaryFile(mode='w+') as temp_file: config_params.to_file(temp_file.name) run_model(train_fp, val_save_fp, test_save_fp, Path(temp_file.name), test_save_fp, predictions_left, None, val_save_fp)