def split_tests(data_: TargetCollection, train_: TargetCollection, test_: TargetCollection, test_split: float): data_size = len(data_) train_size = len(train_) test_size = len(test_) assert train_size == (data_size - int(data_size * test_split)) assert test_size == int(data_size * test_split) assert data_size == (train_size + test_size) train_ids = [] test_ids = [] for data in train_.data_dict(): train_ids.append(re.findall('\d+', data['target_id'])[0]) for data in test_.data_dict(): test_ids.append(re.findall('\d+', data['target_id'])[0]) assert len(train_ids) == len(set(train_ids)) assert len(test_ids) == len(set(test_ids)) for train_id in train_ids: assert train_id not in test_ids for test_id in test_ids: assert test_id not in train_ids return train_ids, test_ids
def _data_to_json(data: TargetCollection, file_path: Path) -> None: ''' Converts the data into json format and saves it to the given file path. The AllenNLP models read the data from json formatted files. :param data: data to be saved into json format. :param file_path: file location to save the data to. ''' target_data = data.data_dict() with file_path.open('w+') as json_file: for index, data in enumerate(target_data): if 'epoch_number' in data: data['epoch_number'] = list(data['epoch_number']) json_encoded_data = json.dumps(data) if index != 0: json_encoded_data = f'\n{json_encoded_data}' json_file.write(json_encoded_data)
def generate_stats(data_path: Path) -> Dict[str, Union[int, float]]: target_data = [] with data_path.open('r') as data_lines: for line in data_lines: line = json.loads(line) line['spans'] = [tuple(span) for span in line['spans']] target_data.append(Target(**line)) target_data = TargetCollection(target_data) target_stats = defaultdict(lambda: 0) data_size = len(target_data) target_stats['size'] = data_size for i in range(1, 3): target_stats[f'Distinct sentiment {i}'] = len( target_data.subset_by_sentiment(i)) for data in target_data.data_dict(): target_stats[data['sentiment']] += 1 for key, value in target_stats.items(): if key == 'size': continue target_stats[key] = value / data_size return target_stats
def augmented_dataset(target_related_words_sim: Dict[str, List[Tuple[str, float]]], dataset: TargetCollection, save_fp: Path, lower: bool) -> None: ''' Given a dictionary of target words from the training dataset and the values being all of the related words with their similarity score associated to the target key, TDSA training dataset it will for each sample in the training set check if the sample's target exists as a key in the given dictionary and if so write the sample to the save file along with the related targets and similarity scores under the following keys; `alternative_targets` and `alternative_similarity` ''' training_targets_in_embeddings = set(list(target_related_words_sim.keys())) with save_fp.open('w+') as save_file: count = 0 for target_dict in dataset.data_dict(): original_target = target_dict['target'] if lower: original_target = original_target.lower() if original_target in training_targets_in_embeddings: alt_targets_similarity = target_related_words_sim[ original_target] alt_targets_similarity = sorted(alt_targets_similarity, key=lambda x: x[1], reverse=True) different_targets = [ target for target, _ in alt_targets_similarity ] alternative_similarity = [ similarity for _, similarity in alt_targets_similarity ] target_dict['alternative_targets'] = different_targets target_dict['alternative_similarity'] = alternative_similarity target_dict['epoch_number'] = list(target_dict['epoch_number']) json_target_dict = json.dumps(target_dict) if count != 0: json_target_dict = f'\n{json_target_dict}' count += 1 save_file.write(json_target_dict)
def _predict_iter(self, data: TargetCollection ) -> Generator[Dict[str, Any], None, None]: ''' Iterates over the predictions and yields one prediction at a time. This is a useful wrapper as it performs the data pre-processing and assertion checks. :param data: Data to predict on :yields: A dictionary containing `class_probabilities` and `label`. ''' no_model_error = 'There is no model to make predictions, either fit '\ 'or load a model.' assert self.model, no_model_error self.model.eval() all_model_params = Params.from_file(self._param_fp) reader_params = all_model_params.get("dataset_reader") dataset_reader = DatasetReader.from_params(reader_params) predictor = TargetPredictor(self.model, dataset_reader) batch_size = 64 if 'iterator' in all_model_params: iter_params = all_model_params.get("iterator") if 'batch_size' in iter_params: batch_size = iter_params['batch_size'] json_data = data.data_dict() # Reference # https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks for i in range(0, len(json_data), batch_size): json_data_batch = json_data[i:i + batch_size] predictions = predictor.predict_batch_json(json_data_batch) for prediction in predictions: yield prediction