def remap_labels(y_pred, y_true, dataset, evaluation_mode='bio'): ''' y_pred: list of predicted labels y_true: list of gold labels evaluation_mode: 'bio', 'token', or 'binary' Both y_pred and y_true must use label indices and names specified in the dataset # (dataset.unique_label_indices_of_interest, dataset.unique_label_indices_of_interest). ''' all_unique_labels = dataset.unique_labels if evaluation_mode == 'bio': # sort label to index new_label_names = all_unique_labels[:] new_label_names.remove('O') new_label_names.sort( key=lambda x: (hd.remove_bio_from_label_name(x), x)) new_label_names.append('O') new_label_indices = list(range(len(new_label_names))) new_label_to_index = dict(zip(new_label_names, new_label_indices)) remap_index = {} for i, label_name in enumerate(new_label_names): label_index = dataset.label_to_index[label_name] remap_index[label_index] = i else: raise ValueError("At this point only 'bio' is accepted") new_y_pred = [remap_index[label_index] for label_index in y_pred] new_y_true = [remap_index[label_index] for label_index in y_true] new_label_indices_with_o = new_label_indices[:] new_label_names_with_o = new_label_names[:] new_label_names.remove('O') new_label_indices.remove(new_label_to_index['O']) return new_y_pred, new_y_true, new_label_indices, new_label_names, new_label_indices_with_o, new_label_names_with_o
def load_dataset(self,avaliable_datasets_sent,avaliable_datasets_labels, dataset_filepaths, parameters, token_to_vector=None,pretrained_dataset=None): ''' dataset_filepaths : dictionary with keys 'train', 'valid', 'test', 'deploy' ''' start_time = time.time() print('Load dataset... \n') if parameters['token_pretrained_embedding_filepath'] != '': if token_to_vector==None: token_to_vector = hd.load_pretrained_token_embeddings(parameters) else: token_to_vector = {} all_tokens_in_pretraining_dataset = [] all_characters_in_pretraining_dataset = [] if parameters['use_pretrained_model']: #temp_pretrained_dataset_adress="./models/NN_models/1235-4/dataset.pickle" #"./models/NN_models/1234-5/dataset.pickle" if pretrained_dataset==None: temp_pretrained_dataset_adress=parameters['model_folder']+os.sep+"dataset.pickle" pretraining_dataset = pickle.load(open(temp_pretrained_dataset_adress, "rb")) print ("Pre-loading Pre-trained dataset objects") else: pretraining_dataset=pretrained_dataset print ("Pretrained dataset was pre-loaded") all_tokens_in_pretraining_dataset = pretraining_dataset.index_to_token.values() all_characters_in_pretraining_dataset = pretraining_dataset.index_to_character.values() remap_to_unk_count_threshold = 1 self.UNK_TOKEN_INDEX = 0 self.PADDING_CHARACTER_INDEX = 0 self.tokens_mapped_to_unk = [] self.UNK = 'UNK' self.unique_labels = [] labels = {} tokens = {} label_count = {} token_count = {} character_count = {} features={} features_file_names={} feature_vector_size={} #deploy for dataset_type in ['train', 'valid', 'test','deploy']: Not_here=False if dataset_type not in avaliable_datasets_sent: Not_here=True #_parse_dataset(self, dataset_filepath,dataset_type,sentences_list="",tags_list="") if Not_here==False: labels[dataset_type], tokens[dataset_type], token_count[dataset_type], label_count[dataset_type], character_count[dataset_type], features[dataset_type], \ features_file_names[dataset_type],feature_vector_size[dataset_type] \ = self._parse_dataset("", dataset_type, sentences_list=avaliable_datasets_sent[dataset_type], tags_list=avaliable_datasets_labels[dataset_type]) if Not_here==True: labels[dataset_type], tokens[dataset_type], token_count[dataset_type], label_count[dataset_type], character_count[dataset_type], features[dataset_type], \ features_file_names[dataset_type],feature_vector_size[dataset_type] \ = self._parse_dataset("", dataset_type, sentences_list=[], tags_list=[]) # token_count['all'] = {} for token in list(token_count['train'].keys()) + list(token_count['valid'].keys()) + list(token_count['test'].keys()) + list(token_count['deploy'].keys()): token_count['all'][token] = token_count['train'][token] + token_count['valid'][token] + token_count['test'][token] + token_count['deploy'][token] if parameters['load_all_pretrained_token_embeddings']: for token in token_to_vector: if token not in token_count['all']: token_count['all'][token] = -1 token_count['train'][token] = -1 for token in all_tokens_in_pretraining_dataset: if token not in token_count['all']: token_count['all'][token] = -1 token_count['train'][token] = -1 character_count['all'] = {} for character in list(character_count['train'].keys()) + list(character_count['valid'].keys()) + list(character_count['test'].keys()) + list(character_count['deploy'].keys()): character_count['all'][character] = character_count['train'][character] + character_count['valid'][character] + character_count['test'][character] + character_count['deploy'][character] for character in all_characters_in_pretraining_dataset: if character not in character_count['all']: character_count['all'][character] = -1 character_count['train'][character] = -1 label_count['all'] = {} for character in list(label_count['train'].keys()) + list(label_count['valid'].keys()) + list(label_count['test'].keys()) + list(label_count['deploy'].keys()): label_count['all'][character] = label_count['train'][character] + label_count['valid'][character] + label_count['test'][character] + label_count['deploy'][character] token_count['all'] = hd.order_dictionary(token_count['all'], 'value_key', reverse = True) label_count['all'] = hd.order_dictionary(label_count['all'], 'key', reverse = False) character_count['all'] = hd.order_dictionary(character_count['all'], 'value', reverse = True) if self.verbose: print('character_count[\'all\']: {0}'.format(character_count['all'])) token_to_index = {} token_to_index[self.UNK] = self.UNK_TOKEN_INDEX iteration_number = 0 number_of_unknown_tokens = 0 if self.verbose: print("parameters['remap_unknown_tokens_to_unk']: {0}".format(parameters['remap_unknown_tokens_to_unk'])) if self.verbose: print("len(token_count['train'].keys()): {0}".format(len(token_count['train'].keys()))) for token, count in token_count['all'].items(): if iteration_number == self.UNK_TOKEN_INDEX: iteration_number += 1 if parameters['remap_unknown_tokens_to_unk'] == 1 and \ (token_count['train'][token] == 0 or \ parameters['load_only_pretrained_token_embeddings']) and \ not hd.is_token_in_pretrained_embeddings(token, token_to_vector, parameters) and \ token not in all_tokens_in_pretraining_dataset: token_to_index[token] = self.UNK_TOKEN_INDEX number_of_unknown_tokens += 1 self.tokens_mapped_to_unk.append(token) else: token_to_index[token] = iteration_number iteration_number += 1 infrequent_token_indices = [] for token, count in token_count['train'].items(): if 0 < count <= remap_to_unk_count_threshold: infrequent_token_indices.append(token_to_index[token]) #if self.verbose: print("len(token_count['train']): {0}".format(len(token_count['train']))) # if self.verbose: print("len(infrequent_token_indices): {0}".format(len(infrequent_token_indices))) # Ensure that both B- and I- versions exist for each label labels_without_bio = set() for label in label_count['all'].keys(): new_label = hd.remove_bio_from_label_name(label) labels_without_bio.add(new_label) for label in labels_without_bio: if label == 'O': continue if parameters['tagging_format'] == 'bioes': prefixes = ['B-', 'I-', 'E-', 'S-'] else: prefixes = ['B-', 'I-'] for prefix in prefixes: l = prefix + label if l not in label_count['all']: label_count['all'][l] = 0 label_count['all'] = hd.order_dictionary(label_count['all'], 'key', reverse = False) if parameters['use_pretrained_model']: print ("USE_PRETRAINED_MODEL ACTIVE") self.unique_labels = sorted(list(pretraining_dataset.label_to_index.keys())) # Make sure labels are compatible with the pretraining dataset. for label in label_count['all']: if label not in pretraining_dataset.label_to_index: raise AssertionError("The label {0} does not exist in the pretraining dataset. ".format(label) + "Please ensure that only the following labels exist in the dataset: {0}".format(', '.join(self.unique_labels))) label_to_index = pretraining_dataset.label_to_index.copy() else: label_to_index = {} iteration_number = 0 for label, count in label_count['all'].items(): label_to_index[label] = iteration_number iteration_number += 1 self.unique_labels.append(label) character_to_index = {} iteration_number = 0 for character, count in character_count['all'].items(): if iteration_number == self.PADDING_CHARACTER_INDEX: iteration_number += 1 character_to_index[character] = iteration_number iteration_number += 1 token_to_index = hd.order_dictionary(token_to_index, 'value', reverse = False) if self.verbose: print('token_to_index: {0}'.format(token_to_index)) index_to_token = hd.reverse_dictionary(token_to_index) if parameters['remap_unknown_tokens_to_unk'] == 1: index_to_token[self.UNK_TOKEN_INDEX] = self.UNK if self.verbose: print('index_to_token: {0}'.format(index_to_token)) label_to_index = hd.order_dictionary(label_to_index, 'value', reverse = False) index_to_label = hd.reverse_dictionary(label_to_index) character_to_index = hd.order_dictionary(character_to_index, 'value', reverse = False) index_to_character = hd.reverse_dictionary(character_to_index) self.token_to_index = token_to_index self.index_to_token = index_to_token self.index_to_character = index_to_character self.character_to_index = character_to_index self.index_to_label = index_to_label self.label_to_index = label_to_index self.tokens = tokens self.labels = labels dataset_types=['train','test','valid','deploy'] token_indices, label_indices, character_indices_padded, character_indices, token_lengths, characters, label_vector_indices = self._convert_to_indices(dataset_types) self.token_indices = token_indices self.label_indices = label_indices self.character_indices_padded = character_indices_padded self.character_indices = character_indices self.token_lengths = token_lengths self.characters = characters self.label_vector_indices = label_vector_indices self.number_of_classes = max(self.index_to_label.keys()) + 1 self.vocabulary_size = max(self.index_to_token.keys()) + 1 self.alphabet_size = max(self.index_to_character.keys()) + 1 # unique_labels_of_interest is used to compute F1-scores. self.unique_labels_of_interest = list(self.unique_labels) self.unique_labels_of_interest.remove('O') self.unique_label_indices_of_interest = [] for lab in self.unique_labels_of_interest: self.unique_label_indices_of_interest.append(label_to_index[lab]) self.infrequent_token_indices = infrequent_token_indices elapsed_time = time.time() - start_time print('done ({0:.2f} seconds)'.format(elapsed_time)) self.feature_vector_size=0 self._log() return token_to_vector