def __token_embedding_dict(self, pretrained_vocabulary: PretrainedVocabulary): token_embedding_dict = dict() unk_index = pretrained_vocabulary.index(pretrained_vocabulary.unk) for index in range(pretrained_vocabulary.embedding_matrix.size(0)): if index in {pretrained_vocabulary.padding_index, unk_index}: # 对 padding_index 和 unk_index 来说,需要略过,因为是在后面合并后的 vocabulary 中,会重新填充 continue token = pretrained_vocabulary.token(index) embedding = pretrained_vocabulary.embedding_matrix[index, :] token_embedding_dict[token] = embedding return token_embedding_dict
def test_save_and_load(pretrained_vocabulary): """ 测试 保存和载入 """ saved_dir = os.path.join(ROOT_PATH, "data/easytext/tests/pretrained_vocabulary") pretrained_vocabulary.save_to_file(saved_dir) loaded_vocab = PretrainedVocabulary.from_file(saved_dir) test_pretrained_vocabulary(loaded_vocab)
def __init__( self, is_training: bool, dataset: Dataset, gaz_vocabulary_dir: str, gaz_pretrained_word_embedding_loader: PretrainedWordEmbeddingLoader ): """ 构建 Gaz 词汇表 :param is_training: 当前是否 Training 状态 :param dataset: 数据集 :param gaz_vocabulary_dir: gaz 词汇表存放目录 :param gaz_pretrained_word_embedding_loader: gaz 预训练 word embedding 载入器 """ super().__init__(is_training=is_training) # gazetter 理论上来说,应该支持持久化的,这里并没有做 gazetteer = Gazetteer(gaz_pretrained_word_embedding_loader= gaz_pretrained_word_embedding_loader) if is_training: gaz_vocabulary_collate = GazVocabularyCollate(gazetteer=gazetteer) data_loader = DataLoader(dataset=dataset, batch_size=100, shuffle=False, num_workers=0, collate_fn=gaz_vocabulary_collate) gaz_words = list() for batch_gaz_words in data_loader: gaz_words.extend(batch_gaz_words) gaz_vocabulary = Vocabulary(tokens=gaz_words, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) gaz_vocabulary = PretrainedVocabulary( vocabulary=gaz_vocabulary, pretrained_word_embedding_loader= gaz_pretrained_word_embedding_loader) gaz_vocabulary.save_to_file(gaz_vocabulary_dir) else: gaz_vocabulary = Vocabulary.from_file(gaz_vocabulary_dir) self.gaz_vocabulary = gaz_vocabulary self.gazetteer = gazetteer
def build_vocabulary(self): training_dataset_file_path = self.config["training_dataset_file_path"] dataset = ACSASemEvalDataset( dataset_file_path=training_dataset_file_path) collate_fn = VocabularyCollate() data_loader = DataLoader(dataset=dataset, batch_size=10, shuffle=False, num_workers=0, collate_fn=collate_fn) tokens = list() categories = list() labels = list() for collate_dict in data_loader: tokens.append(collate_dict["tokens"]) categories.append(collate_dict["categories"]) labels.append(collate_dict["labels"]) token_vocabulary = Vocabulary(tokens=tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) if not self.config["debug"]: pretrained_file_path = self.config["pretrained_file_path"] pretrained_loader = GloveLoader( embedding_dim=300, pretrained_file_path=pretrained_file_path) pretrained_token_vocabulary = PretrainedVocabulary( vocabulary=token_vocabulary, pretrained_word_embedding_loader=pretrained_loader) token_vocabulary = pretrained_token_vocabulary category_vocabulary = LabelVocabulary(labels=categories, padding=None) label_vocabulary = LabelVocabulary(labels=labels, padding=None) return { "token_vocabulary": token_vocabulary, "category_vocabulary": category_vocabulary, "label_vocabulary": label_vocabulary }
def pretrained_vocabulary(): """ 生成 预训练词汇表 """ pretrained_file_path = "data/easytext/tests/pretrained/word_embedding_sample.3d.txt" pretrained_file_path = os.path.join(ROOT_PATH, pretrained_file_path) glove_loader = GloveLoader(embedding_dim=3, pretrained_file_path=pretrained_file_path) tokens = [["我"], ["美丽"]] vocab = Vocabulary(tokens=tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) pretrained_vocab = PretrainedVocabulary( vocabulary=vocab, pretrained_word_embedding_loader=glove_loader) return pretrained_vocab
def build_vocabulary(self, dataset: Dataset): data_loader = DataLoader(dataset=dataset, batch_size=100, shuffle=False, num_workers=0, collate_fn=VocabularyCollate()) batch_tokens = list() batch_sequence_labels = list() for collate_dict in data_loader: batch_tokens.extend(collate_dict["tokens"]) batch_sequence_labels.extend(collate_dict["sequence_labels"]) token_vocabulary = Vocabulary(tokens=batch_tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) model_name = self.config["model_name"] if model_name in {ModelName.NER_V2, ModelName.NER_V3}: pretrained_word_embedding_file_path = self.config[ "pretrained_word_embedding_file_path"] glove_loader = GloveLoader( embedding_dim=100, pretrained_file_path=pretrained_word_embedding_file_path) token_vocabulary = PretrainedVocabulary( vocabulary=token_vocabulary, pretrained_word_embedding_loader=glove_loader) label_vocabulary = LabelVocabulary(labels=batch_sequence_labels, padding=LabelVocabulary.PADDING) return { "token_vocabulary": token_vocabulary, "label_vocabulary": label_vocabulary }
def __init__(self, is_training: bool, dataset: Dataset, vocabulary_collate, token_vocabulary_dir: str, label_vocabulary_dir: str, is_build_token_vocabulary: bool, pretrained_word_embedding_loader: PretrainedWordEmbeddingLoader): """ 词汇表构建器 :param is_training: 因为在 train 和 非 train, 词汇表的构建行为有所不同; 如果是 train, 则一般需要重新构建; 而对于 非train, 使用先前构建好的即可。 :param dataset: 数据集 :param vocabulary_collate: 词汇表 collate :param token_vocabulary_dir: token vocabulary 存放目录 :param label_vocabulary_dir: label vocabulary 存放目录 :param is_build_token_vocabulary: 是否构建 token vocabulary, 因为在使用 Bert 或者 其他模型作为预训练的 embedding, 则没有必要构建 token vocabulary. :param pretrained_word_embedding_loader: 预训练词汇表 """ super().__init__(is_training=is_training) token_vocabulary = None label_vocabulary = None if is_training: data_loader = DataLoader(dataset=dataset, batch_size=100, shuffle=False, num_workers=0, collate_fn=vocabulary_collate) batch_tokens = list() batch_sequence_labels = list() for collate_dict in data_loader: batch_tokens.extend(collate_dict["tokens"]) batch_sequence_labels.extend(collate_dict["sequence_labels"]) if is_build_token_vocabulary: token_vocabulary = Vocabulary(tokens=batch_tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) if pretrained_word_embedding_loader is not None: token_vocabulary = \ PretrainedVocabulary(vocabulary=token_vocabulary, pretrained_word_embedding_loader=pretrained_word_embedding_loader) if token_vocabulary_dir: token_vocabulary.save_to_file(token_vocabulary_dir) label_vocabulary = LabelVocabulary(labels=batch_sequence_labels, padding=LabelVocabulary.PADDING) if label_vocabulary_dir: label_vocabulary.save_to_file(label_vocabulary_dir) else: if is_build_token_vocabulary and token_vocabulary_dir: token_vocabulary = Vocabulary.from_file(token_vocabulary_dir) if label_vocabulary_dir: label_vocabulary = LabelVocabulary.from_file(label_vocabulary_dir) self.token_vocabulary = token_vocabulary self.label_vocabulary = label_vocabulary
def test_flat_model_collate(lattice_ner_demo_dataset, character_pretrained_embedding_loader, gaz_pretrained_embedding_loader): """ 测试 flat model collate :return: """ # 仅仅取前两个作为测试 batch_instances = lattice_ner_demo_dataset[0:2] vocabulary_collate = VocabularyCollate() collate_result = vocabulary_collate(batch_instances) characters = collate_result["tokens"] sequence_label = collate_result["sequence_labels"] character_vocabulary = Vocabulary(tokens=characters, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) character_vocabulary = PretrainedVocabulary( vocabulary=character_vocabulary, pretrained_word_embedding_loader=character_pretrained_embedding_loader) label_vocabulary = LabelVocabulary(labels=sequence_label, padding=LabelVocabulary.PADDING) gazetter = Gazetteer( gaz_pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) gaz_vocabulary_collate = GazVocabularyCollate(gazetteer=gazetter) gaz_words = gaz_vocabulary_collate(batch_instances) gaz_vocabulary = Vocabulary(tokens=gaz_words, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) gaz_vocabulary = PretrainedVocabulary( vocabulary=gaz_vocabulary, pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) flat_vocabulary = FlatPretrainedVocabulary( character_pretrained_vocabulary=character_vocabulary, gaz_word_pretrained_vocabulary=gaz_vocabulary) flat_model_collate = FLATModelCollate(token_vocabulary=flat_vocabulary, gazetter=gazetter, label_vocabulary=label_vocabulary) model_inputs = flat_model_collate(batch_instances) logging.debug(json2str(model_inputs.model_inputs["metadata"])) metadata_0 = model_inputs.model_inputs["metadata"][0] sentence = "陈元呼吁加强国际合作推动世界经济发展" # 陈元呼吁加强国际合作推动世界经济发展 expect_squeeze_gaz_words_0 = [ "陈元", "呼吁", "吁加", "加强", "强国", "国际", "合作", "推动", "世界", "经济", "发展" ] squeeze_gaz_words_0 = metadata_0["squeeze_gaz_words"] ASSERT.assertListEqual(expect_squeeze_gaz_words_0, squeeze_gaz_words_0) expect_tokens = [character for character in sentence] + expect_squeeze_gaz_words_0 tokens = metadata_0["tokens"] ASSERT.assertListEqual(expect_tokens, tokens) character_pos_begin = [index for index in range(len(sentence))] character_pos_end = [index for index in range(len(sentence))] squeeze_gaz_words_begin = list() squeeze_gaz_words_end = list() for squeeze_gaz_word in squeeze_gaz_words_0: index = sentence.find(squeeze_gaz_word) squeeze_gaz_words_begin.append(index) squeeze_gaz_words_end.append(index + len(squeeze_gaz_word) - 1) pos_begin = model_inputs.model_inputs["pos_begin"][0] pos_end = model_inputs.model_inputs["pos_end"][0] expect_pos_begin = character_pos_begin + squeeze_gaz_words_begin expect_pos_begin += [0] * (pos_begin.size(0) - len(expect_pos_begin)) expect_pos_begin = torch.tensor(expect_pos_begin) expect_pos_end = character_pos_end + squeeze_gaz_words_end expect_pos_end += [0] * (pos_end.size(0) - len(expect_pos_end)) expect_pos_end = torch.tensor(expect_pos_end) ASSERT.assertTrue(tensor_util.is_tensor_equal(expect_pos_begin, pos_begin)) ASSERT.assertTrue(tensor_util.is_tensor_equal(expect_pos_end, pos_end)) expect_character_length = len(sentence) expect_squeeze_gaz_word_length = len(expect_squeeze_gaz_words_0) character_length = model_inputs.model_inputs["sequence_length"][0] squeeze_word_length = model_inputs.model_inputs["squeeze_gaz_word_length"][ 0] ASSERT.assertEqual(expect_character_length, character_length.item()) ASSERT.assertEqual(expect_squeeze_gaz_word_length, squeeze_word_length.item())
def __call__(self, config: Dict, train_type: int): serialize_dir = config["serialize_dir"] vocabulary_dir = config["vocabulary_dir"] pretrained_embedding_file_path = config["pretrained_embedding_file_path"] word_embedding_dim = config["word_embedding_dim"] pretrained_embedding_max_size = config["pretrained_embedding_max_size"] is_fine_tuning = config["fine_tuning"] word_vocab_dir = os.path.join(vocabulary_dir, "vocabulary", "word_vocabulary") event_type_vocab_dir = os.path.join(vocabulary_dir, "vocabulary", "event_type_vocabulary") entity_tag_vocab_dir = os.path.join(vocabulary_dir, "vocabulary", "entity_tag_vocabulary") if train_type == Train.NEW_TRAIN: if os.path.isdir(serialize_dir): shutil.rmtree(serialize_dir) os.makedirs(serialize_dir) if os.path.isdir(vocabulary_dir): shutil.rmtree(vocabulary_dir) os.makedirs(vocabulary_dir) os.makedirs(word_vocab_dir) os.makedirs(event_type_vocab_dir) os.makedirs(entity_tag_vocab_dir) elif train_type == Train.RECOVERY_TRAIN: pass else: assert False, f"train_type: {train_type} error!" train_dataset_file_path = config["train_dataset_file_path"] validation_dataset_file_path = config["validation_dataset_file_path"] num_epoch = config["epoch"] batch_size = config["batch_size"] if train_type == Train.NEW_TRAIN: # 构建词汇表 ace_dataset = ACEDataset(train_dataset_file_path) vocab_data_loader = DataLoader(dataset=ace_dataset, batch_size=10, shuffle=False, num_workers=0, collate_fn=EventVocabularyCollate()) tokens: List[List[str]] = list() event_types: List[List[str]] = list() entity_tags: List[List[str]] = list() for colleta_dict in vocab_data_loader: tokens.extend(colleta_dict["tokens"]) event_types.extend(colleta_dict["event_types"]) entity_tags.extend(colleta_dict["entity_tags"]) word_vocabulary = Vocabulary(tokens=tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) glove_loader = GloveLoader(embedding_dim=word_embedding_dim, pretrained_file_path=pretrained_embedding_file_path, max_size=pretrained_embedding_max_size) pretrained_word_vocabulary = PretrainedVocabulary(vocabulary=word_vocabulary, pretrained_word_embedding_loader=glove_loader) pretrained_word_vocabulary.save_to_file(word_vocab_dir) event_type_vocabulary = Vocabulary(tokens=event_types, padding="", unk="Negative", special_first=True) event_type_vocabulary.save_to_file(event_type_vocab_dir) entity_tag_vocabulary = LabelVocabulary(labels=entity_tags, padding=LabelVocabulary.PADDING) entity_tag_vocabulary.save_to_file(entity_tag_vocab_dir) else: pretrained_word_vocabulary = PretrainedVocabulary.from_file(word_vocab_dir) event_type_vocabulary = Vocabulary.from_file(event_type_vocab_dir) entity_tag_vocabulary = Vocabulary.from_file(entity_tag_vocab_dir) model = EventModel(alpha=0.5, activate_score=True, sentence_vocab=pretrained_word_vocabulary, sentence_embedding_dim=word_embedding_dim, entity_tag_vocab=entity_tag_vocabulary, entity_tag_embedding_dim=50, event_type_vocab=event_type_vocabulary, event_type_embedding_dim=300, lstm_hidden_size=300, lstm_encoder_num_layer=1, lstm_encoder_droupout=0.4) trainer = Trainer( serialize_dir=serialize_dir, num_epoch=num_epoch, model=model, loss=EventLoss(), optimizer_factory=EventOptimizerFactory(is_fine_tuning=is_fine_tuning), metrics=EventF1MetricAdapter(event_type_vocabulary=event_type_vocabulary), patient=10, num_check_point_keep=5, devices=None ) train_dataset = EventDataset(dataset_file_path=train_dataset_file_path, event_type_vocabulary=event_type_vocabulary) validation_dataset = EventDataset(dataset_file_path=validation_dataset_file_path, event_type_vocabulary=event_type_vocabulary) event_collate = EventCollate(word_vocabulary=pretrained_word_vocabulary, event_type_vocabulary=event_type_vocabulary, entity_tag_vocabulary=entity_tag_vocabulary, sentence_max_len=512) train_data_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=0, collate_fn=event_collate) validation_data_loader = DataLoader(dataset=validation_dataset, batch_size=batch_size, num_workers=0, collate_fn=event_collate) if train_type == Train.NEW_TRAIN: trainer.train(train_data_loader=train_data_loader, validation_data_loader=validation_data_loader) else: trainer.recovery_train(train_data_loader=train_data_loader, validation_data_loader=validation_data_loader)
def test_bilstm_gat_model_collate(lattice_ner_demo_dataset, gaz_pretrained_embedding_loader): """ 测试 bilstm gat model collate :return: """ # 仅仅取前两个作为测试 batch_instances = lattice_ner_demo_dataset[0:2] vocabulary_collate = VocabularyCollate() collate_result = vocabulary_collate(batch_instances) tokens = collate_result["tokens"] sequence_label = collate_result["sequence_labels"] token_vocabulary = Vocabulary(tokens=tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) label_vocabulary = LabelVocabulary(labels=sequence_label, padding=LabelVocabulary.PADDING) gazetter = Gazetteer( gaz_pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) gaz_vocabulary_collate = GazVocabularyCollate(gazetteer=gazetter) gaz_words = gaz_vocabulary_collate(batch_instances) gaz_vocabulary = Vocabulary(tokens=gaz_words, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) gaz_vocabulary = PretrainedVocabulary( vocabulary=gaz_vocabulary, pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) bilstm_gat_model_collate = BiLstmGATModelCollate( token_vocabulary=token_vocabulary, gazetter=gazetter, gaz_vocabulary=gaz_vocabulary, label_vocabulary=label_vocabulary) model_inputs = bilstm_gat_model_collate(batch_instances) logging.debug(json2str(model_inputs.model_inputs["metadata"])) t_graph_0 = model_inputs.model_inputs["t_graph"][0] c_graph_0 = model_inputs.model_inputs["c_graph"][0] l_graph_0 = model_inputs.model_inputs["l_graph"][0] expect_t_graph_tensor = torch.tensor(expect_t_graph, dtype=torch.uint8) ASSERT.assertTrue(is_tensor_equal(expect_t_graph_tensor, t_graph_0)) expect_c_graph_tensor = torch.tensor(expect_c_graph, dtype=torch.uint8) ASSERT.assertTrue(is_tensor_equal(expect_c_graph_tensor, c_graph_0)) expect_l_graph_tensor = torch.tensor(expect_l_graph, dtype=torch.uint8) ASSERT.assertTrue(is_tensor_equal(expect_l_graph_tensor, l_graph_0)) gaz_words_indices = model_inputs.model_inputs["gaz_words"] ASSERT.assertEqual((2, 11), gaz_words_indices.size()) metadata_0 = model_inputs.model_inputs["metadata"][0] # 陈元呼吁加强国际合作推动世界经济发展 expect_squeeze_gaz_words_0 = [ "陈元", "呼吁", "吁加", "加强", "强国", "国际", "合作", "推动", "世界", "经济", "发展" ] sequeeze_gaz_words_0 = metadata_0["sequeeze_gaz_words"] ASSERT.assertListEqual(expect_squeeze_gaz_words_0, sequeeze_gaz_words_0) expect_squeeze_gaz_words_indices_0 = torch.tensor( [gaz_vocabulary.index(word) for word in expect_squeeze_gaz_words_0], dtype=torch.long) ASSERT.assertTrue( is_tensor_equal(expect_squeeze_gaz_words_indices_0, gaz_words_indices[0]))
def test_gaz_model_collate(lattice_ner_demo_dataset, gaz_pretrained_embedding_loader): # 仅仅取前两个作为测试 batch_instances = lattice_ner_demo_dataset[0:2] vocabulary_collate = VocabularyCollate() collate_result = vocabulary_collate(batch_instances) tokens = collate_result["tokens"] sequence_label = collate_result["sequence_labels"] token_vocabulary = Vocabulary(tokens=tokens, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) label_vocabulary = LabelVocabulary(labels=sequence_label, padding=LabelVocabulary.PADDING) gazetter = Gazetteer( gaz_pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) gaz_vocabulary_collate = GazVocabularyCollate(gazetteer=gazetter) gaz_words = gaz_vocabulary_collate(batch_instances) gaz_vocabulary = Vocabulary(tokens=gaz_words, padding=Vocabulary.PADDING, unk=Vocabulary.UNK, special_first=True) gaz_vocabulary = PretrainedVocabulary( vocabulary=gaz_vocabulary, pretrained_word_embedding_loader=gaz_pretrained_embedding_loader) lattice_model_collate = LatticeModelCollate( token_vocabulary=token_vocabulary, gazetter=gazetter, gaz_vocabulary=gaz_vocabulary, label_vocabulary=label_vocabulary) model_inputs = lattice_model_collate(batch_instances) logging.debug(json2str(model_inputs.model_inputs["metadata"])) metadata_0 = model_inputs.model_inputs["metadata"][0] # 陈元呼吁加强国际合作推动世界经济发展 expect_gaz_words_0 = [["陈元"], [], ["呼吁"], ["吁加"], ["加强"], ["强国"], ["国际"], [], ["合作"], [], ["推动"], [], ["世界"], [], ["经济"], [], ["发展"], []] gaz_words_0 = metadata_0["gaz_words"] ASSERT.assertListEqual(expect_gaz_words_0, gaz_words_0) gaz_list_0 = model_inputs.model_inputs["gaz_list"][0] expect_gaz_list_0 = list() for expect_gaz_word in expect_gaz_words_0: if len(expect_gaz_word) > 0: indices = [gaz_vocabulary.index(word) for word in expect_gaz_word] lengthes = [len(word) for word in expect_gaz_word] expect_gaz_list_0.append([indices, lengthes]) else: expect_gaz_list_0.append([]) logging.debug( f"expect_gaz_list_0: {json2str(expect_gaz_list_0)}\n gaz_list_0:{json2str(gaz_list_0)}" ) ASSERT.assertListEqual(expect_gaz_list_0, gaz_list_0) tokens_0 = model_inputs.model_inputs["tokens"] ASSERT.assertEqual((2, 19), tokens_0.size()) sequence_label_0 = model_inputs.labels ASSERT.assertEqual((2, 19), sequence_label_0.size()) # 新华社华盛顿4月28日电(记者翟景升) expect_gaz_word_1 = [ ["新华社", "新华"], # 新 ["华社"], # 华 ["社华"], # 社 ["华盛顿", "华盛"], # 华 ["盛顿"], # 盛 [], # 顿 [], # 4 [], # 月 [], # 2 [], # 8 [], # 日 [], # 电 [], # ( ["记者"], # 记 [], # 者 ["翟景升", "翟景"], # 翟 ["景升"], # 景 [], # 升 [] ] # ) metadata_1 = model_inputs.model_inputs["metadata"][1] gaz_words_1 = metadata_1["gaz_words"] ASSERT.assertListEqual(expect_gaz_word_1, gaz_words_1) expect_gaz_list_1 = list() for expect_gaz_word in expect_gaz_word_1: if len(expect_gaz_word) > 0: indices = [gaz_vocabulary.index(word) for word in expect_gaz_word] lengthes = [len(word) for word in expect_gaz_word] expect_gaz_list_1.append([indices, lengthes]) else: expect_gaz_list_1.append([]) gaz_list_1 = model_inputs.model_inputs["gaz_list"][1] ASSERT.assertListEqual(expect_gaz_list_1, gaz_list_1)