def _load_processed_data(self):
""" Load processed data from the disk
Returns
-------
train_examples : List[Tuple]
Processed train examples. Each tuple consists of question_id, record_index,
context_tokens_indices, question_tokens_indices, context_chars_indices,
question_char_indices, start_token_index_of_the_answer, end_token_index_of_the_answer,
context, context_tokens_spans
dev_examples : List[Tuple]
Processed dev examples. Each tuple consists of question_id, record_index,
context_tokens_indices, question_tokens_indices, context_chars_indices,
question_char_indices, start_token_index_of_the_answer, end_token_index_of_the_answer,
context, context_tokens_spans
word_vocab : Vocab
Word-level vocabulary
char_vocab : Vocab
Char-level vocabulary
"""
with open(os.path.join(self._data_root_path, self._processed_train_data_file_name),
'r') as f:
train_examples = json.load(f)
with open(os.path.join(self._data_root_path, self._processed_dev_data_file_name), 'r') as f:
dev_examples = json.load(f)
with open(os.path.join(self._data_root_path, self._word_vocab_file_name), 'r') as f:
word_vocab = Vocab.from_json(json.load(f))
with open(os.path.join(self._data_root_path, self._char_vocab_file_name), 'r') as f:
char_vocab = Vocab.from_json(json.load(f))
return train_examples, dev_examples, word_vocab, char_vocab
def _get_vocabs(self):
word_list = []
char_list = []
for ds in self._datasets:
for item in ds:
words = self._get_word_tokens(item[1])
word_list.extend(words)
for word in words:
char_list.extend(iter(word))
word_counter = data.count_tokens(word_list)
char_counter = data.count_tokens(char_list)
word_vocab = Vocab(word_counter)
char_vocab = Vocab(char_counter)
# embedding_zh = gluonnlp.embedding.create('fasttext', source='cc.zh.300')
# embedding_eng = gluonnlp.embedding.create('fasttext', source='cc.en.300')
# embedding_ko = gluonnlp.embedding.create('fasttext', source='cc.ko.300')
# word_vocab.set_embedding(embedding_eng, embedding_zh, embedding_ko)
#
# count = 0
# for token, times in word_counter.items():
# if (word_vocab.embedding[token].sum() != 0).asscalar():
# count += 1
# else:
# print(token)
#
# print("{}/{} words have embeddings".format(count, len(word_counter)))
return word_vocab, char_vocab
def get_vocab(datasets):
all_words = [
word for dataset in datasets for item in dataset for word in item[0]
]
vocab = Vocab(data.count_tokens(all_words))
glove = embedding.create('glove',
source='glove.6B.' + str(args.embedding_dim) +
'd')
vocab.set_embedding(glove)
return vocab
def __init__(self, num_classes: int, embedding_dim: int, k_max: int, vocab: Vocab) -> None:
"""Instantiating VDCNN class
Args:
num_classes (int): the number of classes
embedding_dim (int): the dimension of embedding vector for token
k_max (int): the parameter of k-max pooling following last convolution block
vocab (gluonnlp.Vocab): the instance of gluonnlp.Vocab
"""
super(VDCNN, self).__init__()
self._extractor = nn.Sequential(nn.Embedding(len(vocab), embedding_dim, vocab.to_indices(vocab.padding_token)),
Permute(),
nn.Conv1d(embedding_dim, 64, 3, 1, 1),
ConvBlock(64, 64),
ConvBlock(64, 64),
nn.MaxPool1d(2, 2),
ConvBlock(64, 128),
ConvBlock(128, 128),
nn.MaxPool1d(2, 2),
ConvBlock(128, 256),
ConvBlock(256, 256),
nn.MaxPool1d(2, 2),
ConvBlock(256, 512),
ConvBlock(512, 512),
nn.AdaptiveMaxPool1d(k_max),
Flatten())
self._classifier = nn.Sequential(nn.Linear(512 * k_max, 2048),
nn.ReLU(),
nn.Linear(2048, 2048),
nn.ReLU(),
nn.Linear(2048, num_classes))
def __init__(self, num_classes: int, embedding_dim: int,
vocab: Vocab) -> None:
"""Instantiating CharCNN class
Args:
num_classes (int): the number of classes
embedding_dim (int): the dimension of embedding vector for token
vocab (gluonnlp.Vocab): the instance of gluonnlp.Vocab
"""
super(CharCNN, self).__init__()
self._extractor = nn.Sequential(
nn.Embedding(len(vocab), embedding_dim,
vocab.to_indices(vocab.padding_token)), Permute(),
nn.Conv1d(in_channels=embedding_dim,
out_channels=256,
kernel_size=7), nn.ReLU(), nn.MaxPool1d(3, 3),
nn.Conv1d(in_channels=256, out_channels=256, kernel_size=7),
nn.ReLU(), nn.MaxPool1d(3, 3),
nn.Conv1d(in_channels=256, out_channels=256, kernel_size=3),
nn.ReLU(),
nn.Conv1d(in_channels=256, out_channels=256, kernel_size=3),
nn.ReLU(),
nn.Conv1d(in_channels=256, out_channels=256, kernel_size=3),
nn.ReLU(),
nn.Conv1d(in_channels=256, out_channels=256, kernel_size=3),
nn.ReLU(), nn.MaxPool1d(3, 3), Flatten())
self._classifier = nn.Sequential(
nn.Linear(in_features=1792, out_features=512), nn.ReLU(),
nn.Dropout(), nn.Linear(in_features=512, out_features=512),
nn.ReLU(), nn.Dropout(),
nn.Linear(in_features=512, out_features=num_classes))
self.apply(self._init_weights)
def test_gluon_nlp(self):
# get corpus statistics
counter = count_tokens(['alpha', 'beta', 'gamma', 'beta'])
# create Vocab
vocab = Vocab(counter)
# find index based on token
self.assertEqual(4, vocab['beta'])
def __init__(self, label_vocab: Vocab, token_vocab: Vocab, lstm_hidden_dim: int) -> None:
"""Instantiating BilstmCRF class
Args:
token_vocab: (gluonnlp.Vocab): the instance of gluonnlp.Vocab that has token information
label_vocab: (gluonnlp.Vocab): the instance of gluonnlp.Vocab that has label information
lstm_hidden_dim (int): the number of hidden dimension of lstm
"""
super(BilstmCRF, self).__init__()
self._embedding = Embedding(token_vocab, padding_idx=token_vocab.to_indices(token_vocab.padding_token),
freeze=False, permuting=False, tracking=True)
self._pipe = Linker(permuting=False)
self._bilstm = BiLSTM(self._embedding._ops.embedding_dim, lstm_hidden_dim, using_sequence=True)
self._fc = nn.Linear(2 * lstm_hidden_dim, len(label_vocab))
self._crf = CRF(len(label_vocab), bos_tag_id=label_vocab.to_indices(label_vocab.bos_token),
eos_tag_id=label_vocab.to_indices(label_vocab.eos_token),
pad_tag_id=label_vocab.to_indices(label_vocab.padding_token))
def __init__(self, vocab: Vocab, word_dropout_ratio: float = .2) -> None:
"""Instantiating MultiChannelEmbedding class
Args:
vocab (gluonnlp.Vocab): the instance of gluonnlp.Vocab
word_dropout_ratio (float): ratio of replacing token with "<unk>" in the sequence
"""
super(MultiChannelEmbedding, self).__init__()
self._static = nn.Embedding.from_pretrained(
torch.from_numpy(vocab.embedding.idx_to_vec.asnumpy()),
freeze=True,
padding_idx=vocab.to_indices(vocab.padding_token))
self._non_static = nn.Embedding.from_pretrained(
torch.from_numpy(vocab.embedding.idx_to_vec.asnumpy()),
freeze=False,
padding_idx=vocab.to_indices(vocab.padding_token))
self._word_dropout_ratio = word_dropout_ratio
def test_join_embedding():
counter = data.Counter(["love", "走秀", "vacation"])
vocab1 = Vocab(counter)
vocab2 = Vocab(counter)
chinese_embedding = gluonnlp.embedding.create('fasttext', source='wiki.zh')
eng_embedding = gluonnlp.embedding.create('fasttext', source='wiki.simple')
vocab1.set_embedding(chinese_embedding)
vocab2.set_embedding(eng_embedding)
print(vocab1.embedding['vacation'] + vocab2.embedding['vacation'])
def _create_squad_vocab(tokenization_fn, dataset):
all_tokens = []
for data_item in dataset:
all_tokens.extend(tokenization_fn(data_item[1]))
all_tokens.extend(tokenization_fn(data_item[2]))
counter = data.count_tokens(all_tokens)
vocab = Vocab(counter)
return vocab
def __init__(self, lstm_hidden_dim: int, da: int, r: int, vocab: Vocab) -> None:
"""Instantiating SentenceEncoder class
Args:
lstm_hidden_dim (int): the number of features in the hidden states in bi-directional lstm
da (int): the number of features in hidden layer from self-attention
r (int): the number of aspects of self-attention
vocab (gluonnlp.Vocab): the instance of gluonnlp.Vocab
"""
super(SentenceEncoder, self).__init__()
self._embedding = Embedding(vocab, padding_idx=vocab.to_indices(vocab.padding_token), freeze=False,
permuting=False, tracking=True)
self._pipe = Linker(permuting=False)
self._bilstm = BiLSTM(self._embedding._ops.embedding_dim, lstm_hidden_dim, using_sequence=True)
self._attention = SelfAttention(2 * lstm_hidden_dim, da, r)
def __init__(self, num_classes: int, embedding_dim: int, hidden_dim: int,
vocab: Vocab) -> None:
"""Instantiating ConvRec class
Args:
num_classes (int): the number of classes
embedding_dim (int) : the dimension of embedding vector for token
vocab (gluonnlp.Vocab): the instance of gluonnlp.Vocab
"""
super(ConvRec, self).__init__()
self._ops = nn.Sequential(
Embedding(len(vocab),
embedding_dim,
vocab.to_indices(vocab.padding_token),
permuting=True,
tracking=True),
Conv1d(embedding_dim, hidden_dim, 5, 1, 1, F.relu, tracking=True),
MaxPool1d(2, 2, tracking=True),
Conv1d(hidden_dim, hidden_dim, 3, 1, 1, F.relu, tracking=True),
MaxPool1d(2, 2, tracking=True), Linker(permuting=True),
BiLSTM(hidden_dim, hidden_dim, using_sequence=False), nn.Dropout(),
nn.Linear(in_features=2 * hidden_dim, out_features=num_classes))
self.apply(self._init_weights)
def vocab(self):
path = os.path.join(self._path, 'vocab.json')
with io.open(path, 'r', encoding='utf-8') as in_file:
return Vocab.from_json(in_file.read())
def _get_vocabs(train_examples, dev_examples, emb_file_name,
is_cased_embedding, shrink_word_vocab, pool):
"""Create both word-level and character-level vocabularies. Vocabularies are built using
data from both train and dev datasets.
Parameters
----------
train_examples : List[dict]
Tokenized training examples
dev_examples : List[dict]
Tokenized dev examples
emb_file_name : str
Glove embedding file name
is_cased_embedding : bool
When True, provided embedding file is cased, uncased otherwise
shrink_word_vocab : bool
When True, only tokens that have embeddings in the embedding file are remained in the
word_vocab. Otherwise tokens with no embedding also stay
pool : Pool
Multiprocessing pool to use
Returns
-------
word_vocab : Vocab
Word-level vocabulary
char_vocab : Vocab
Char-level vocabulary
"""
tic = time.time()
print('Word counters receiving started.')
word_mapper = SQuADAsyncVocabMapper()
word_reducer = SQuADAsyncVocabReducer()
word_mapped = list(
tqdm.tqdm(word_mapper.run_async(
itertools.chain(train_examples, dev_examples), pool),
total=len(train_examples) + len(dev_examples)))
word_partitioned = tqdm.tqdm(SQuADDataPipeline._partition(
itertools.chain(*word_mapped)),
total=len(word_mapped))
word_counts = list(
tqdm.tqdm(word_reducer.run_async(word_partitioned, pool),
total=len(word_partitioned)))
print('Word counters received in {:.3f} sec'.format(time.time() - tic))
tic = time.time()
print('Char counters receiving started.')
char_mapper = SQuADAsyncVocabMapper(iterate_over_example=True)
char_reducer = SQuADAsyncVocabReducer()
char_mapped = list(
tqdm.tqdm(char_mapper.run_async(
itertools.chain(train_examples, dev_examples), pool),
total=len(train_examples) + len(dev_examples)))
char_partitioned = SQuADDataPipeline._partition(
itertools.chain(*char_mapped))
char_counts = list(
tqdm.tqdm(char_reducer.run_async(char_partitioned, pool),
total=len(char_partitioned)))
print('Char counters received in {:.3f} sec'.format(time.time() - tic))
embedding = nlp.embedding.create('glove', source=emb_file_name)
if is_cased_embedding:
word_counts = itertools.chain(
*[[(item[0],
item[1]), (item[0].lower(),
item[1]), (item[0].capitalize(),
item[1]), (item[0].upper(), item[1])]
for item in word_counts])
else:
word_counts = [(item[0].lower(), item[1]) for item in word_counts]
word_vocab = Vocab(
{
item[0]: item[1]
for item in word_counts
if not shrink_word_vocab or item[0] in embedding.token_to_idx
},
bos_token=None,
eos_token=None)
word_vocab.set_embedding(embedding)
char_vocab = Vocab({item[0]: item[1]
for item in char_counts},
bos_token=None,
eos_token=None)
return word_vocab, char_vocab
def read_data(word_path, label_path, nature_path, max_seq_len, PAD, NOT, PAD_NATURE, UNK):
'''
读取数据中的每个句子的词，词性，词所对应的实体的标记。对每条句子的词的长度进行长截短补到指定的
max_seq_len 的长度，对词的填充使用 PAD, 词性填充使用 PAD_NATURE， 标记填充使用 NOT。
构建 词的字典，词性的字典以及标记的字典，字典中保留位置符号 UNK
Args:
word_path: 包含每条句子的词的数据的路径
label_path: 包含每条句子的词的标记的数据的路径
nature_path: 包含每条句子的词的词性的数据的路径
max_seq_len: 最大句子长度，以词为单位
PAD: 词的填充符号
NOT: 标记的填充符号
PAD_NATURE: 词性的填充符号
UNK: 未知符号
Returns:
word_vocab：词的字典
label_vocab：词所对应的实体的标记的字典
nature_vocab：词的词性的字典
input_seqs：所有句子的输入的词的列表 [[word1, word2, ...], [word1, word2, ...], ...]
output_seqs: 所有句子的词的标记的列表 [[label1, label2, ...], [label1, label2, ...], ...]
nature_seqs：所有句子的词的词性的列表 [[nature1, nature2, ...], [nature1, nature2, ...], ...]
'''
input_tokens, output_tokens, nature_tokens = [], [], []
input_seqs, output_seqs, nature_seqs = [], [], []
with open(word_path, 'r', encoding='utf-8') as fx, open(label_path, 'r', encoding='utf-8') as fy, open(nature_path, 'r', encoding='utf-8') as fn:
word_lines = fx.readlines()
label_lines = fy.readlines()
word_natures = fn.readlines()
assert len(word_lines) == len(word_natures)
assert len(word_natures) == len(label_lines)
for word_line, label_line, word_nature in zip(word_lines, label_lines, word_natures):
input_seq = word_line.strip()
output_seq = label_line.strip()
nature_seq = word_nature.strip()
cur_input_tokens = input_seq.split(' ')
cur_output_tokens = output_seq.split(' ')
cur_nature_tokens = nature_seq.split(' ')
assert len(cur_input_tokens) == len(cur_output_tokens)
assert len(cur_output_tokens) == len(cur_nature_tokens)
# 跳过奇怪的实体类别标注
if '' in cur_output_tokens:
continue
# if-else: 长截短补
if len(cur_input_tokens) < max_seq_len or len(cur_output_tokens) < max_seq_len or len(cur_nature_tokens) < max_seq_len:
# 添加 PAD 符号使每个序列长度都为 max_seq_len
while len(cur_input_tokens) < max_seq_len:
cur_input_tokens.append(PAD)
cur_output_tokens.append(NOT)
cur_nature_tokens.append(PAD_NATURE)
else:
cur_input_tokens = cur_input_tokens[0:max_seq_len]
cur_output_tokens = cur_output_tokens[0:max_seq_len]
cur_nature_tokens = cur_nature_tokens[0:max_seq_len]
input_tokens.extend(cur_input_tokens)
output_tokens.extend(cur_output_tokens)
nature_tokens.extend(cur_nature_tokens)
# 记录序列
input_seqs.append(cur_input_tokens)
output_seqs.append(cur_output_tokens)
nature_seqs.append(cur_nature_tokens)
# 创建字典
word_vocab = Vocab(count_tokens(input_tokens), unknown_token=UNK, padding_token=PAD)
label_vocab = Vocab(count_tokens(output_tokens), unknown_token=UNK, padding_token=NOT)
nature_vocab = Vocab(count_tokens(nature_tokens), unknown_token=UNK, padding_token=PAD_NATURE)
return word_vocab, label_vocab, nature_vocab, input_seqs, output_seqs, nature_seqs
def vocab(self):
path = os.path.join(self._path, 'vocab.json')
with io.open(path, 'r', encoding='utf-8') as in_file:
return Vocab.from_json(in_file.read())