def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
for dir_path in [
args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir
]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.max_word_len, args.train_files, args.dev_files,
args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
for word in brc_data.word_iter('test'):
vocab.add(word)
for word in brc_data.word_iter('dev'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=5)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size)
#vocab.load_pretrained_embeddings(args.embedding_path)
if args.use_char_level:
for char in brc_data.char_iter('train'):
vocab.add(char)
for char in brc_data.char_iter('test'):
vocab.add(char)
for char in brc_data.char_iter('dev'):
vocab.add(char)
unfiltered_char_vocab_size = vocab.char_size()
vocab.filter_chars_by_cnt(min_cnt=5)
filtered_num = unfiltered_char_vocab_size - vocab.char_size()
logger.info('After filter {} chars, the final vocab size is {}'.format(
filtered_num, vocab.char_size()))
logger.info('Assigning char embeddings...')
vocab.randomly_init_char_embeddings(args.char_embed_size)
#vocab.load_pretrained_char_embeddings(args.char_embedding_path)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def calculate_unk(train_files, target_files):
brc_data = BRCDataset(5, 500, 60, train_files, target_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
vocab.filter_tokens_by_cnt(min_cnt=2)
overlap_num = 0
dev_vocab = set()
for word in brc_data.word_iter('dev'):
dev_vocab.add(word)
for word in dev_vocab:
if word in vocab.token2id:
overlap_num += 1
print("over lap word is {} in {}".format(overlap_num, len(dev_vocab)))
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
for dir_path in [
args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir
]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
# if args.train_as:
# brc_data = BRCsysDataset(args, args.max_p_num, args.max_p_len, args.max_q_len, args.max_a_len,
# args.train_files, args.dev_files, args.test_files, prepare=True)
# else:
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.max_a_len, args.train_files, args.dev_files,
args.test_files)
vocab = Vocab(lower=True)
vocab1 = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
for word in brc_data.word_iter('train'):
vocab1.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
vocab1.filter_tokens_by_cnt(min_cnt=28)
filtered_num = unfiltered_vocab_size - vocab.size()
#logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,vocab1.size()))
logger.info('Assigning embeddings...')
# vocab.load_pretrained_embeddings('../data/jwe_word2vec_size300.txt')
# vocab1.load_pretrained_embeddings('../data/jwe_word2vec_size300.txt')
vocab.randomly_init_embeddings(args.embed_size)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
with open(os.path.join(args.vocab_dir, 'tar_vocab.data'), 'wb') as fout1:
pickle.dump(vocab1, fout1)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
vocab.size()))
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'): #构建词典只包含训练集
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
logger.info('Assigning embeddings...')
# vocab.randomly_init_embeddings(args.embed_size)#TODO-load_pretrained_embeddings
vocab.load_pretrained_embeddings(args.embedding_path) #glove pre-trained
logger.info('Saving vocab...')
# with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
with open(args.vocab_path, 'wb') as fout: #不区分search&zhidao
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepare(logger, args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger.info('Checking the data files...')
for data_path in args.trainset + args.devset + args.testset:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.save_dir, args.result_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.trainset, args.devset, args.testset)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepro(args):
logger = logging.getLogger("QAPointNet")
logger.info("====== preprocessing ======")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
dataloader = BRCDataset(args.max_p_num,args.max_p_len, args.max_q_len,args.save_dir,
args.train_files, args.dev_files, args.test_files, prepare=True)
vocab = Vocab(lower=True)
for word in dataloader.word_iter('train'):
vocab.add_word(word)
del dataloader
unfiltered_vocab_size = vocab.word_size()
vocab.filter_words_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.word_size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,vocab.word_size()))
logger.info('Assigning embeddings...')
if args.pretrained_word_path is not None:
vocab.load_pretrained_word_embeddings(args.pretrained_word_path)
else:
vocab.randomly_init_word_embeddings(args.embed_size)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('====== Done with preparing! ======')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("Cail")
logger.info('Checking the data files...')
print("Checking the data files...")
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files) # 创建这个类
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'): # 构建词典只包含训练集
vocab.add(word)
logger.info("Tokens num {}".format(vocab.size()))
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=5) # 过滤低频token
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
vocab.size()))
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size) # 随机初始化的embedding 第一步
# vocab.load_pretrained_embeddings(args.embedding_path) # glove pre-trained 第二步 提取vocab
# logger.info("Vocab size is {} from embedding".format(vocab.size()))
logger.info('Saving vocab...')
with open(args.vocab_path, 'wb') as fout: # vocab存入
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepare(args):
logger = logging.getLogger("rc")
logger.info('train test split...')
train_test_split(args.all_file, args.train_file, args.test_file,
args.train_rate)
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_file, args.dev_file, args.test_file)
data_vocabs = DataVocabs()
for word, pos, ner in brc_data.word_iter('train'):
data_vocabs.word_vocab.add(word)
data_vocabs.pos_vocab.add(pos)
data_vocabs.ner_vocab.add(ner)
unfiltered_vocab_size = data_vocabs.word_vocab.size()
data_vocabs.word_vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - data_vocabs.word_vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, data_vocabs.word_vocab.size()))
logger.info('Assigning embeddings...')
# vocab.randomly_init_embeddings(args.embed_size)
data_vocabs.word_vocab.load_pretrained_embeddings(args.embedding_path)
logger.info('embedding size: {}, {}'.format(
len(data_vocabs.word_vocab.embeddings),
len(data_vocabs.word_vocab.embeddings[0])))
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(data_vocabs, fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
for dir_path in [args.vocab_dir, args.model_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
# unfiltered_vocab_size = vocab.size()
print("vocab size is ", vocab.size())
vocab.filter_tokens_by_cnt(min_cnt=2)
print("after filtered vocab size is ", vocab.size())
# filtered_num = unfiltered_vocab_size - vocab.size()
vocab.randomly_init_embeddings(args.embed_size)
if args.use_pre_train:
vocab.load_pretrained_embeddings(args.pre_train_file)
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
def prepare(args):
"""
检查数据，创建目录，准备词汇表和词嵌入
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('检查数据文件...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} 文件不存在.'.format(data_path)
logger.info('建立目录...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('创建词汇表...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('过滤掉 {} 个词语, 最终的词汇量是 {}'.format(filtered_num,
vocab.size()))
logger.info('指定词向量...')
vocab.randomly_init_embeddings(args.embed_size)
logger.info('保存词汇表...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('完成预备过程!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
if args.use_char_embed:
char_vocab = Vocab(lower=True)
fout = open(os.path.join(args.vocab_dir, 'word.txt'), 'w')
for word in brc_data.word_iter('train'):
if word == 'mosi':
print('xxxxxxxxxx:%s\n' % word)
vocab.add(word)
if args.use_char_embed:
for char in list(word):
char_vocab.add(char)
fout.write(word + ', ' + ' '.join(list(word)) + '\n')
fout.close()
idx = vocab.get_id('mosi')
print('yyyyyyyy:mosi_idx:%d\n' % idx)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=args.min_cnt)
logger.info('min_cnt = %d ' % args.min_cnt)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
vocab.size()))
#logger.info('The final char vocab size is {}'.format(char_vocab.size()))
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size)
if args.use_char_embed:
char_vocab.randomly_init_embeddings(args.char_embed_size)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
if args.use_char_embed:
logger.info('Saving char vocab...')
with open(os.path.join(args.vocab_dir, 'char_vocab.data'), 'wb') as fout:
pickle.dump(char_vocab, fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Building vocabulary...')
# 载入数据
brc_data = BRCDataset(args.max_p_num,
args.max_p_len,
args.max_q_len,
args.train_files,
args.dev_files,
args.test_files,
prepare=True)
# 构建词典
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
logger.info('Assigning embeddings...')
# 1.随机初始化词向量 2.载入预训练词向量
if not args.pretrain:
# 保留至少出现2次的token
vocab.filter_tokens_by_cnt(min_cnt=2)
vocab.randomly_init_embeddings(args.embed_size)
else:
pre_train(brc_data, args.segmented_dir)
vocab.load_pretrained_embeddings(
os.path.join(args.segmented_dir, 'w2v_dic.data'))
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.pkl'), 'wb') as fout:
pickle.dump(vocab, fout)
fout.close()
logger.info('Saving sets...')
with open(os.path.join(args.prepared_dir, 'train_set.pkl'),
'wb') as f_train_out:
pickle.dump(brc_data.train_set, f_train_out)
f_train_out.close()
with open(os.path.join(args.prepared_dir, 'dev_set.pkl'),
'wb') as f_dev_out:
pickle.dump(brc_data.dev_set, f_dev_out)
f_dev_out.close()
with open(os.path.join(args.prepared_dir, 'test_set.pkl'),
'wb') as f_test_out:
pickle.dump(brc_data.test_set, f_test_out)
f_test_out.close()
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
# 确保路径存在
# train_files dev_files test_files
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
# 准备保存数据的目录，如果目录不存在，则创建
# vocab_dir model_dir result_dir summary_dir
logger.info('Preparing the directories...')
for dir_path in [
args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir
]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# 准备数据
# 传入最大p数量 最大p长度 最大q长度 以及 训练 开发 测试文件目录
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
# 创建字典
vocab = Vocab(lower=True)
# 得到所有的字词，创建 token2id id2token等
for word in brc_data.word_iter('train'):
vocab.add(word)
# 未过滤之前的大小
unfiltered_vocab_size = vocab.size()
# 过滤词频低于2的词
vocab.filter_tokens_by_cnt(min_cnt=2)
# 过滤的数量 = 未过滤前的数量 - 过滤后的数量
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
#
logger.info('Assigning embeddings...')
# 随机初始化词向量 size * 300
# 5006 * 300
vocab.randomly_init_embeddings(args.embed_size)
logger.info('Saving vocab...')
# 保存词汇表
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
#建立数据集
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,args.word_size,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'): #将问题和选中答案的所有词作为vocabulary
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=1) #5
filtered_num = unfiltered_vocab_size - vocab.size() #得到过滤了多少个单词
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
vocab.size()))
logger.info('Assigning embeddings...')
#vocab.randomly_init_embeddings(args.embed_size)
vocab.word2vec_init_embeddings(args.embed_size,sentences=brc_data.generate_word2vec_trainset(),min_cnt=1)
#vocab.word2vec_init_embeddings(args.embed_size,sentences=brc_data.generate_word2vec_testset(),min_cnt=1)
#vocab.glove_init_embeddings(args.embed_size)
char_vocab=Vocab(lower=True)
for char in brc_data.char_iter('train'):
char_vocab.add(char)
unfiltered_vocab_size=char_vocab.size()
char_vocab.filter_tokens_by_cnt(min_cnt=1)
filtered_num=unfiltered_vocab_size-char_vocab.size()
logger.info('after filter {} tokens,the char vocab size is {}'.format(filtered_num,char_vocab.size()))
char_vocab.char2vec_init_embeddings(args.char_embed_size,sentences=brc_data.generate_char2vec_trainset(),min_cnt=1)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
with open(os.path.join(args.vocab_dir,'char_vocab.data'),'wb') as fout:
pickle.dump(char_vocab,fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
for dir_path in [
args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir
]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
# logger.info('Assigning embeddings...')
# vocab.randomly_init_embeddings(args.embed_size) #random init in prepare!!
#save the datasets to records files.
logger.info('Saving the datasets.')
brc_data.convert_to_ids(vocab)
pad_id = vocab.get_id(vocab.pad_token)
brc_data.save_records(pad_id)
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
def prepare(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
# 验证输入文件地址正确
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(data_path)
# 准备vocab,model,result,summary目录
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir, args.summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# brc_data 初始化 默认（5,500,60,/data/demo下的文件)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files, args.dev_files, args.test_files)
# vocab 关键是有几个列表： id2token token2id 问题和文章里 所有的单词和id 对应起来 还统计了单词出现的次数token_cnt
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size() # vocab size id2token列表长度
vocab.filter_tokens_by_cnt(min_cnt=2)# 过滤出现次数少于2的 然后重建id2token 和 token2id
# demo里过滤了5225 剩下 5006
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
vocab.size()))
# 开始随机分配 embedding
logger.info('Assigning embeddings...')
vocab.randomly_init_embeddings(args.embed_size) # 参数 词嵌入向量维度 默认30 注意 <blank> and <unk> 全是0
# 保存 pickle 保存运行中对象 保存到data/vocab中
logger.info('Saving vocab...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
'--train_files',
nargs='+',
# default=['./data/demo/trainset/search.train.json'],
# default=['./data/preprocessed/trainset/search.train.json',
# './data/preprocessed/trainset/zhidao.train.json'],
default=['./data/preprocessed/trainset/search.train.json'],
help='list of files that contain the preprocessed train data')
parser.add_argument('--pre_train_file',
type=str,
default='./data/wiki.zh.new.vec',
help='pre_train files')
args = parser.parse_args()
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.train_files)
vocab_set = set()
for word in brc_data.word_iter('train'):
vocab_set.add(word.lower())
f = load_model(args.model_name)
write_file = codecs.open(args.pre_train_file, 'w', 'utf-8')
for vocab in vocab_set:
value_str = [str(i) for i in f.get_word_vector(vocab)]
write_file.write(vocab + " " + " ".join(value_str) + '\n')
write_file.close()
print("over!")
# vocab_set = Vocab(lower=True)
# for word in brc_data.word_iter('train'):
# vocab_set.add(word)
# print("vocab size is ", vocab_set.size())
# vocab_set.filter_tokens_by_cnt(min_cnt=2)
def train(args):
"""
checks data, creates the directories, prepare the vocabulary and embeddings
"""
logger = logging.getLogger("brc")
logger.info('Checking the data files...')
for data_path in args.train_files + args.dev_files + args.test_files:
assert os.path.exists(data_path), '{} file does not exist.'.format(
data_path)
logger.info('Preparing the directories...')
for dir_path in [args.vocab_dir, args.model_dir, args.result_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Building vocabulary...')
brc_data = BRCDataset(args.max_p_num, args.max_p_len, args.max_q_len,
args.max_train_sample_num, args.train_files)
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
logger.info('Assigning embeddings...')
vocab.load_pretrained_embeddings(args.word_embedding_path)
#vocab.randomly_init_embeddings(300)
#vocab1.randomly_init_embeddings(300)
logger.info('Saving vocab...')
brc_data.convert_to_ids(vocab)
logger.info('Initialize the model...')
with open(os.path.join(args.vocab_dir, 'vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
rc_model = S_netModel(vocab, args)
logger.info('Training the model...')
#rc_model.restore(model_dir=args.model_dir, model_prefix=args.algo +'sys')
#if args.train_as:
# rc_model.restore(model_dir=args.model_dir, model_prefix=args.algo + 'syst')
rc_model.train(brc_data,
args.epochs,
args.batch_size,
save_dir=args.model_dir,
save_prefix=args.algo,
dropout_keep_prob=args.dropout_keep_prob)
logger.info('Done with model training!')
logger.info('evaluate the trained model!')
test_batches = brc_data.gen_mini_batches('test',
args.batch_size,
pad_id=vocab.get_id(
vocab.pad_token),
shuffle=False)
rc_model.evaluate(test_batches,
result_dir=args.result_dir,
result_prefix='test.predicted')
logger.info('Done with model evaluating !')
def gen_vocab_pretrain_w2v():
"""
在预训练的过程也将结果写到vocab中
:return:
"""
max_p_num = 5
max_p_len = 500
max_q_len = 60
embed_size = 300
pretrain = True
pretrained_word_path = "" # 是否采用第三方预训练的词向量
train_files = [
'../data/DuReader2.0/trainset/search.train.json',
'../data/DuReader2.0/trainset/zhidao.train.json'
]
dev_files = [
'../data/DuReader2.0/devset/search.dev.json',
'../data/DuReader2.0/devset/zhidao.dev.json'
]
test_files = [
'../data/DuReader2.0/testset/search.test.json',
'../data/DuReader2.0/testset/zhidao.test.json'
]
prepared_dir = ""
vocab_dir = "../data/test_write_read_tf/vocab"
segmented_dir = "../data/DuReader2.0/whole_segmented"
for dir_path in [segmented_dir, vocab_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
brc_data = BRCDataset(max_p_num,
max_p_len,
max_q_len,
train_files,
dev_files,
test_files,
prepared_dir,
prepare=True)
# 此时占用内存27GB
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
print('Assigning embeddings...')
if pretrain:
# 基于已有的分词结果训练词向量
if pretrained_word_path:
# 如果指定了预训练路径，则不需要重新训练，否则根据给定的语料重新训练
vocab.load_pretrained_embeddings(pretrained_word_path)
else:
vocab.load_pretrained_embeddings(
os.path.join(segmented_dir, 'w2v_dic.data'))
else:
vocab.randomly_init_embeddings(embed_size)
print('Saving vocab,only train data...')
with open(os.path.join(vocab_dir, 'vocab_traindata_pretrainW2V.data'),
'wb') as fout:
pickle.dump(vocab, fout)
def gen_vocab():
"""
一次性遍历数据集，生成vocab。该vocab是为了后期将token转为id
:return:
"""
max_p_num = 5
max_p_len = 500
max_q_len = 60
embed_size = 300
pretrain = True
pretrained_word_path = ""
train_files = [
'../data/DuReader2.0/trainset/search.train.json',
'../data/DuReader2.0/trainset/zhidao.train.json'
]
dev_files = [
'../data/DuReader2.0/devset/search.dev.json',
'../data/DuReader2.0/devset/zhidao.dev.json'
]
test_files = [
'../data/DuReader2.0/testset/search.test.json',
'../data/DuReader2.0/testset/zhidao.test.json'
]
prepared_dir = "../data/test_write_read_tf/prepared"
vocab_dir = "../data/test_write_read_tf/vocab"
segmented_dir = "../data/test_write_read_tf/segmented"
for dir_path in [prepared_dir, segmented_dir, vocab_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# pdb.set_trace()
brc_data = BRCDataset(max_p_num,
max_p_len,
max_q_len,
train_files,
dev_files,
test_files,
prepared_dir,
prepare=True)
# 此时占用内存27GB
vocab = Vocab(lower=True)
for word in brc_data.word_iter('train'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
print('Assigning embeddings...')
if pretrain:
# 基于已有的分词结果训练词向量
if pretrained_word_path:
# 如果指定了预训练路径，则不需要重新训练，否则根据给定的语料重新训练
vocab.load_pretrained_embeddings(pretrained_word_path)
else:
# 训练耗时20分钟左右
# pre_train(brc_data, segmented_dir) 预训练和生成vocab是分开的。严格上来说，预训练包括了vocab的生成
vocab.load_pretrained_embeddings(
os.path.join(segmented_dir, 'w2v_dic.data'))
else:
vocab.randomly_init_embeddings(embed_size)
print('Saving vocab,only train data...')
with open(os.path.join(vocab_dir, 'vocab_traindata_pretrainW2V.data'),
'wb') as fout:
pickle.dump(vocab, fout)
del brc_data.train_set
# 加入test data
print('Saving vocab,train + test data...')
for word in brc_data.word_iter('test'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
print('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
with open(os.path.join(vocab_dir, 'vocab_train_test_data.data'),
'wb') as fout1:
pickle.dump(vocab, fout1)
# 加入dev data
print('Saving vocab,train + test + dev data...')
for word in brc_data.word_iter('dev'):
vocab.add(word)
unfiltered_vocab_size = vocab.size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_num = unfiltered_vocab_size - vocab.size()
print('After filter {} tokens, the final vocab size is {}'.format(
filtered_num, vocab.size()))
with open(os.path.join(vocab_dir, 'vocab_train_test_dev_data.data'),
'wb') as fout2:
pickle.dump(vocab, fout2)
logger.info('Done with preparing!')