def prepare(config):
"""
checks data, creates the directories,
prepare the vocabulary and embeddings
"""
logger = logging.getLogger('qarc')
logger.info('Checking the data files...')
for data_path in config.train_files + config.dev_files + config.test_files:
assert os.path.exists(data_path),\
'{} file does not exist.'.format(data_path)
logger.info('Preparing the directories...')
train_summary_dir = os.path.join(config.summary_dir, 'train')
dev_summary_dir = os.path.join(config.summary_dir, 'dev')
for dir_path in [config.vocab_dir, config.model_dir, config.result_dir, train_summary_dir, dev_summary_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info('Load dataset...')
if config.dataset_name.startswith('cmrc2018'):
qarc_data = CMRCDataset(config.max_p_len, config.max_q_len, config.max_char_len, config.max_py_len,
config.train_files, config.dev_files, config.test_files)
else:
qarc_data = BRCDataset(config.max_p_num, config.max_p_len, config.max_q_len, config.max_char_len,
config.train_files, config.dev_files, config.test_files)
logger.info('Building vocabulary...')
vocab = Vocab(lower=True)
for word in qarc_data.word_iter('train'):
vocab.add_word(word)
for char in qarc_data.char_iter('train'):
vocab.add_char(char)
for py in qarc_data.py_iter('train'):
vocab.add_py(py)
unfiltered_vocab_word_size = vocab.word_size()
vocab.filter_tokens_by_cnt(min_cnt=2)
filtered_word_num = unfiltered_vocab_word_size - vocab.word_size()
logger.info('After filter {} tokens, the final vocab size is {}'.format(
filtered_word_num, vocab.word_size()))
unfiltered_vocab_char_size = vocab.char_size()
vocab.filter_chars_by_cnt(min_cnt=2)
filtered_char_num = unfiltered_vocab_char_size - vocab.char_size()
logger.info('After filter {} chars, the final chars size is {}'.format(
filtered_char_num, vocab.char_size()))
unfiltered_vocab_py_size = vocab.py_size()
vocab.filter_pys_by_cnt(min_cnt=2)
filtered_py_num = unfiltered_vocab_py_size - vocab.py_size()
logger.info('After filter {} pys, the final pys size is {}'.format(
filtered_py_num, vocab.py_size()))
logger.info('Assigning word embeddings...')
vocab.load_pretrained_word_embeddings(
config.word2vec, config.word_embed_dim)
logger.info('Assigning char embeddings...')
# vocab.randomly_init_char_embeddings(config.char_embed_dim)
vocab.load_pretrained_char_embeddings(
config.word2vec, config.char_embed_dim)
logger.info('Assigning py embeddings...')
vocab.randomly_init_py_embeddings(config.py_embed_dim)
logger.info('Saving vocab...')
with open(os.path.join(config.vocab_dir, config.dataset_name + '_vocab.data'), 'wb') as fout:
pickle.dump(vocab, fout)
logger.info('Done with preparing!')
