prefix='bert_meizhuang' #test_file = None, ) from data.vocab import Vocab vocab = Vocab(lower=True) import sys for word in brc_data.word_iter(None): vocab.add(word) for char in word: vocab.add_char(char) logger.info(' char size {}'.format(vocab.get_char_vocab_size())) logger.info(' vocab size {} '.format(vocab.get_word_vocab())) # unfiltered_vocab_size = vocab.size() unfiltered_char_size = vocab.get_char_vocab_size() vocab.filter_tokens_by_cnt(min_cnt=2) vocab.filter_chars_by_cnt(min_cnt=2) filtered_num = unfiltered_vocab_size - vocab.size() logger.info('After filter {} tokens, the final vocab size is {}'.format( filtered_num, vocab.size())) filtered_num = unfiltered_char_size - vocab.get_char_vocab_size() logger.info('After filter {} tokens, the final vocab size is {}'.format( filtered_num, vocab.get_char_vocab_size())) logger.info('after load embedding vocab size is {}'.format(vocab.size())) brc_data.convert_to_ids(vocab) from model.bert_base import BertBaseline
use_bert=False) from data.vocab import Vocab do_inference = True #from data.vocab import Vocab vocab = Vocab(lower=True, prefix='third_level_baihuo_') if not do_inference: for word in brc_data.word_iter(None): vocab.add(word) for char in word: vocab.add_char(char) logger.info(' char size {}'.format(vocab.get_char_vocab_size())) logger.info(' vocab size {} '.format(vocab.get_word_vocab())) # unfiltered_vocab_size = vocab.size() unfiltered_char_size = vocab.get_char_vocab_size() vocab.filter_tokens_by_cnt(min_cnt=2) do_inference: vocab.filter_chars_by_cnt(min_cnt=2) filtered_num = unfiltered_vocab_size - vocab.size() logger.info( 'After filter {} tokens, the final vocab size is {}'.format( filtered_num, vocab.size())) filtered_num = unfiltered_char_size - vocab.get_char_vocab_size() logger.info( 'After filter {} tokens, the final vocab size is {}'.format( filtered_num, vocab.get_char_vocab_size())) # # sys.exit(1) import os vocab_file = 'first_third_baihuo_vocab.txt' # vocab.load_from_file('vocab_bool.txt') if os.path.exists(vocab_file): vocab.load_from_file(vocab_file) if os.path.exists(vocab_file): vocab.load()