train_token_iv_batches, train_token_ooev_batches, train_char_batches, train_label_batches, train_mask_batches \
= pickle.load(f)
f.close()
f = open(config.dev_data_path, 'rb')
dev_token_iv_batches, dev_token_ooev_batches, dev_char_batches, dev_label_batches, dev_mask_batches \
= pickle.load(f)
f.close()
f = open(config.test_data_path, 'rb')
test_token_iv_batches, test_token_ooev_batches, test_char_batches, test_label_batches, test_mask_batches \
= pickle.load(f)
f.close()
# misc info
misc_config: Dict[str,
Alphabet] = pickle.load(open(config.config_data_path, 'rb'))
voc_iv_dict, voc_ooev_dict, char_dict, label_dict = load_dynamic_config(
misc_config)
config.voc_iv_size = voc_iv_dict.size()
config.voc_ooev_size = voc_ooev_dict.size()
config.char_size = char_dict.size()
config.label_size = label_dict.size()
with open(config.embed_path, 'rb') as f:
vectors: List[np.ndarray] = pickle.load(f)
config.token_embed = vectors[0].size
embedd_word: Tensor = Tensor(vectors)
config.if_gpu = config.if_gpu and torch.cuda.is_available()
logger.info(config) # print training setting
ner_model = BiRecurrentConvCRF4NestedNER(config.token_embed,
# load data
f = open(config.train_data_path, 'rb')
train_token_batches, train_label_batches, train_mask_batches = pickle.load(f)
f.close()
f = open(config.dev_data_path, 'rb')
dev_token_batches, dev_label_batches, dev_mask_batches = pickle.load(f)
f.close()
f = open(config.test_data_path, 'rb')
test_token_batches, test_label_batches, test_mask_batches = pickle.load(f)
f.close()
# misc info
misc_config: Dict[str,
Alphabet] = pickle.load(open(config.config_data_path, 'rb'))
label_dict = load_dynamic_config(misc_config)
config.label_size = label_dict.size()
config.if_gpu = config.if_gpu and torch.cuda.is_available()
logger.info(config) # print training setting
ner_model = BiRecurrentConvCRF4NestedNER(config.embed_path,
config.char_embed,
config.num_filters,
config.label_size,
hidden_size=config.hidden_size,
layers=config.layers,
word_dropout=config.word_dropout,
lstm_dropout=config.lstm_dropout)
if config.if_gpu: