train_token_iv_batches, train_token_ooev_batches, train_char_batches, train_label_batches, train_mask_batches \ = pickle.load(f) f.close() f = open(config.dev_data_path, 'rb') dev_token_iv_batches, dev_token_ooev_batches, dev_char_batches, dev_label_batches, dev_mask_batches \ = pickle.load(f) f.close() f = open(config.test_data_path, 'rb') test_token_iv_batches, test_token_ooev_batches, test_char_batches, test_label_batches, test_mask_batches \ = pickle.load(f) f.close() # misc info misc_config: Dict[str, Alphabet] = pickle.load(open(config.config_data_path, 'rb')) voc_iv_dict, voc_ooev_dict, char_dict, label_dict = load_dynamic_config( misc_config) config.voc_iv_size = voc_iv_dict.size() config.voc_ooev_size = voc_ooev_dict.size() config.char_size = char_dict.size() config.label_size = label_dict.size() with open(config.embed_path, 'rb') as f: vectors: List[np.ndarray] = pickle.load(f) config.token_embed = vectors[0].size embedd_word: Tensor = Tensor(vectors) config.if_gpu = config.if_gpu and torch.cuda.is_available() logger.info(config) # print training setting ner_model = BiRecurrentConvCRF4NestedNER(config.token_embed,
# load data f = open(config.train_data_path, 'rb') train_token_batches, train_label_batches, train_mask_batches = pickle.load(f) f.close() f = open(config.dev_data_path, 'rb') dev_token_batches, dev_label_batches, dev_mask_batches = pickle.load(f) f.close() f = open(config.test_data_path, 'rb') test_token_batches, test_label_batches, test_mask_batches = pickle.load(f) f.close() # misc info misc_config: Dict[str, Alphabet] = pickle.load(open(config.config_data_path, 'rb')) label_dict = load_dynamic_config(misc_config) config.label_size = label_dict.size() config.if_gpu = config.if_gpu and torch.cuda.is_available() logger.info(config) # print training setting ner_model = BiRecurrentConvCRF4NestedNER(config.embed_path, config.char_embed, config.num_filters, config.label_size, hidden_size=config.hidden_size, layers=config.layers, word_dropout=config.word_dropout, lstm_dropout=config.lstm_dropout) if config.if_gpu: