args = parser.parse_args()
pw = wrapper(os.path.join(args.cp_root, args.checkpoint_name), args.checkpoint_name, enable_git_track=args.git_tracking, seed = args.seed)
gpu_index = pw.auto_device() if 'auto' == args.gpu else int(args.gpu)
device = torch.device("cuda:" + str(gpu_index) if gpu_index >= 0 else "cpu")
# device = "cpu"
logger.info('loading dataset')
key_list = ['emb_array', 'w_map', 'c_map', 'tl_map', 'cl_map', 'range', 'test_data', 'dev_data']
dataset = pickle.load(open(args.eval_dataset, 'rb'))
emb_array, w_map, c_map, tl_map, cl_map, range_idx, test_data, dev_data = [dataset[tup] for tup in key_list]
id2label = {v: k for k, v in tl_map.items()}
assert len(emb_array) == len(w_map)
train_loader = TrainDataset(args.train_dataset, w_map['<\n>'], c_map['<\n>'], args.batch_token_number, sample_ratio = args.sample_ratio)
test_loader = NERDataset(test_data, w_map['<\n>'], c_map['<\n>'], args.batch_token_number)
dev_loader = NERDataset(dev_data, w_map['<\n>'], c_map['<\n>'], args.batch_token_number)
logger.info('building model')
rnn_map = {'Basic': BasicRNN}
rnn_layer = rnn_map[args.rnn_layer](args.layer_num, args.rnn_unit, args.word_dim + args.char_dim, args.hid_dim, args.droprate, args.batch_norm)
ner_model = NER(rnn_layer, len(w_map), args.word_dim, len(c_map), args.char_dim, args.label_dim, len(tl_map), args.droprate)
ner_model.rand_ini()
ner_model.load_pretrained_word_embedding(torch.FloatTensor(emb_array))
ner_config = ner_model.to_params()
ner_model.to(device)
optim_map = {'Adam' : optim.Adam, 'Adagrad': optim.Adagrad, 'Adadelta': optim.Adadelta, 'SGD': functools.partial(optim.SGD, momentum=0.9)}
'emb_array', 'w_map', 'c_map', 'tl_map', 'cl_map', 'range',
'test_data', 'dev_data'
]
dataset = pickle.load(open(args.eval_dataset, 'rb'))
emb_array, w_map, c_map, tl_map, cl_map, range_idx, test_data, dev_data = [
dataset[tup] for tup in key_list
]
id2label = {v: k for k, v in tl_map.items()}
assert len(emb_array) == len(w_map)
train_loader = TrainDataset(args.train_dataset,
w_map['<\n>'],
c_map['<\n>'],
args.batch_token_number,
sample_ratio=args.sample_ratio)
test_loader = NERDataset(test_data, w_map['<\n>'], c_map['<\n>'],
args.batch_token_number)
dev_loader = NERDataset(dev_data, w_map['<\n>'], c_map['<\n>'],
args.batch_token_number)
pw.info('building model')
rnn_map = {'Basic': BasicRNN}
rnn_layer = rnn_map[args.rnn_layer](args.layer_num, args.rnn_unit,
args.word_dim + args.char_dim,
args.hid_dim, args.droprate,
args.batch_norm)
ner_model = NER(rnn_layer, len(w_map), args.word_dim, len(c_map),
args.char_dim, args.label_dim, len(tl_map), args.droprate)
ner_model.rand_ini()