def main(config_path, out_path):
logger.info('------------Match-LSTM Evaluate--------------')
logger.info('loading config file...')
global_config = read_config(config_path)
# set random seed
seed = global_config['model']['global']['random_seed']
torch.manual_seed(seed)
enable_cuda = global_config['test']['enable_cuda']
device = torch.device("cuda" if enable_cuda else "cpu")
if torch.cuda.is_available() and not enable_cuda:
logger.warning("CUDA is avaliable, you can enable CUDA in config file")
elif not torch.cuda.is_available() and enable_cuda:
raise ValueError("CUDA is not abaliable, please unable CUDA in config file")
torch.no_grad() # make sure all tensors below have require_grad=False
logger.info('reading squad dataset...')
dataset = SquadDataset(global_config)
logger.info('constructing model...')
model = MatchLSTMModel(global_config).to(device)
model.eval() # let training = False, make sure right dropout
# load model weight
logger.info('loading model weight...')
model_weight_path = global_config['data']['model_path']
assert os.path.exists(model_weight_path), "not found model weight file on '%s'" % model_weight_path
weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage)
if enable_cuda:
weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage.cuda())
model.load_state_dict(weight, strict=False)
# forward
logger.info('forwarding...')
enable_char = global_config['model']['encoder']['enable_char']
batch_size = global_config['test']['batch_size']
# batch_dev_data = dataset.get_dataloader_dev(batch_size)
batch_dev_data = list(dataset.get_batch_dev(batch_size))
# to just evaluate score or write answer to file
if out_path is None:
criterion = MyNLLLoss()
score_em, score_f1, sum_loss = eval_on_model(model=model,
criterion=criterion,
batch_data=batch_dev_data,
epoch=None,
device=device,
enable_char=enable_char,
batch_char_func=dataset.gen_batch_with_char)
logger.info("test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" % (score_em, score_f1, sum_loss))
else:
predict_ans = predict_on_model(model=model,
batch_data=batch_dev_data,
device=device,
enable_char=enable_char,
batch_char_func=dataset.gen_batch_with_char,
id_to_word_func=dataset.sentence_id2word)
samples_id = dataset.get_all_samples_id_dev()
ans_with_id = dict(zip(samples_id, predict_ans))
logging.info('writing predict answer to file %s' % out_path)
with open(out_path, 'w') as f:
json.dump(ans_with_id, f)
logging.info('finished.')
def test(config_path, out_path):
logger.info('------------MODEL PREDICT--------------')
logger.info('loading config file...')
global_config = read_config(config_path)
# set random seed
seed = global_config['global']['random_seed']
torch.manual_seed(seed)
enable_cuda = global_config['test']['enable_cuda']
device = torch.device("cuda" if enable_cuda else "cpu")
if torch.cuda.is_available() and not enable_cuda:
logger.warning("CUDA is avaliable, you can enable CUDA in config file")
elif not torch.cuda.is_available() and enable_cuda:
raise ValueError(
"CUDA is not abaliable, please unable CUDA in config file")
torch.set_grad_enabled(
False) # make sure all tensors below have require_grad=False,
logger.info('reading squad dataset...')
dataset = SquadDataset(global_config)
logger.info('constructing model...')
model_choose = global_config['global']['model']
dataset_h5_path = global_config['data']['dataset_h5']
if model_choose == 'base':
model_config = read_config('config/base_model.yaml')
model = BaseModel(dataset_h5_path, model_config)
elif model_choose == 'match-lstm':
model = MatchLSTM(dataset_h5_path)
elif model_choose == 'match-lstm+':
model = MatchLSTMPlus(dataset_h5_path)
elif model_choose == 'r-net':
model = RNet(dataset_h5_path)
elif model_choose == 'm-reader':
model = MReader(dataset_h5_path)
else:
raise ValueError('model "%s" in config file not recoginized' %
model_choose)
model = model.to(device)
model.eval() # let training = False, make sure right dropout
# load model weight
logger.info('loading model weight...')
model_weight_path = global_config['data']['model_path']
assert os.path.exists(
model_weight_path
), "not found model weight file on '%s'" % model_weight_path
weight = torch.load(model_weight_path,
map_location=lambda storage, loc: storage)
if enable_cuda:
weight = torch.load(model_weight_path,
map_location=lambda storage, loc: storage.cuda())
model.load_state_dict(weight, strict=False)
# forward
logger.info('forwarding...')
batch_size = global_config['test']['batch_size']
num_workers = global_config['global']['num_data_workers']
batch_dev_data = dataset.get_dataloader_dev(batch_size, num_workers)
# to just evaluate score or write answer to file
if out_path is None:
criterion = MyNLLLoss()
score_em, score_f1, sum_loss = eval_on_model(model=model,
criterion=criterion,
batch_data=batch_dev_data,
epoch=None,
device=device)
logger.info(
"test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" %
(score_em, score_f1, sum_loss))
else:
context_right_space = dataset.get_all_ct_right_space_dev()
predict_ans = predict_on_model(
model=model,
batch_data=batch_dev_data,
device=device,
id_to_word_func=dataset.sentence_id2word,
right_space=context_right_space)
samples_id = dataset.get_all_samples_id_dev()
ans_with_id = dict(zip(samples_id, predict_ans))
logging.info('writing predict answer to file %s' % out_path)
with open(out_path, 'w') as f:
json.dump(ans_with_id, f)
logging.info('finished.')
def test(config_path, out_path):
logger.info('------------MODEL PREDICT--------------')
logger.info('loading config file...')
global_config = read_config(config_path)
# set random seed
seed = global_config['global']['random_seed']
torch.manual_seed(seed)
enable_cuda = global_config['test']['enable_cuda']
device = torch.device("cuda" if enable_cuda else "cpu")
if torch.cuda.is_available() and not enable_cuda:
logger.warning("CUDA is avaliable, you can enable CUDA in config file")
elif not torch.cuda.is_available() and enable_cuda:
raise ValueError("CUDA is not abaliable, please unable CUDA in config file")
torch.set_grad_enabled(False) # make sure all tensors below have require_grad=False,
logger.info('reading squad dataset...')
dataset = SquadDataset(global_config)
logger.info('constructing model...')
model_choose = global_config['global']['model']
dataset_h5_path = global_config['data']['dataset_h5']
if model_choose == 'base':
model_config = read_config('config/base_model.yaml')
model = BaseModel(dataset_h5_path,
model_config)
elif model_choose == 'match-lstm':
model = MatchLSTM(dataset_h5_path)
elif model_choose == 'match-lstm+':
model = MatchLSTMPlus(dataset_h5_path)
elif model_choose == 'r-net':
model = RNet(dataset_h5_path)
elif model_choose == 'm-reader':
model = MReader(dataset_h5_path)
else:
raise ValueError('model "%s" in config file not recoginized' % model_choose)
model = model.to(device)
model.eval() # let training = False, make sure right dropout
# load model weight
logger.info('loading model weight...')
model_weight_path = global_config['data']['model_path']
assert os.path.exists(model_weight_path), "not found model weight file on '%s'" % model_weight_path
weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage)
if enable_cuda:
weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage.cuda())
model.load_state_dict(weight, strict=False)
# forward
logger.info('forwarding...')
batch_size = global_config['test']['batch_size']
num_workers = global_config['global']['num_data_workers']
batch_dev_data = dataset.get_dataloader_dev(batch_size, num_workers)
# to just evaluate score or write answer to file
if out_path is None:
criterion = MyNLLLoss()
score_em, score_f1, sum_loss = eval_on_model(model=model,
criterion=criterion,
batch_data=batch_dev_data,
epoch=None,
device=device)
logger.info("test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" % (score_em, score_f1, sum_loss))
else:
context_right_space = dataset.get_all_ct_right_space_dev()
predict_ans = predict_on_model(model=model,
batch_data=batch_dev_data,
device=device,
id_to_word_func=dataset.sentence_id2word,
right_space=context_right_space)
samples_id = dataset.get_all_samples_id_dev()
ans_with_id = dict(zip(samples_id, predict_ans))
logging.info('writing predict answer to file %s' % out_path)
with open(out_path, 'w') as f:
json.dump(ans_with_id, f)
logging.info('finished.')