batch_size, gpu, mpath, checkpoint, bert_type, hans = parse_args()
if mpath == 'default':
mpath = 'output/{}.state_dict'.format(bert_type)
gpu = bool(gpu)
hans = bool(hans)
checkpoint = bool(checkpoint)
print('=====Arguments=====')
print('bert type:\t{}'.format(bert_type))
print('trained model path:\t{}'.format(mpath))
print('gpu:\t{}'.format(gpu))
print('checkpoint:\t{}'.format(checkpoint))
print('batch size:\t{}'.format(batch_size))
print('hans data:\t{}'.format(hans))
# Read the dataset
nli_reader = NLIDataReader('./datasets/SUFE')
test_data = nli_reader.get_examples('try.gz', max_examples=50)
if hans:
nli_reader = NLIDataReader('./datasets/Hans')
test_data += nli_reader.get_hans_examples(
'heuristics_evaluation_set.txt')
model = BertNLIModel(model_path=mpath,
batch_size=batch_size,
bert_type=bert_type)
print('test data size: {}'.format(len(test_data)))
evaluate(model, test_data, checkpoint, test_bs=batch_size)
print('all done!')
label_num = 3
model_save_path = 'output/nli_{}-{}'.format(
bert_type,
datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
print('model save path', model_save_path)
#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#### /print debug information to stdout
# Read the dataset
nli_reader = NLIDataReader('datasets/AllNLI')
train_num_labels = nli_reader.get_num_labels()
all_data = nli_reader.get_examples('train.gz') #,max_examples=5000)
random.shuffle(all_data)
train_data = all_data[:int(train_rate * len(all_data))]
dev_data = all_data[int(train_rate * len(all_data)):]
logging.info('train data size {}'.format(len(train_data)))
logging.info('dev data size {}'.format(len(dev_data)))
total_steps = math.ceil(epoch_num * len(train_data) * 1. / batch_size)
warmup_steps = int(total_steps * warmup_percent)
model = BertNLIModel(gpu=gpu,
batch_size=batch_size,
bert_type=bert_type,
if __name__ == '__main__':
batch_size, gpu, mpath, checkpoint, bert_type, hans = parse_args()
if mpath == 'default': mpath = 'output/{}.state_dict'.format(bert_type)
gpu = bool(gpu)
hans = bool(hans)
checkpoint = bool(checkpoint)
print('=====Arguments=====')
print('bert type:\t{}'.format(bert_type))
print('trained model path:\t{}'.format(mpath))
print('gpu:\t{}'.format(gpu))
print('checkpoint:\t{}'.format(checkpoint))
print('batch size:\t{}'.format(batch_size))
print('hans data:\t{}'.format(hans))
# Read the dataset
nli_reader = NLIDataReader('./datasets/AllNLI')
test_data = nli_reader.get_examples('dev.gz') #,max_examples=50)
if hans:
nli_reader = NLIDataReader('./datasets/Hans')
test_data += nli_reader.get_hans_examples(
'heuristics_evaluation_set.txt')
model = BertNLIModel(model_path=mpath,
batch_size=batch_size,
bert_type=bert_type)
print('test data size: {}'.format(len(test_data)))
evaluate(model, test_data, checkpoint, test_bs=batch_size)
model_save_path = 'output/nli_{}-{}'.format(
bert_type,
datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
print('model save path', model_save_path)
#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#### /print debug information to stdout
# Read the dataset
if hans:
nli_reader = NLIDataReader('datasets/Hans')
hans_data = nli_reader.get_hans_examples('heuristics_train_set.txt')
else:
hans_data = []
nli_reader = NLIDataReader('datasets/AllNLI')
train_num_labels = nli_reader.get_num_labels()
msnli_data = nli_reader.get_examples('train.gz') #,max_examples=5000)
all_data = msnli_data + hans_data
random.shuffle(all_data)
train_num = int(len(all_data) * 0.95)
train_data = all_data[:train_num]
dev_data = all_data[train_num:]
logging.info('train data size {}'.format(len(train_data)))
assert len(all_predict) == len(all_labels)
acc = len([
i for i in range(len(all_labels)) if all_predict[i] == all_labels[i]
]) * 1. / len(all_labels)
prf = precision_recall_fscore_support(all_labels,
all_predict,
average=None,
labels=[0, 1, 2])
if not mute:
print('==>acc<==', acc)
print('label meanings: 0: contradiction, 1: entail, 2: neutral')
print('==>precision-recall-f1<==\n', prf)
return acc
if __name__ == '__main__':
gpu = True
batch_size = 16
# Read the dataset
nli_reader = NLIDataReader('./datasets/AllNLI')
model = BertNLIModel(model_path='output/sample_model.state_dict',
batch_size=batch_size)
test_data = nli_reader.get_examples('dev.gz') #,max_examples=50)
print('test data size: {}'.format(len(test_data)))
evaluate(model, test_data)