data_dir = "/search/hadoop02/suanfa/songyingxin/Github/chiqianqian"
output_dir = ".demo_output/"
cache_dir = ".demo_cache"
log_dir = ".demo_log/"
model_times = "model_1/" # 第几次保存的模型，主要是用来获取最佳结果
bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased-vocab.txt"
bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased"
from Processors.DemoProcessor import DemoProcessor
if model_name == "BertOrigin":
from BertOrigin import args
main(
args.get_args(data_dir, output_dir, cache_dir, bert_vocab_file,
bert_model_dir, log_dir), model_times, DemoProcessor)
elif model_name == "BertCNN":
from BertCNN import args
main(
args.get_args(data_dir, output_dir, cache_dir, bert_vocab_file,
bert_model_dir, log_dir), model_times, DemoProcessor)
elif model_name == "BertATT":
from BertATT import args
main(
args.get_args(data_dir, output_dir, cache_dir, bert_vocab_file,
bert_model_dir, log_dir), model_times, DemoProcessor)
elif model_name == "BertRCNN":
from BertRCNN import args
main(
args.get_args(data_dir, output_dir, cache_dir, bert_vocab_file,
# model_times = "model_1/" # 第几次保存的模型，主要是用来获取最佳结果
# bert-base
bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased-vocab.txt"
bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased"
# # bert-large
# bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-large-uncased-vocab.txt"
# bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-large-uncased"
if model_name == "BertOrigin":
from BertOrigin import args
elif model_name == "BertCNN":
from BertCNN import args
elif model_name == "BertATT":
from BertATT import args
elif model_name == "BertRCNN":
from BertRCNN import args
elif model_name == "BertCNNPlus":
from BertCNNPlus import args
config = args.get_args(data_dir, output_dir, cache_dir, bert_vocab_file,
bert_model_dir, log_dir)
main(config, config.save_name, label_list)
model_name = "BertOrigin"
data_dir = "/search/hadoop02/suanfa/songyingxin/Github/Competition"
output_dir = ".sst_output/"
cache_dir = ".sst_cache"
log_dir = ".sst_log/"
model_times = "model_1/" # 第几次保存的模型，主要是用来获取最佳结果
bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased-vocab.txt"
bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased"
from Processors.CompetitionProcessor import CompetitionProcessor
if model_name == "BertOrigin":
from BertOrigin import args
main(args.get_args(data_dir, output_dir, cache_dir,
bert_vocab_file, bert_model_dir, log_dir), model_times, CompetitionProcessor)
elif model_name == "BertCNN":
from BertCNN import args
main(args.get_args(data_dir, output_dir, cache_dir,
bert_vocab_file, bert_model_dir, log_dir),
model_times, CompetitionProcessor)
elif model_name == "BertATT":
from BertATT import args
main(args.get_args(data_dir, output_dir, cache_dir,
bert_vocab_file, bert_model_dir, log_dir),
model_times, CompetitionProcessor)
elif model_name == "BertRCNN":
from BertRCNN import args
for label in label_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
print_list = ['macro avg', 'weighted avg']
for label in print_list:
print('\t {}: Precision: {} | recall: {} | f1 score: {}'.format(
label, test_report[label]['precision'],
test_report[label]['recall'], test_report[label]['f1-score']))
if __name__ == "__main__":
model_name = "BertSegment"
data_dir = "/search/hadoop02/suanfa/songyingxin/data/RACE/all/segment/400"
output_dir = ".BertSegment_output"
cache_dir = ".BertSegment_cache"
log_dir = ".BertSegment_log"
# bert-base
bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased-vocab.txt"
bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-base-uncased"
# bert_vocab_file = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-large-uncased-vocab.txt"
# bert_model_dir = "/search/hadoop02/suanfa/songyingxin/pytorch_Bert/bert-large-uncased"
if model_name == "BertSegment":
from BertSegment import args
main(args.get_args(data_dir, output_dir, cache_dir, log_dir),
bert_vocab_file, bert_model_dir)
test_model = BertOrigin(config, num_classes=2)
pretrained_model_dict = torch.load(config.best_model_file)
new_state_dict = OrderedDict()
for k, value in pretrained_model_dict.items():
#name = k[7:] # remove `module.`
new_state_dict[k] = value
test_model.load_state_dict(new_state_dict, strict=True)
test_model.to(device)
if n_gpu > 1:
test_model = nn.DataParallel(test_model, device_ids=gpu_ids)
test_acc, test_f1 = evaluate(test_model, test_dataloader, device)
print(f'\t Acc: {test_acc*100: .3f}% | f1: {test_f1*100: .3f}%')
if __name__ == "__main__":
model_name = "BertOrigin"
data_dir = "dataset/"
gpu_ids = "3"
# roberta
bert_vocab_file = "model/bert/vocab.txt"
# do_prediction: False(训练), True(预测)
main(args.get_args(data_dir, gpu_ids), bert_vocab_file, do_prediction=True)
