def do_eval(dataset=None,
vocab_file="",
eval_json="",
load_checkpoint_path="",
seq_length=384):
""" do eval """
if load_checkpoint_path == "":
raise ValueError(
"Finetune model missed, evaluation task must load finetune model!")
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=True)
eval_examples = read_squad_examples(eval_json, False)
eval_features = convert_examples_to_features(examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=seq_length,
doc_stride=128,
max_query_length=64,
is_training=False,
output_fn=None,
verbose_logging=False)
net = BertSquad(bert_net_cfg, False, 2)
net.set_train(False)
param_dict = load_checkpoint(load_checkpoint_path)
load_param_into_net(net, param_dict)
model = Model(net)
output = []
RawResult = collections.namedtuple(
"RawResult", ["unique_id", "start_logits", "end_logits"])
columns_list = ["input_ids", "input_mask", "segment_ids", "unique_ids"]
for data in dataset.create_dict_iterator():
input_data = []
for i in columns_list:
input_data.append(Tensor(data[i]))
input_ids, input_mask, segment_ids, unique_ids = input_data
start_positions = Tensor([1], mstype.float32)
end_positions = Tensor([1], mstype.float32)
is_impossible = Tensor([1], mstype.float32)
logits = model.predict(input_ids, input_mask, segment_ids,
start_positions, end_positions, unique_ids,
is_impossible)
ids = logits[0].asnumpy()
start = logits[1].asnumpy()
end = logits[2].asnumpy()
for i in range(bert_net_cfg.batch_size):
unique_id = int(ids[i])
start_logits = [float(x) for x in start[i].flat]
end_logits = [float(x) for x in end[i].flat]
output.append(
RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits))
write_predictions(eval_examples, eval_features, output, 20, 30, True,
"./predictions.json", None, None)
def test_eval():
"""Evaluation function for SQuAD task"""
tokenizer = tokenization.FullTokenizer(vocab_file="./vocab.txt",
do_lower_case=True)
input_file = "dataset/v1.1/dev-v1.1.json"
eval_examples = read_squad_examples(input_file, False)
eval_features = convert_examples_to_features(examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=384,
doc_stride=128,
max_query_length=64,
is_training=False,
output_fn=None,
verbose_logging=False)
device_id = int(os.getenv('DEVICE_ID'))
context.set_context(mode=context.GRAPH_MODE,
device_target='Ascend',
device_id=device_id)
dataset = get_squad_dataset(bert_net_cfg.batch_size, 1)
net = BertSquad(bert_net_cfg, False, 2)
net.set_train(False)
param_dict = load_checkpoint(cfg.finetune_ckpt)
load_param_into_net(net, param_dict)
model = Model(net)
output = []
RawResult = collections.namedtuple(
"RawResult", ["unique_id", "start_logits", "end_logits"])
columns_list = ["input_ids", "input_mask", "segment_ids", "unique_ids"]
for data in dataset.create_dict_iterator():
input_data = []
for i in columns_list:
input_data.append(Tensor(data[i]))
input_ids, input_mask, segment_ids, unique_ids = input_data
start_positions = Tensor([1], mstype.float32)
end_positions = Tensor([1], mstype.float32)
is_impossible = Tensor([1], mstype.float32)
logits = model.predict(input_ids, input_mask, segment_ids,
start_positions, end_positions, unique_ids,
is_impossible)
ids = logits[0].asnumpy()
start = logits[1].asnumpy()
end = logits[2].asnumpy()
for i in range(bert_net_cfg.batch_size):
unique_id = int(ids[i])
start_logits = [float(x) for x in start[i].flat]
end_logits = [float(x) for x in end[i].flat]
output.append(
RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits))
write_predictions(eval_examples, eval_features, output, 20, 30, True,
"./predictions.json", None, None, False, False)
def run_squad():
"""run squad task"""
parser = argparse.ArgumentParser(description="run squad")
parser.add_argument("--device_target",
type=str,
default="Ascend",
choices=["Ascend", "GPU"],
help="Device type, default is Ascend")
parser.add_argument("--do_train",
type=str,
default="false",
choices=["true", "false"],
help="Eable train, default is false")
parser.add_argument("--do_eval",
type=str,
default="false",
choices=["true", "false"],
help="Eable eval, default is false")
parser.add_argument("--device_id",
type=int,
default=0,
help="Device id, default is 0.")
parser.add_argument("--epoch_num",
type=int,
default=3,
help="Epoch number, default is 1.")
parser.add_argument("--num_class",
type=int,
default=2,
help="The number of class, default is 2.")
parser.add_argument("--train_data_shuffle",
type=str,
default="true",
choices=["true", "false"],
help="Enable train data shuffle, default is true")
parser.add_argument("--eval_data_shuffle",
type=str,
default="false",
choices=["true", "false"],
help="Enable eval data shuffle, default is false")
parser.add_argument("--train_batch_size",
type=int,
default=32,
help="Train batch size, default is 32")
parser.add_argument("--eval_batch_size",
type=int,
default=1,
help="Eval batch size, default is 1")
parser.add_argument("--vocab_file_path",
type=str,
default="",
help="Vocab file path")
parser.add_argument("--eval_json_path",
type=str,
default="",
help="Evaluation json file path, can be eval.json")
parser.add_argument("--save_finetune_checkpoint_path",
type=str,
default="",
help="Save checkpoint path")
parser.add_argument("--load_pretrain_checkpoint_path",
type=str,
default="",
help="Load checkpoint file path")
parser.add_argument("--load_finetune_checkpoint_path",
type=str,
default="",
help="Load checkpoint file path")
parser.add_argument("--train_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
parser.add_argument("--schema_file_path",
type=str,
default="",
help="Schema path, it is better to use absolute path")
args_opt = parser.parse_args()
epoch_num = args_opt.epoch_num
load_pretrain_checkpoint_path = args_opt.load_pretrain_checkpoint_path
save_finetune_checkpoint_path = args_opt.save_finetune_checkpoint_path
load_finetune_checkpoint_path = args_opt.load_finetune_checkpoint_path
if args_opt.do_train.lower() == "false" and args_opt.do_eval.lower(
) == "false":
raise ValueError(
"At least one of 'do_train' or 'do_eval' must be true")
if args_opt.do_train.lower(
) == "true" and args_opt.train_data_file_path == "":
raise ValueError(
"'train_data_file_path' must be set when do finetune task")
if args_opt.do_eval.lower() == "true":
if args_opt.vocab_file_path == "":
raise ValueError(
"'vocab_file_path' must be set when do evaluation task")
if args_opt.eval_json_path == "":
raise ValueError(
"'tokenization_file_path' must be set when do evaluation task")
target = args_opt.device_target
if target == "Ascend":
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=args_opt.device_id)
elif target == "GPU":
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
if bert_net_cfg.compute_type != mstype.float32:
logger.warning('GPU only support fp32 temporarily, run with fp32.')
bert_net_cfg.compute_type = mstype.float32
else:
raise Exception("Target error, GPU or Ascend is supported.")
netwithloss = BertSquad(bert_net_cfg, True, 2, dropout_prob=0.1)
if args_opt.do_train.lower() == "true":
ds = create_squad_dataset(
batch_size=args_opt.train_batch_size,
repeat_count=1,
data_file_path=args_opt.train_data_file_path,
schema_file_path=args_opt.schema_file_path,
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"))
do_train(ds, netwithloss, load_pretrain_checkpoint_path,
save_finetune_checkpoint_path, epoch_num)
if args_opt.do_eval.lower() == "true":
if save_finetune_checkpoint_path == "":
load_finetune_checkpoint_dir = _cur_dir
else:
load_finetune_checkpoint_dir = make_directory(
save_finetune_checkpoint_path)
load_finetune_checkpoint_path = LoadNewestCkpt(
load_finetune_checkpoint_dir, ds.get_dataset_size(), epoch_num,
"squad")
if args_opt.do_eval.lower() == "true":
from src import tokenization
from src.create_squad_data import read_squad_examples, convert_examples_to_features
from src.squad_get_predictions import write_predictions
from src.squad_postprocess import SQuad_postprocess
tokenizer = tokenization.FullTokenizer(
vocab_file=args_opt.vocab_file_path, do_lower_case=True)
eval_examples = read_squad_examples(args_opt.eval_json_path, False)
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=bert_net_cfg.seq_length,
doc_stride=128,
max_query_length=64,
is_training=False,
output_fn=None,
vocab_file=args_opt.vocab_file_path)
ds = create_squad_dataset(
batch_size=args_opt.eval_batch_size,
repeat_count=1,
data_file_path=eval_features,
schema_file_path=args_opt.schema_file_path,
is_training=False,
do_shuffle=(args_opt.eval_data_shuffle.lower() == "true"))
outputs = do_eval(ds, load_finetune_checkpoint_path,
args_opt.eval_batch_size)
all_predictions = write_predictions(eval_examples, eval_features,
outputs, 20, 30, True)
SQuad_postprocess(args_opt.eval_json_path,
all_predictions,
output_metrics="output.json")