def do_eval(args):
paddle.set_device(args.device)
# Create dataset, tokenizer and dataloader.
train_ds, eval_ds = load_dataset('msra_ner',
splits=('train', 'test'),
lazy=False)
tokenizer = BertTokenizer.from_pretrained(args.model_name_or_path)
label_list = train_ds.label_list
label_num = len(label_list)
no_entity_id = label_num - 1
trans_func = partial(tokenize_and_align_labels,
tokenizer=tokenizer,
no_entity_id=no_entity_id,
max_seq_len=args.max_seq_length)
ignore_label = -100
batchify_fn = lambda samples, fn=Dict({
'input_ids':
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'), # input
'token_type_ids':
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'
), # segment
'seq_len':
Stack(dtype='int64'),
'labels':
Pad(axis=0, pad_val=ignore_label, dtype='int64') # label
}): fn(samples)
eval_ds = eval_ds.map(trans_func)
eval_data_loader = DataLoader(dataset=eval_ds,
collate_fn=batchify_fn,
num_workers=0,
batch_size=args.batch_size,
return_list=True)
# Define the model netword and its loss
model = BertForTokenClassification.from_pretrained(args.model_name_or_path,
num_classes=label_num)
if args.init_checkpoint_path:
model_dict = paddle.load(args.init_checkpoint_path)
model.set_dict(model_dict)
loss_fct = paddle.nn.loss.CrossEntropyLoss(ignore_index=ignore_label)
metric = ChunkEvaluator(label_list=label_list)
model.eval()
metric.reset()
for step, batch in enumerate(eval_data_loader):
input_ids, token_type_ids, length, labels = batch
logits = model(input_ids, token_type_ids)
loss = loss_fct(logits, labels)
avg_loss = paddle.mean(loss)
preds = logits.argmax(axis=2)
num_infer_chunks, num_label_chunks, num_correct_chunks = metric.compute(
length, preds, labels)
metric.update(num_infer_chunks.numpy(), num_label_chunks.numpy(),
num_correct_chunks.numpy())
precision, recall, f1_score = metric.accumulate()
print("eval loss: %f, precision: %f, recall: %f, f1: %f" %
(avg_loss, precision, recall, f1_score))
def evaluate(args):
paddle.set_device(args.device)
# create dataset.
test_ds = load_dataset(datafiles=(os.path.join(args.data_dir, 'test.tsv')))
word_vocab = load_vocab(os.path.join(args.data_dir, 'word.dic'))
label_vocab = load_vocab(os.path.join(args.data_dir, 'tag.dic'))
# q2b.dic is used to replace DBC case to SBC case
normlize_vocab = load_vocab(os.path.join(args.data_dir, 'q2b.dic'))
trans_func = partial(
convert_example,
max_seq_len=args.max_seq_len,
word_vocab=word_vocab,
label_vocab=label_vocab,
normlize_vocab=normlize_vocab)
test_ds.map(trans_func)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=0, dtype='int64'), # word_ids
Stack(dtype='int64'), # length
Pad(axis=0, pad_val=0, dtype='int64'), # label_ids
): fn(samples)
# Create sampler for dataloader
test_sampler = paddle.io.BatchSampler(
dataset=test_ds,
batch_size=args.batch_size,
shuffle=False,
drop_last=False)
test_loader = paddle.io.DataLoader(
dataset=test_ds,
batch_sampler=test_sampler,
return_list=True,
collate_fn=batchify_fn)
# Define the model network and metric evaluator
model = BiGruCrf(args.emb_dim, args.hidden_size,
len(word_vocab), len(label_vocab))
chunk_evaluator = ChunkEvaluator(label_list=label_vocab.keys(), suffix=True)
# Load the model and start predicting
model_dict = paddle.load(args.init_checkpoint)
model.load_dict(model_dict)
model.eval()
chunk_evaluator.reset()
for batch in test_loader:
token_ids, length, labels = batch
preds = model(token_ids, length)
num_infer_chunks, num_label_chunks, num_correct_chunks = chunk_evaluator.compute(
length, preds, labels)
chunk_evaluator.update(num_infer_chunks.numpy(),
num_label_chunks.numpy(),
num_correct_chunks.numpy())
precision, recall, f1_score = chunk_evaluator.accumulate()
print("eval precision: %f, recall: %f, f1: %f" %
(precision, recall, f1_score))
def test_ner_dataset(client):
from paddlenlp.metrics import ChunkEvaluator
from datasets import load_dataset
import paddle
dev_ds = load_dataset("msra_ner", split="test")
import os
if os.environ.get('https_proxy'):
del os.environ['https_proxy']
if os.environ.get('http_proxy'):
del os.environ['http_proxy']
print("Start infer...")
metric = ChunkEvaluator(
label_list=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'])
idx = 0
batch_size = 32
max_len = len(dev_ds["tokens"]) - 1
while idx < max_len:
end_idx = idx + batch_size if idx + batch_size < max_len else max_len
data = dev_ds["tokens"][idx:end_idx]
ret = client.predict(feed_dict={"tokens": data})
if ret.err_no != 0:
raise ValueError("err_no", ret.err_no, "err_msg: ", ret.err_msg)
# print("ret:", ret)
if idx < batch_size * 2:
print_ret(json.loads(ret.value[0]), data)
# calculate metric
preds = json.loads(ret.value[1])
label_list = dev_ds["ner_tags"][idx:end_idx]
label_list = label_pad(label_list, preds)
label_list = paddle.to_tensor(label_list)
preds = paddle.to_tensor(preds)
seq_len = [preds.shape[1]] * preds.shape[0]
num_infer_chunks, num_label_chunks, num_correct_chunks = metric.compute(
paddle.to_tensor(seq_len), preds, label_list)
metric.update(num_infer_chunks.numpy(), num_label_chunks.numpy(),
num_correct_chunks.numpy())
idx += batch_size
print(idx)
res = metric.accumulate()
print("acc: ", res)
class Electra(nn.Layer):
"""
Electra model
"""
def __init__(
self,
task: str = None,
load_checkpoint: str = None,
label_map: Dict = None,
num_classes: int = 2,
**kwargs,
):
super(Electra, self).__init__()
if label_map:
self.label_map = label_map
self.num_classes = len(label_map)
else:
self.num_classes = num_classes
if task == 'sequence_classification':
task = 'seq-cls'
logger.warning(
"current task name 'sequence_classification' was renamed to 'seq-cls', "
"'sequence_classification' has been deprecated and will be removed in the future.",
)
if task == 'seq-cls':
self.model = ElectraForSequenceClassification.from_pretrained(
pretrained_model_name_or_path='electra-small',
num_classes=self.num_classes,
**kwargs)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric = paddle.metric.Accuracy()
elif task == 'token-cls':
self.model = ElectraForTokenClassification.from_pretrained(
pretrained_model_name_or_path='electra-small',
num_classes=self.num_classes,
**kwargs)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric = ChunkEvaluator(label_list=[
self.label_map[i] for i in sorted(self.label_map.keys())
])
elif task == 'text-matching':
self.model = ElectraModel.from_pretrained(
pretrained_model_name_or_path='electra-small', **kwargs)
self.dropout = paddle.nn.Dropout(0.1)
self.classifier = paddle.nn.Linear(
self.model.config['hidden_size'] * 3, 2)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric = paddle.metric.Accuracy()
elif task is None:
self.model = ElectraModel.from_pretrained(
pretrained_model_name_or_path='electra-small', **kwargs)
else:
raise RuntimeError(
"Unknown task {}, task should be one in {}".format(
task, self._tasks_supported))
self.task = task
if load_checkpoint is not None and os.path.isfile(load_checkpoint):
state_dict = paddle.load(load_checkpoint)
self.set_state_dict(state_dict)
logger.info('Loaded parameters from %s' %
os.path.abspath(load_checkpoint))
def forward(self,
input_ids=None,
token_type_ids=None,
position_ids=None,
attention_mask=None,
query_input_ids=None,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_input_ids=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
seq_lengths=None,
labels=None):
if self.task != 'text-matching':
result = self.model(input_ids, token_type_ids, position_ids,
attention_mask)
else:
query_result = self.model(query_input_ids, query_token_type_ids,
query_position_ids, query_attention_mask)
title_result = self.model(title_input_ids, title_token_type_ids,
title_position_ids, title_attention_mask)
if self.task == 'seq-cls':
logits = result
probs = F.softmax(logits, axis=1)
if labels is not None:
loss = self.criterion(logits, labels)
correct = self.metric.compute(probs, labels)
acc = self.metric.update(correct)
return probs, loss, {'acc': acc}
return probs
elif self.task == 'token-cls':
logits = result
token_level_probs = F.softmax(logits, axis=-1)
preds = token_level_probs.argmax(axis=-1)
if labels is not None:
loss = self.criterion(logits, labels.unsqueeze(-1))
num_infer_chunks, num_label_chunks, num_correct_chunks = \
self.metric.compute(None, seq_lengths, preds, labels)
self.metric.update(num_infer_chunks.numpy(),
num_label_chunks.numpy(),
num_correct_chunks.numpy())
_, _, f1_score = map(float, self.metric.accumulate())
return token_level_probs, loss, {'f1_score': f1_score}
return token_level_probs
elif self.task == 'text-matching':
query_token_embedding = query_result
query_token_embedding = self.dropout(query_token_embedding)
query_attention_mask = paddle.unsqueeze(
(query_input_ids != self.model.pad_token_id).astype(
query_token_embedding.dtype),
axis=2)
query_token_embedding = query_token_embedding * query_attention_mask
query_sum_embedding = paddle.sum(query_token_embedding, axis=1)
query_sum_mask = paddle.sum(query_attention_mask, axis=1)
query_mean = query_sum_embedding / query_sum_mask
title_token_embedding = title_result
title_token_embedding = self.dropout(title_token_embedding)
title_attention_mask = paddle.unsqueeze(
(title_input_ids != self.model.pad_token_id).astype(
title_token_embedding.dtype),
axis=2)
title_token_embedding = title_token_embedding * title_attention_mask
title_sum_embedding = paddle.sum(title_token_embedding, axis=1)
title_sum_mask = paddle.sum(title_attention_mask, axis=1)
title_mean = title_sum_embedding / title_sum_mask
sub = paddle.abs(paddle.subtract(query_mean, title_mean))
projection = paddle.concat([query_mean, title_mean, sub], axis=-1)
logits = self.classifier(projection)
probs = F.softmax(logits)
if labels is not None:
loss = self.criterion(logits, labels)
correct = self.metric.compute(probs, labels)
acc = self.metric.update(correct)
return probs, loss, {'acc': acc}
return probs
else:
sequence_output, pooled_output = result
return sequence_output, pooled_output
@staticmethod
def get_tokenizer(*args, **kwargs):
"""
Gets the tokenizer that is customized for this module.
"""
return ElectraTokenizer.from_pretrained(
pretrained_model_name_or_path='electra-small', *args, **kwargs)
class ErnieV2(nn.Layer):
"""
Ernie model
"""
def __init__(
self,
task: str = None,
load_checkpoint: str = None,
label_map: Dict = None,
num_classes: int = 2,
**kwargs,
):
super(ErnieV2, self).__init__()
if label_map:
self.label_map = label_map
self.num_classes = len(label_map)
else:
self.num_classes = num_classes
if task == 'sequence_classification':
task = 'seq-cls'
logger.warning(
"current task name 'sequence_classification' was renamed to 'seq-cls', "
"'sequence_classification' has been deprecated and will be removed in the future.",
)
if task == 'seq-cls':
self.model = ErnieForSequenceClassification.from_pretrained(
pretrained_model_name_or_path='ernie-2.0-en',
num_classes=self.num_classes,
**kwargs)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric = paddle.metric.Accuracy()
elif task == 'token-cls':
self.model = ErnieForTokenClassification.from_pretrained(
pretrained_model_name_or_path='ernie-2.0-en',
num_classes=self.num_classes,
**kwargs)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric = ChunkEvaluator(label_list=[
self.label_map[i] for i in sorted(self.label_map.keys())
])
elif task is None:
self.model = ErnieModel.from_pretrained(
pretrained_model_name_or_path='ernie-2.0-en', **kwargs)
else:
raise RuntimeError(
"Unknown task {}, task should be one in {}".format(
task, self._tasks_supported))
self.task = task
if load_checkpoint is not None and os.path.isfile(load_checkpoint):
state_dict = paddle.load(load_checkpoint)
self.set_state_dict(state_dict)
logger.info('Loaded parameters from %s' %
os.path.abspath(load_checkpoint))
def forward(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None,
seq_lengths=None,
labels=None):
result = self.model(input_ids, token_type_ids, position_ids,
attention_mask)
if self.task == 'seq-cls':
logits = result
probs = F.softmax(logits, axis=1)
if labels is not None:
loss = self.criterion(logits, labels)
correct = self.metric.compute(probs, labels)
acc = self.metric.update(correct)
return probs, loss, {'acc': acc}
return probs
elif self.task == 'token-cls':
logits = result
token_level_probs = F.softmax(logits, axis=-1)
preds = token_level_probs.argmax(axis=-1)
if labels is not None:
loss = self.criterion(logits, labels.unsqueeze(-1))
num_infer_chunks, num_label_chunks, num_correct_chunks = \
self.metric.compute(None, seq_lengths, preds, labels)
self.metric.update(num_infer_chunks.numpy(),
num_label_chunks.numpy(),
num_correct_chunks.numpy())
_, _, f1_score = map(float, self.metric.accumulate())
return token_level_probs, loss, {'f1_score': f1_score}
return token_level_probs
else:
sequence_output, pooled_output = result
return sequence_output, pooled_output
@staticmethod
def get_tokenizer(*args, **kwargs):
"""
Gets the tokenizer that is customized for this module.
"""
return ErnieTokenizer.from_pretrained(
pretrained_model_name_or_path='ernie-2.0-en', *args, **kwargs)
def run(args):
paddle.set_device(args.device)
if paddle.distributed.get_world_size() > 1:
paddle.distributed.init_parallel_env()
raw_datasets = load_dataset(args.task_name)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
train_ds = raw_datasets['train']
column_names = train_ds.column_names
label_list = train_ds.features['ner_tags'].feature.names
label_num = len(label_list)
batchify_fn = DataCollatorForTokenClassification(tokenizer=tokenizer)
# Define the model netword and its loss
model = AutoModelForTokenClassification.from_pretrained(
args.model_name_or_path, num_classes=label_num)
if paddle.distributed.get_world_size() > 1:
model = paddle.DataParallel(model)
def tokenize_and_align_labels(examples, no_entity_id=0):
tokenized_inputs = tokenizer(
examples['tokens'],
max_seq_len=args.max_seq_length,
# We use this argument because the texts in our dataset are lists of words (with a label for each word).
is_split_into_words=True,
return_length=True)
labels = []
for i, label in enumerate(examples['ner_tags']):
label_ids = label
if len(tokenized_inputs['input_ids'][i]) - 2 < len(label_ids):
label_ids = label_ids[:len(tokenized_inputs['input_ids'][i]) -
2]
label_ids = [no_entity_id] + label_ids + [no_entity_id]
label_ids += [no_entity_id] * (
len(tokenized_inputs['input_ids'][i]) - len(label_ids))
labels.append(label_ids)
tokenized_inputs["labels"] = labels
return tokenized_inputs
test_ds = raw_datasets['test']
test_ds = test_ds.select(range(len(test_ds) - 1))
test_ds = test_ds.map(tokenize_and_align_labels,
batched=True,
remove_columns=column_names)
test_data_loader = DataLoader(dataset=test_ds,
collate_fn=batchify_fn,
num_workers=0,
batch_size=args.batch_size,
return_list=True)
if args.do_train:
train_ds = train_ds.select(range(len(train_ds) - 1))
train_ds = train_ds.map(tokenize_and_align_labels,
batched=True,
remove_columns=column_names)
train_batch_sampler = paddle.io.DistributedBatchSampler(
train_ds, batch_size=args.batch_size, shuffle=True, drop_last=True)
train_data_loader = DataLoader(dataset=train_ds,
collate_fn=batchify_fn,
num_workers=0,
batch_sampler=train_batch_sampler,
return_list=True)
num_training_steps = args.max_steps if args.max_steps > 0 else len(
train_data_loader) * args.num_train_epochs
lr_scheduler = LinearDecayWithWarmup(args.learning_rate,
num_training_steps,
args.warmup_steps)
# Generate parameter names needed to perform weight decay.
# All bias and LayerNorm parameters are excluded.
decay_params = [
p.name for n, p in model.named_parameters()
if not any(nd in n for nd in ["bias", "norm"])
]
optimizer = paddle.optimizer.AdamW(
learning_rate=lr_scheduler,
epsilon=args.adam_epsilon,
parameters=model.parameters(),
weight_decay=args.weight_decay,
apply_decay_param_fun=lambda x: x in decay_params)
loss_fct = paddle.nn.loss.CrossEntropyLoss()
metric = ChunkEvaluator(label_list=label_list)
global_step = 0
best_f1 = 0.0
last_step = args.num_train_epochs * len(train_data_loader)
tic_train = time.time()
for epoch in range(args.num_train_epochs):
for step, batch in enumerate(train_data_loader):
global_step += 1
logits = model(batch['input_ids'], batch['token_type_ids'])
loss = loss_fct(logits, batch['labels'])
avg_loss = paddle.mean(loss)
if global_step % args.logging_steps == 0:
print(
"global step %d, epoch: %d, batch: %d, loss: %f, speed: %.2f step/s"
% (global_step, epoch, step, avg_loss,
args.logging_steps / (time.time() - tic_train)))
tic_train = time.time()
avg_loss.backward()
optimizer.step()
lr_scheduler.step()
optimizer.clear_grad()
if global_step % args.save_steps == 0 or global_step == num_training_steps:
if paddle.distributed.get_rank() == 0:
f1 = evaluate(model, loss_fct, metric,
test_data_loader, label_num, "test")
if f1 > best_f1:
best_f1 = f1
output_dir = args.output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Need better way to get inner model of DataParallel
model_to_save = model._layers if isinstance(
model, paddle.DataParallel) else model
model_to_save.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
if global_step >= num_training_steps:
print("best_f1: ", best_f1)
return
print("best_f1: ", best_f1)
if args.do_eval:
eval_data_loader = DataLoader(dataset=test_ds,
collate_fn=batchify_fn,
num_workers=0,
batch_size=args.batch_size,
return_list=True)
# Define the model netword and its loss
model = AutoModelForTokenClassification.from_pretrained(
args.model_name_or_path, num_classes=label_num)
loss_fct = paddle.nn.loss.CrossEntropyLoss()
metric = ChunkEvaluator(label_list=label_list)
model.eval()
metric.reset()
for step, batch in enumerate(eval_data_loader):
logits = model(batch["input_ids"], batch["token_type_ids"])
loss = loss_fct(logits, batch["labels"])
avg_loss = paddle.mean(loss)
preds = logits.argmax(axis=2)
num_infer_chunks, num_label_chunks, num_correct_chunks = metric.compute(
batch["length"], preds, batch["labels"])
metric.update(num_infer_chunks.numpy(), num_label_chunks.numpy(),
num_correct_chunks.numpy())
precision, recall, f1_score = metric.accumulate()
print("eval loss: %f, precision: %f, recall: %f, f1: %f" %
(avg_loss, precision, recall, f1_score))
def do_eval(args):
paddle.set_device(args.device)
# Create dataset, tokenizer and dataloader.
train_ds, eval_ds = load_dataset('msra_ner', split=('train', 'test'))
tokenizer = BertTokenizer.from_pretrained(args.model_name_or_path)
label_list = train_ds.features['ner_tags'].feature.names
label_num = len(label_list)
no_entity_id = 0
def tokenize_and_align_labels(examples):
tokenized_inputs = tokenizer(
examples['tokens'],
max_seq_len=args.max_seq_length,
# We use this argument because the texts in our dataset are lists of words (with a label for each word).
is_split_into_words=True,
return_length=True)
labels = []
for i, label in enumerate(examples['ner_tags']):
label_ids = label
if len(tokenized_inputs['input_ids'][i]) - 2 < len(label_ids):
label_ids = label_ids[:len(tokenized_inputs['input_ids'][i]) -
2]
label_ids = [no_entity_id] + label_ids + [no_entity_id]
label_ids += [no_entity_id] * (
len(tokenized_inputs['input_ids'][i]) - len(label_ids))
labels.append(label_ids)
tokenized_inputs["labels"] = labels
return tokenized_inputs
ignore_label = -100
batchify_fn = lambda samples, fn=Dict({
'input_ids':
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'), # input
'token_type_ids':
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'
), # segment
'seq_len':
Stack(dtype='int64'),
'labels':
Pad(axis=0, pad_val=ignore_label, dtype='int64') # label
}): fn(samples)
eval_ds = eval_ds.select(range(len(eval_ds) - 1))
eval_ds = eval_ds.map(tokenize_and_align_labels, batched=True)
eval_data_loader = DataLoader(dataset=eval_ds,
collate_fn=batchify_fn,
num_workers=0,
batch_size=args.batch_size,
return_list=True)
# Define the model netword and its loss
model = BertForTokenClassification.from_pretrained(args.model_name_or_path,
num_classes=label_num)
if args.init_checkpoint_path:
model_dict = paddle.load(args.init_checkpoint_path)
model.set_dict(model_dict)
loss_fct = paddle.nn.loss.CrossEntropyLoss(ignore_index=ignore_label)
metric = ChunkEvaluator(label_list=label_list)
model.eval()
metric.reset()
for step, batch in enumerate(eval_data_loader):
input_ids, token_type_ids, length, labels = batch
logits = model(input_ids, token_type_ids)
loss = loss_fct(logits, labels)
avg_loss = paddle.mean(loss)
preds = logits.argmax(axis=2)
num_infer_chunks, num_label_chunks, num_correct_chunks = metric.compute(
length, preds, labels)
metric.update(num_infer_chunks.numpy(), num_label_chunks.numpy(),
num_correct_chunks.numpy())
precision, recall, f1_score = metric.accumulate()
print("eval loss: %f, precision: %f, recall: %f, f1: %f" %
(avg_loss, precision, recall, f1_score))
def predict(self,
dataset,
tokenizer,
batchify_fn,
args,
dev_example=None,
dev_ds_ori=None):
if args.collect_shape:
self.set_dynamic_shape(args.max_seq_length, args.batch_size)
if args.task_name == "cmrc2018":
dataset_removed = dataset.remove_columns(
["offset_mapping", "attention_mask", "example_id"])
sample_num = len(dataset)
batches = []
for i in range(0, sample_num, args.batch_size):
batch_size = min(args.batch_size, sample_num - i)
batch = [dataset_removed[i + j] for j in range(batch_size)]
batches.append(batch)
else:
sample_num = len(dataset)
batches = []
for i in range(0, sample_num, args.batch_size):
batch_size = min(args.batch_size, sample_num - i)
batch = [dataset[i + j] for j in range(batch_size)]
batches.append(batch)
if args.perf:
for i, batch in enumerate(batches):
batch = batchify_fn(batch)
input_ids, segment_ids = batch["input_ids"].numpy(
), batch["token_type_ids"].numpy()
output = self.predict_batch([input_ids, segment_ids])
if i > args.perf_warmup_steps:
break
time1 = time.time()
nums = 0
for batch in batches:
batch = batchify_fn(batch)
input_ids, segment_ids = batch["input_ids"].numpy(
), batch["token_type_ids"].numpy()
nums = nums + input_ids.shape[0]
output = self.predict_batch([input_ids, segment_ids])
total_time = time.time() - time1
print("task name: %s, sample nums: %s, time: %s, QPS: %s " %
(args.task_name, nums, total_time, nums / total_time))
else:
if args.task_name == "msra_ner":
metric = ChunkEvaluator(label_list=args.label_list)
metric.reset()
all_predictions = []
batch_num = len(dataset['input_ids'])
for batch in batches:
batch = batchify_fn(batch)
input_ids, segment_ids = batch["input_ids"].numpy(
), batch["token_type_ids"].numpy()
output = self.predict_batch([input_ids, segment_ids])[0]
preds = np.argmax(output, axis=2)
all_predictions.append(preds.tolist())
num_infer_chunks, num_label_chunks, num_correct_chunks = metric.compute(
batch["seq_len"], paddle.to_tensor(preds),
batch["labels"])
metric.update(num_infer_chunks.numpy(),
num_label_chunks.numpy(),
num_correct_chunks.numpy())
res = metric.accumulate()
print("task name: %s, (precision, recall, f1): %s, " %
(args.task_name, res))
elif args.task_name == "cmrc2018":
all_start_logits = []
all_end_logits = []
for batch in batches:
batch = batchify_fn(batch)
input_ids, segment_ids = batch["input_ids"].numpy(
), batch["token_type_ids"].numpy()
start_logits, end_logits = self.predict_batch(
[input_ids, segment_ids])
for idx in range(start_logits.shape[0]):
if len(all_start_logits) % 1000 == 0 and len(
all_start_logits):
print("Processing example: %d" %
len(all_start_logits))
all_start_logits.append(start_logits[idx])
all_end_logits.append(end_logits[idx])
all_predictions, _, _ = compute_prediction(
dev_example, dataset, (all_start_logits, all_end_logits),
False, args.n_best_size, args.max_answer_length)
res = squad_evaluate(
examples=[raw_data for raw_data in dev_example],
preds=all_predictions,
is_whitespace_splited=False)
print("task name: %s, EM: %s, F1: %s" %
(args.task_name, res['exact'], res['f1']))
return all_predictions
else:
all_predictions = []
metric = METRIC_CLASSES[args.task_name]()
metric.reset()
for i, batch in enumerate(batches):
batch = batchify_fn(batch)
output = self.predict_batch([
batch["input_ids"].numpy(),
batch["token_type_ids"].numpy()
])[0]
preds = np.argmax(output, axis=1)
all_predictions.append(preds.tolist())
correct = metric.compute(paddle.to_tensor(output),
batch["labels"])
metric.update(correct)
res = metric.accumulate()
print("task name: %s, acc: %s, " % (args.task_name, res))
return all_predictions
