def create_model(pyreader_name, bert_config):
pyreader = fluid.layers.py_reader(
capacity=70,
shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, args.max_seq_len], [-1, 1], [-1, 1],
[-1, 1], [-1, 1]],
dtypes=[
'int64', 'int64', 'int64', 'float', 'int64', 'int64', 'int64',
'int64'
],
lod_levels=[0, 0, 0, 0, 0, 0, 0, 0],
name=pyreader_name,
use_double_buffer=True)
(src_ids, pos_ids, sent_ids, self_attn_mask, mask_label, mask_pos, labels,
next_sent_index) = fluid.layers.read_file(pyreader)
bert = BertModel(src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
self_attn_mask=self_attn_mask,
config=bert_config,
weight_sharing=args.weight_sharing,
use_fp16=args.use_fp16)
next_sent_acc, mask_lm_loss, total_loss = bert.get_pretraining_output(
mask_label, mask_pos, labels, next_sent_index)
if args.use_fp16 and args.loss_scaling > 1.0:
total_loss *= args.loss_scaling
return pyreader, next_sent_acc, mask_lm_loss, total_loss
def create_model_for_pretrain(args, vocab_size):
if args['vocab_size'] > 0:
vocab_size = args['vocab_size']
# 输入定义
src_ids = fluid.data(name='src_ids',
dtype='int64',
shape=[-1, args['max_seq_length'], 1])
pos_ids = fluid.data(name='pos_ids',
dtype='int64',
shape=[-1, args['max_seq_length'], 1])
sent_ids = fluid.data(name='sent_ids',
dtype='int64',
shape=[-1, args['max_seq_length'], 1])
input_mask = fluid.data(name='input_mask',
dtype='float32',
shape=[-1, args['max_seq_length'], 1])
mask_labels = fluid.data(name='mask_labels', dtype='int64', shape=[-1, 1])
mask_pos = fluid.data(name='mask_pos', dtype='int64', shape=[-1, 1])
otheranswer_labels = fluid.data(name='otheranswer_labels',
dtype='int64',
shape=[-1, 1])
# 根据任务的不同调整所需的数据，预测任务相比训练任务缺少label这一项数据
feed_list = [
src_ids, pos_ids, sent_ids, input_mask, mask_labels, mask_pos,
otheranswer_labels
]
reader = fluid.io.DataLoader.from_generator(feed_list=feed_list,
capacity=64,
iterable=True)
# 模型部分
# 由bert后接一层全连接完成预测任务
# bert部分
config = args
config['vocab_size'] = vocab_size
bert = BertModel(src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
input_mask=input_mask,
config=config,
use_fp16=False)
results = bert.get_pretraining_output(mask_labels, mask_pos,
otheranswer_labels)
qa_acc, mean_mask_lm_loss, loss = results
return reader, qa_acc, mean_mask_lm_loss, loss
def create_model(bert_config):
input_fields = {
'names': [
'src_ids', 'pos_ids', 'sent_ids', 'input_mask', 'mask_label',
'mask_pos', 'labels'
],
'shapes': [[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
[-1, 1], [-1, 1], [-1, 1]],
'dtypes':
['int64', 'int64', 'int64', 'float32', 'int64', 'int64', 'int64'],
'lod_levels': [0, 0, 0, 0, 0, 0, 0],
}
inputs = [
fluid.layers.data(name=input_fields['names'][i],
shape=input_fields['shapes'][i],
dtype=input_fields['dtypes'][i],
lod_level=input_fields['lod_levels'][i])
for i in range(len(input_fields['names']))
]
(src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos,
labels) = inputs
pyreader = fluid.io.PyReader(feed_list=inputs, capacity=50, iterable=False)
bert = BertModel(src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
input_mask=input_mask,
config=bert_config,
weight_sharing=args.weight_sharing,
use_fp16=args.use_fp16)
next_sent_acc, mask_lm_loss, total_loss = bert.get_pretraining_output(
mask_label, mask_pos, labels)
if args.use_fp16 and args.loss_scaling > 1.0:
total_loss *= args.loss_scaling
return pyreader, next_sent_acc, mask_lm_loss, total_loss
def create_model(bert_config):
input_fields = {
'names': [
'src_ids', 'pos_ids', 'sent_ids', 'input_mask', 'mask_label',
'mask_pos', 'labels'
],
'shapes': [[None, None], [None, None], [None, None], [None, None, 1],
[None, 1], [None, 1], [None, 1]],
'dtypes':
['int64', 'int64', 'int64', 'float32', 'int64', 'int64', 'int64'],
'lod_levels': [0, 0, 0, 0, 0, 0, 0],
}
inputs = [
fluid.data(name=input_fields['names'][i],
shape=input_fields['shapes'][i],
dtype=input_fields['dtypes'][i],
lod_level=input_fields['lod_levels'][i])
for i in range(len(input_fields['names']))
]
(src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos,
labels) = inputs
data_loader = fluid.io.DataLoader.from_generator(feed_list=inputs,
capacity=50,
iterable=False)
bert = BertModel(src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
input_mask=input_mask,
config=bert_config,
weight_sharing=args.weight_sharing,
use_fp16=args.use_fp16)
next_sent_acc, mask_lm_loss, total_loss = bert.get_pretraining_output(
mask_label, mask_pos, labels)
return data_loader, next_sent_acc, mask_lm_loss, total_loss
