def create_model_for_multi_task(args, vocab_size, is_prediction=False): # 处理词典大小 if args['vocab_size'] > 0: vocab_size = args['vocab_size'] # 输入定义 qas_ids = fluid.data(name='qas_ids', dtype='int64', shape=[-1, 1]) src_ids = fluid.data(name='src_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) pos_ids = fluid.data(name='pos_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) sent_ids = fluid.data(name='sent_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) input_mask = fluid.data(name='input_mask', dtype='float32', shape=[-1, args['max_seq_length'], 1]) labels = fluid.data(name='labels', dtype='int64', shape=[-1, 1]) labels_for_reverse = fluid.data(name='labels_for_reverse', dtype='int64', shape=[-1, 1]) # 根据任务的不同调整所需的数据,预测任务相比训练任务缺少label这一项数据 if is_prediction: feed_list = [qas_ids, src_ids, pos_ids, sent_ids, input_mask] else: feed_list = [ qas_ids, src_ids, pos_ids, sent_ids, input_mask, labels, labels_for_reverse ] reader = fluid.io.DataLoader.from_generator(feed_list=feed_list, capacity=64, iterable=True) # 模型部分 # 由bert后接一层全连接完成预测任务 # bert部分 config = args config['vocab_size'] = vocab_size bert = BertModel(src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=config, use_fp16=False, is_prediction=is_prediction) mrc_layer = config['mrc_layer'] freeze_pretrained_model = config['freeze_pretrained_model'] cls_feats, reverse_feats = bert.get_pooled_outputs() bert_encode = bert.get_sequence_output() if freeze_pretrained_model: cls_feats.stop_gradient = True bert_encode.stop_gradient = True logits = None if mrc_layer == "cls_fc": # 取[CLS]的输出经全连接进行预测 cls_feats = fluid.layers.dropout( x=cls_feats, dropout_prob=0.1, dropout_implementation="upscale_in_train", is_test=is_prediction) logits = fluid.layers.fc( input=cls_feats, size=args['num_labels'], param_attr=fluid.ParamAttr( name="cls_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="cls_out_b", initializer=fluid.initializer.Constant(0.))) logits_for_reverse = fluid.layers.fc( input=reverse_feats, size=2, param_attr=fluid.ParamAttr( name="reverse_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="reverse_out_b", initializer=fluid.initializer.Constant(0.))) elif mrc_layer == "capsNet": # 取完整的bert_output,输入胶囊网络 bert_output = bert_encode param_attr = fluid.ParamAttr( name='conv2d.weight', initializer=fluid.initializer.Xavier(uniform=False), learning_rate=0.001) bert_output = fluid.layers.unsqueeze(input=bert_output, axes=[1]) capsules = fluid.layers.conv2d(input=bert_output, num_filters=256, filter_size=32, stride=15, padding="VALID", act="relu", param_attr=param_attr) # (batch_size, 256, 33, 50) primaryCaps = CapsLayer(num_outputs=32, vec_len=8, with_routing=False, layer_type='CONV') caps1 = primaryCaps(capsules, kernel_size=9, stride=2) # (batch_size, 8736, 8, 1) classifierCaps = CapsLayer(num_outputs=args['num_labels'], vec_len=16, with_routing=True, layer_type='FC') caps2 = classifierCaps(caps1) # (batch_size, 3, 16, 1) epsilon = 1e-9 v_length = fluid.layers.sqrt( fluid.layers.reduce_sum( fluid.layers.square(caps2), -2, keep_dim=True) + epsilon) logits = fluid.layers.squeeze(v_length, axes=[2, 3]) elif mrc_layer == "lstm": hidden_size = 128 cell = fluid.layers.LSTMCell(hidden_size=hidden_size) cell_r = fluid.layers.LSTMCell(hidden_size=hidden_size) encoded = bert_encode[:, 1:, :] encoded = fluid.layers.dropout( x=encoded, dropout_prob=0.1, dropout_implementation="upscale_in_train") outputs = fluid.layers.rnn(cell, encoded)[0][:, -1, :] outputs_r = fluid.layers.rnn(cell_r, encoded, is_reverse=True)[0][:, -1, :] outputs = fluid.layers.concat(input=[outputs, outputs_r], axis=1) cls_feats = outputs cls_feats = fluid.layers.dropout( x=cls_feats, dropout_prob=0.1, dropout_implementation="upscale_in_train") # fc = fluid.layers.fc(input=cls_feats, size=hidden_size*2) # fc = fluid.layers.dropout( # x=fc, # dropout_prob=0.1, # dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=cls_feats, size=args['num_labels'], param_attr=fluid.ParamAttr( name="lstm_fc_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="lstm_fc_b", initializer=fluid.initializer.Constant(0.))) # 根据任务返回不同的结果 # 预测任务仅返回dataloader和预测出的每个label对应的概率 if is_prediction: probs = fluid.layers.softmax(logits) return reader, probs, qas_ids # 训练任务则计算loss ce_loss, probs = fluid.layers.softmax_with_cross_entropy( logits=logits, label=labels, return_softmax=True) loss = fluid.layers.mean(x=ce_loss) ce_loss_for_reverse, probs_for_reverse = fluid.layers.softmax_with_cross_entropy( logits=logits_for_reverse, label=labels_for_reverse, return_softmax=True) loss_for_reverse = fluid.layers.mean(x=ce_loss_for_reverse) if args['use_fp16'] and args.loss_scaling > 1.0: loss *= args.loss_scaling num_seqs = fluid.layers.create_tensor(dtype='int64') accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs) accuracy_for_reverse = fluid.layers.accuracy(input=probs_for_reverse, label=labels_for_reverse, total=num_seqs) # 返回dataloader,loss,预测结果,和准确度 return reader, loss + loss_for_reverse, probs, accuracy, accuracy_for_reverse, qas_ids
def create_model(pyreader_name, bert_config, is_training=False): if is_training: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, 1], [-1, 1]], dtypes=[ 'int64', 'int64', 'int64', 'float32', 'int64', 'int64'], lod_levels=[0, 0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, input_mask, start_positions, end_positions) = fluid.layers.read_file(pyreader) else: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, 1]], dtypes=['int64', 'int64', 'int64', 'float32', 'int64'], lod_levels=[0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, input_mask, unique_id) = fluid.layers.read_file(pyreader) bert = BertModel( src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=bert_config, use_fp16=args.use_fp16) enc_out = bert.get_sequence_output() logits = fluid.layers.fc( input=enc_out, size=2, num_flatten_dims=2, param_attr=fluid.ParamAttr( name="cls_squad_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.))) logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1]) start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0) batch_ones = fluid.layers.fill_constant_batch_size_like( input=start_logits, dtype='int64', shape=[1], value=1) num_seqs = fluid.layers.reduce_sum(input=batch_ones) if is_training: def compute_loss(logits, positions): loss = fluid.layers.softmax_with_cross_entropy( logits=logits, label=positions) loss = fluid.layers.mean(x=loss) return loss start_loss = compute_loss(start_logits, start_positions) end_loss = compute_loss(end_logits, end_positions) total_loss = (start_loss + end_loss) / 2.0 if args.use_fp16 and args.loss_scaling > 1.0: total_loss = total_loss * args.loss_scaling return pyreader, total_loss, num_seqs else: return pyreader, unique_id, start_logits, end_logits, num_seqs
def create_model(args, vocab_size, is_prediction=False, is_validate=False): """ 搭建分类模型 被训练模块和预测模块直接调用 返回相关的计算结果和对应的dataloader对象 :param args: 参数 :param vocab_size: 词典大小,用于构建词嵌入层。注意当参数设置词典大小时,该项无效 :param is_prediction: 是否是预测模式,将禁用dropout等。 :param is_validate: 是否是验证模式,除了禁用dropout,还将返回loss和acc,如果输入数据中没有对应项,则会报错。 :return: """ # 处理词典大小 if args['vocab_size'] > 0: vocab_size = args['vocab_size'] # 输入定义 qas_ids = fluid.data(name='qas_ids', dtype='int64', shape=[-1, 1]) src_ids = fluid.data(name='src_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) pos_ids = fluid.data(name='pos_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) sent_ids = fluid.data(name='sent_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) input_mask = fluid.data(name='input_mask', dtype='float32', shape=[-1, args['max_seq_length'], 1]) # 根据任务的不同调整所需的数据,预测任务相比训练任务缺少label这一项数据 labels = fluid.data(name='labels', dtype='int64', shape=[-1, 1]) # engineer_ids = fluid.data(name='engineer_ids', dtype='int64', shape=[-1, args['max_seq_length']+1, 1]) engineer_ids = fluid.data(name='engineer_ids', dtype='int64', shape=[-1, args['max_seq_length'], 1]) config = args if is_prediction: feed_list = [qas_ids, src_ids, pos_ids, sent_ids, input_mask] else: feed_list = [qas_ids, src_ids, pos_ids, sent_ids, input_mask, labels] if config['use_engineer']: feed_list.append(engineer_ids) reader = fluid.io.DataLoader.from_generator(feed_list=feed_list, capacity=64, iterable=True) # 模型部分 # 由bert后接一层全连接完成预测任务 # bert部分 config['vocab_size'] = vocab_size bert = BertModel(src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=config, use_fp16=False, is_prediction=(is_prediction or is_validate)) mrc_layer = config['mrc_layer'] freeze_pretrained_model = config['freeze_pretrained_model'] cls_feats = bert.get_pooled_output() bert_encode = bert.get_sequence_output() if freeze_pretrained_model: cls_feats.stop_gradient = True bert_encode.stop_gradient = True if config['use_engineer']: # entity_sim = engineer_ids[:,-1,:] # entity_sim_code = fluid.layers.one_hot(input=entity_sim, depth=2, allow_out_of_range=False) # engineer_emb = fluid.layers.embedding(input=engineer_ids[:,:-1,:], size=[32, 8]) engineer_emb = fluid.layers.embedding(input=engineer_ids, size=[32, 8]) bert_encode = fluid.layers.concat(input=[bert_encode, engineer_emb], axis=-1) logits = None if mrc_layer == "cls_fc": # 取[CLS]的输出经全连接进行预测 cls_feats = fluid.layers.dropout( x=cls_feats, dropout_prob=0.1, is_test=(is_prediction or is_validate), dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=cls_feats, size=args['num_labels'], param_attr=fluid.ParamAttr( name="cls_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="cls_out_b", initializer=fluid.initializer.Constant(0.))) elif mrc_layer == "capsNet": # 取完整的bert_output,输入胶囊网络 bert_output = bert_encode param_attr = fluid.ParamAttr( name='conv2d.weight', initializer=fluid.initializer.Xavier(uniform=False), learning_rate=0.001) bert_output = fluid.layers.unsqueeze(input=bert_output, axes=[1]) capsules = fluid.layers.conv2d(input=bert_output, num_filters=256, filter_size=32, stride=15, padding="VALID", act="relu", param_attr=param_attr) # (batch_size, 256, 33, 50) primaryCaps = CapsLayer(num_outputs=32, vec_len=8, with_routing=False, layer_type='CONV') caps1 = primaryCaps(capsules, kernel_size=9, stride=2) # (batch_size, 8736, 8, 1) classifierCaps = CapsLayer(num_outputs=args['num_labels'], vec_len=16, with_routing=True, layer_type='FC') caps2 = classifierCaps(caps1) # (batch_size, 3, 16, 1) epsilon = 1e-9 v_length = fluid.layers.sqrt( fluid.layers.reduce_sum( fluid.layers.square(caps2), -2, keep_dim=True) + epsilon) logits = fluid.layers.squeeze(v_length, axes=[2, 3]) elif mrc_layer == "lstm": hidden_size = args['lstm_hidden_size'] cell = fluid.layers.LSTMCell(hidden_size=hidden_size) cell_r = fluid.layers.LSTMCell(hidden_size=hidden_size) encoded = bert_encode[:, 1:, :] encoded = fluid.layers.dropout( x=encoded, is_test=(is_prediction or is_validate), dropout_prob=0.1, dropout_implementation="upscale_in_train") outputs = fluid.layers.rnn(cell, encoded)[0][:, -1, :] outputs_r = fluid.layers.rnn(cell_r, encoded, is_reverse=True)[0][:, -1, :] outputs = fluid.layers.concat(input=[outputs, outputs_r], axis=1) cls_feats = outputs cls_feats = fluid.layers.dropout( x=cls_feats, is_test=(is_prediction or is_validate), dropout_prob=0.1, dropout_implementation="upscale_in_train") # fc = fluid.layers.fc(input=cls_feats, size=hidden_size*2) # fc = fluid.layers.dropout( # x=fc, # dropout_prob=0.1, # dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=cls_feats, size=args['num_labels'], param_attr=fluid.ParamAttr( name="lstm_fc_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="lstm_fc_b", initializer=fluid.initializer.Constant(0.))) elif mrc_layer == "highway_lstm": hidden_size = 128 cell = fluid.layers.LSTMCell(hidden_size=hidden_size) cell_r = fluid.layers.LSTMCell(hidden_size=hidden_size) encoded = bert_encode[:, 1:, :] encoded = fluid.layers.dropout( x=encoded, is_test=(is_prediction or is_validate), dropout_prob=0.1, dropout_implementation="upscale_in_train") encoded = highway_layer(encoded, name="highway1", num_flatten_dims=2) encoded = fluid.layers.dropout( x=encoded, is_test=(is_prediction or is_validate), dropout_prob=0.1, dropout_implementation="upscale_in_train") outputs = fluid.layers.rnn(cell, encoded)[0][:, -1, :] outputs_r = fluid.layers.rnn(cell_r, encoded, is_reverse=True)[0][:, -1, :] outputs = fluid.layers.concat(input=[outputs, outputs_r], axis=1) cls_feats = outputs cls_feats = fluid.layers.dropout( x=cls_feats, is_test=(is_prediction or is_validate), dropout_prob=0.1, dropout_implementation="upscale_in_train") # fc = fluid.layers.fc(input=cls_feats, size=hidden_size*2) # fc = fluid.layers.dropout( # x=fc, # dropout_prob=0.1, # dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=cls_feats, size=args['num_labels'], param_attr=fluid.ParamAttr( name="lstm_fc_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="lstm_fc_b", initializer=fluid.initializer.Constant(0.))) # 根据任务返回不同的结果 # 预测任务仅返回dataloader和预测出的每个label对应的概率 if is_prediction and not is_validate: probs = fluid.layers.softmax(logits) return reader, probs, qas_ids # 训练任务则计算loss ce_loss, probs = fluid.layers.softmax_with_cross_entropy( logits=logits, label=labels, return_softmax=True) # loss = fluid.layers.mean(x=ce_loss) weight = fluid.layers.assign(np.array([[1.], [1.], [1.3]], dtype='float32')) def lossweighed(ce_loss, labels): one_hot = fluid.one_hot(input=labels, depth=args["num_labels"]) lw = fluid.layers.matmul(one_hot, weight) lw = fluid.layers.reduce_sum(lw, dim=1) loss = fluid.layers.elementwise_mul(lw, ce_loss) loss = fluid.layers.mean(loss) return loss loss = lossweighed(ce_loss, labels) if args['use_fp16'] and args.loss_scaling > 1.0: loss *= args.loss_scaling num_seqs = fluid.layers.create_tensor(dtype='int64') accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs) # 返回dataloader,loss,预测结果,和准确度 return reader, loss, probs, accuracy, qas_ids
def create_model(pyreader_name, bert_config, max_wn_concept_length, max_nell_concept_length, wn_concept_embedding_mat, nell_concept_embedding_mat, is_training=False, freeze=False): if is_training: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, max_wn_concept_length, 1], [-1, args.max_seq_len, max_nell_concept_length, 1], [-1, args.max_seq_len, 1], [-1, 1], [-1, 1]], dtypes=[ 'int64', 'int64', 'int64', 'int64', 'int64', 'float32', 'int64', 'int64' ], lod_levels=[0, 0, 0, 0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, wn_concept_ids, nell_concept_ids, input_mask, start_positions, end_positions) = fluid.layers.read_file(pyreader) else: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, max_wn_concept_length, 1], [-1, args.max_seq_len, max_nell_concept_length, 1], [-1, args.max_seq_len, 1], [-1, 1]], dtypes=[ 'int64', 'int64', 'int64', 'int64', 'int64', 'float32', 'int64' ], lod_levels=[0, 0, 0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, wn_concept_ids, nell_concept_ids, input_mask, unique_id) = fluid.layers.read_file(pyreader) '''1st Layer: BERT Layer''' bert = BertModel(src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=bert_config, use_fp16=args.use_fp16) enc_out = bert.get_sequence_output() if freeze: enc_out.stop_gradient = True logger.info("enc_out.stop_gradient: {}".format(enc_out.stop_gradient)) '''2nd layer: Memory Layer''' # get memory embedding wn_concept_vocab_size = wn_concept_embedding_mat.shape[0] wn_concept_dim = wn_concept_embedding_mat.shape[1] nell_concept_vocab_size = nell_concept_embedding_mat.shape[0] nell_concept_dim = nell_concept_embedding_mat.shape[1] wn_memory_embs = fluid.layers.embedding( wn_concept_ids, size=(wn_concept_vocab_size, wn_concept_dim), param_attr=fluid.ParamAttr(name="wn_concept_emb_mat", do_model_average=False, trainable=False), dtype='float32') nell_memory_embs = fluid.layers.embedding( nell_concept_ids, size=(nell_concept_vocab_size, nell_concept_dim), param_attr=fluid.ParamAttr(name="nell_concept_emb_mat", do_model_average=False, trainable=False), dtype='float32') # get memory length wn_concept_ids_reduced = fluid.layers.equal( wn_concept_ids, fluid.layers.fill_constant( shape=[1], value=0, dtype="int64")) # [batch_size, sent_size, concept_size, 1] wn_concept_ids_reduced = fluid.layers.cast( wn_concept_ids_reduced, dtype="float32") # [batch_size, sent_size, concept_size, 1] wn_concept_ids_reduced = fluid.layers.scale(fluid.layers.elementwise_sub( wn_concept_ids_reduced, fluid.layers.fill_constant([1], "float32", 1)), scale=-1) wn_mem_length = fluid.layers.reduce_sum( wn_concept_ids_reduced, dim=2) # [batch_size, sent_size, 1] nell_concept_ids_reduced = fluid.layers.equal( nell_concept_ids, fluid.layers.fill_constant( shape=[1], value=0, dtype="int64")) # [batch_size, sent_size, concept_size, 1] nell_concept_ids_reduced = fluid.layers.cast( nell_concept_ids_reduced, dtype="float32") # [batch_size, sent_size, concept_size, 1] nell_concept_ids_reduced = fluid.layers.scale(fluid.layers.elementwise_sub( nell_concept_ids_reduced, fluid.layers.fill_constant([1], "float32", 1)), scale=-1) nell_mem_length = fluid.layers.reduce_sum( nell_concept_ids_reduced, dim=2) # [batch_size, sent_size, 1] # select and integrate wn_memory_layer = MemoryLayer(bert_config, max_wn_concept_length, wn_concept_dim, mem_method='raw', prefix='wn') wn_memory_output = wn_memory_layer.forward(enc_out, wn_memory_embs, wn_mem_length, ignore_no_memory_token=True) nell_memory_layer = MemoryLayer(bert_config, max_nell_concept_length, nell_concept_dim, mem_method='raw', prefix='nell') nell_memory_output = nell_memory_layer.forward(enc_out, nell_memory_embs, nell_mem_length, ignore_no_memory_token=True) memory_output = fluid.layers.concat( [enc_out, wn_memory_output, nell_memory_output], axis=2) '''3rd layer: Self-Matching Layer''' # calculate input dim for self-matching layer memory_output_size = bert_config[ 'hidden_size'] + wn_concept_dim + nell_concept_dim logger.info("memory_output_size: {}".format(memory_output_size)) # do matching self_att_layer = TriLinearTwoTimeSelfAttentionLayer( memory_output_size, dropout_rate=0.0, cat_mul=True, cat_sub=True, cat_twotime=True, cat_twotime_mul=False, cat_twotime_sub=True) # [bs, sq, concat_hs] att_output = self_att_layer.forward(memory_output, input_mask) # [bs, sq, concat_hs] '''4th layer: Output Layer''' logits = fluid.layers.fc( input=att_output, size=2, num_flatten_dims=2, param_attr=fluid.ParamAttr( name="cls_squad_out_w", initializer=fluid.initializer.NormalInitializer( loc=0.0, scale=bert_config['initializer_range'])), bias_attr=fluid.ParamAttr(name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.))) logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1]) start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0) batch_ones = fluid.layers.fill_constant_batch_size_like(input=start_logits, dtype='int64', shape=[1], value=1) num_seqs = fluid.layers.reduce_sum(input=batch_ones) if is_training: def compute_loss(logits, positions): loss = fluid.layers.softmax_with_cross_entropy(logits=logits, label=positions) loss = fluid.layers.mean(x=loss) return loss start_loss = compute_loss(start_logits, start_positions) end_loss = compute_loss(end_logits, end_positions) total_loss = (start_loss + end_loss) / 2.0 if args.use_fp16 and args.loss_scaling > 1.0: total_loss = total_loss * args.loss_scaling return pyreader, total_loss, num_seqs else: return pyreader, unique_id, start_logits, end_logits, num_seqs
def create_model(bert_config, is_training=False): if is_training: input_fields = { 'names': [ 'src_ids', 'pos_ids', 'sent_ids', 'input_mask', 'start_positions', 'end_positions' ], 'shapes': [[None, None], [None, None], [None, None], [None, None, 1], [None, 1], [None, 1]], 'dtypes': ['int64', 'int64', 'int64', 'float32', 'int64', 'int64'], 'lod_levels': [0, 0, 0, 0, 0, 0], } else: input_fields = { 'names': ['src_ids', 'pos_ids', 'sent_ids', 'input_mask', 'unique_id'], 'shapes': [[None, None], [None, None], [None, None], [None, None, 1], [None, 1]], 'dtypes': ['int64', 'int64', 'int64', 'float32', 'int64'], 'lod_levels': [0, 0, 0, 0, 0], } inputs = [ fluid.data(name=input_fields['names'][i], shape=input_fields['shapes'][i], dtype=input_fields['dtypes'][i], lod_level=input_fields['lod_levels'][i]) for i in range(len(input_fields['names'])) ] data_loader = fluid.io.DataLoader.from_generator(feed_list=inputs, capacity=50, iterable=False) if is_training: (src_ids, pos_ids, sent_ids, input_mask, start_positions, end_positions) = inputs else: (src_ids, pos_ids, sent_ids, input_mask, unique_id) = inputs bert = BertModel(src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=bert_config, use_fp16=args.use_fp16) enc_out = bert.get_sequence_output() logits = fluid.layers.fc( input=enc_out, size=2, num_flatten_dims=2, param_attr=fluid.ParamAttr( name="cls_squad_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr(name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.))) logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1]) start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0) batch_ones = fluid.layers.fill_constant_batch_size_like(input=start_logits, dtype='int64', shape=[1], value=1) num_seqs = fluid.layers.reduce_sum(input=batch_ones) if is_training: def compute_loss(logits, positions): loss = fluid.layers.softmax_with_cross_entropy(logits=logits, label=positions) loss = fluid.layers.mean(x=loss) return loss start_loss = compute_loss(start_logits, start_positions) end_loss = compute_loss(end_logits, end_positions) total_loss = (start_loss + end_loss) / 2.0 return data_loader, total_loss, num_seqs else: return data_loader, unique_id, start_logits, end_logits, num_seqs
def create_model(pyreader_name, bert_config, is_training=False): if is_training: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, 1], [-1, 1], [-1, args.max_seq_len], [-1, args.max_seq_len], [-1, 1], [-1, args.max_seq_len]], dtypes=[ 'int64', 'int64', 'int64', 'float32', 'int64', 'int64', 'float32', 'float32', 'float32', 'float32' ], lod_levels=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, input_mask, start_positions, end_positions, KD_start_logits, KD_end_logits, la, loss_weights) = fluid.layers.read_file(pyreader) else: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, 1]], dtypes=['int64', 'int64', 'int64', 'float32', 'int64'], lod_levels=[0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, input_mask, unique_id) = fluid.layers.read_file(pyreader) bert = BertModel(src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=bert_config, use_fp16=args.use_fp16) enc_out = bert.get_sequence_output() logits = fluid.layers.fc( input=enc_out, size=2, num_flatten_dims=2, param_attr=fluid.ParamAttr( name="cls_squad_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr(name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.))) logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1]) start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0) batch_ones = fluid.layers.fill_constant_batch_size_like(input=start_logits, dtype='int64', shape=[1], value=1) num_seqs = fluid.layers.reduce_sum(input=batch_ones) if is_training: def compute_loss(logits, positions, loss_weights): logits = fluid.layers.softmax(logits) loss = fluid.layers.cross_entropy( input=logits, label=positions) * loss_weights[:, 0] loss = fluid.layers.mean(x=loss) return loss # KLloss_start = fluid.layers.kldiv_loss(x=start_logits, target=KD_start_logits, reduction='mean') # KLloss_end = fluid.layers.kldiv_loss(x=end_logits, target=KD_end_logits, reduction='mean') # KLloss = (KLloss_start + KLloss_end) / 2.0 KD_loss_mask = fluid.layers.cast(KD_start_logits < 999999999, 'int64') def diff_loss(batched_a, batched_b, KD_loss_mask, loss_weights): diff = batched_a - batched_b loss = diff * diff * KD_loss_mask * loss_weights loss = fluid.layers.reduce_sum(loss) / fluid.layers.reduce_sum( KD_loss_mask) return loss start_loss = compute_loss(start_logits, start_positions, loss_weights) end_loss = compute_loss(end_logits, end_positions, loss_weights) KDloss_start = diff_loss(start_logits, KD_start_logits, KD_loss_mask, loss_weights) KDloss_end = diff_loss(end_logits, KD_end_logits, KD_loss_mask, loss_weights) KDloss = (KDloss_start + KDloss_end) / 2.0 total_loss = (1 - la) * (start_loss + end_loss) / 2.0 + la * KDloss if args.use_fp16 and args.loss_scaling > 1.0: total_loss = total_loss * args.loss_scaling return pyreader, total_loss, num_seqs else: return pyreader, unique_id, start_logits, end_logits, num_seqs
def create_model(args, pyreader_name, bert_config, num_labels, is_prediction=False): """ define fine-tuning model """ if args.binary: pyreader = fluid.layers.py_reader( capacity=50, shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1], [-1, 1], [-1, 1]], dtypes=['int64', 'int64', 'int64', 'float32', 'int64', 'int64'], lod_levels=[0, 0, 0, 0, 0, 0], name=pyreader_name, use_double_buffer=True) (src_ids, pos_ids, sent_ids, input_mask, seq_len, labels) = fluid.layers.read_file(pyreader) bert = BertModel( src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, input_mask=input_mask, config=bert_config, use_fp16=args.use_fp16) if args.sub_model_type == 'raw': cls_feats = bert.get_pooled_output() elif args.sub_model_type == 'cnn': bert_seq_out = bert.get_sequence_output() bert_seq_out = fluid.layers.sequence_unpad(bert_seq_out, seq_len) cnn_hidden_size = 100 convs = [] for h in [3, 4, 5]: conv_feats = fluid.layers.sequence_conv( input=bert_seq_out, num_filters=cnn_hidden_size, filter_size=h) conv_feats = fluid.layers.batch_norm(input=conv_feats, act="relu") conv_feats = fluid.layers.sequence_pool( input=conv_feats, pool_type='max') convs.append(conv_feats) cls_feats = fluid.layers.concat(input=convs, axis=1) elif args.sub_model_type == 'gru': bert_seq_out = bert.get_sequence_output() bert_seq_out = fluid.layers.sequence_unpad(bert_seq_out, seq_len) gru_hidden_size = 1024 gru_input = fluid.layers.fc(input=bert_seq_out, size=gru_hidden_size * 3) gru_forward = fluid.layers.dynamic_gru( input=gru_input, size=gru_hidden_size, is_reverse=False) gru_backward = fluid.layers.dynamic_gru( input=gru_input, size=gru_hidden_size, is_reverse=True) gru_output = fluid.layers.concat([gru_forward, gru_backward], axis=1) cls_feats = fluid.layers.sequence_pool( input=gru_output, pool_type='max') elif args.sub_model_type == 'ffa': bert_seq_out = bert.get_sequence_output() attn = fluid.layers.fc(input=bert_seq_out, num_flatten_dims=2, size=1, act='tanh') attn = fluid.layers.softmax(attn) weighted_input = bert_seq_out * attn weighted_input = fluid.layers.sequence_unpad(weighted_input, seq_len) cls_feats = fluid.layers.sequence_pool(weighted_input, pool_type='sum') else: raise NotImplementedError("%s is not implemented!" % args.sub_model_type) cls_feats = fluid.layers.dropout( x=cls_feats, dropout_prob=0.1, dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=cls_feats, size=num_labels, param_attr=fluid.ParamAttr( name="cls_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)), bias_attr=fluid.ParamAttr( name="cls_out_b", initializer=fluid.initializer.Constant(0.))) probs = fluid.layers.softmax(logits) if is_prediction: feed_targets_name = [ src_ids.name, pos_ids.name, sent_ids.name, input_mask.name ] return pyreader, probs, feed_targets_name ce_loss = fluid.layers.softmax_with_cross_entropy( logits=logits, label=labels) loss = fluid.layers.mean(x=ce_loss) if args.use_fp16 and args.loss_scaling > 1.0: loss *= args.loss_scaling num_seqs = fluid.layers.create_tensor(dtype='int64') accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs) return (pyreader, loss, probs, accuracy, labels, num_seqs)