def test_lod_reset(self):
program = Program()
with program_guard(program):
x = layers.data(name='x', shape=[10], dtype='float32')
y = layers.data(
name='y', shape=[10, 20], dtype='float32', lod_level=2)
print(layers.lod_reset(x=x, y=y))
print(str(program))
def recv_func(message):
# feature of src and dst node on each edge
dst_feat = message['dst_node_feat']
src_feat = message['src_node_feat']
# feature of center node
x = L.sequence_pool(dst_feat, 'average')
# feature of neighbors of center node
z = L.sequence_pool(src_feat, 'average')
# compute gate
feat_gate = message['feat_gate']
g_max = L.sequence_pool(feat_gate, 'max')
g = L.concat([x, g_max, z], axis=1)
g = L.fc(g, heads, bias_attr=False, act="sigmoid")
# softmax
alpha = message['alpha']
alpha = paddle_helper.sequence_softmax(alpha) # E * M
feat_value = message['feat_value'] # E * (M * D2)
old = feat_value
feat_value = L.reshape(feat_value, [-1, heads, hidden_size_v]) # E * M * D2
feat_value = L.elementwise_mul(feat_value, alpha, axis=0)
feat_value = L.reshape(feat_value, [-1, heads*hidden_size_v]) # E * (M * D2)
feat_value = L.lod_reset(feat_value, old)
feat_value = L.sequence_pool(feat_value, 'sum') # N * (M * D2)
feat_value = L.reshape(feat_value, [-1, heads, hidden_size_v]) # N * M * D2
output = L.elementwise_mul(feat_value, g, axis=0)
output = L.reshape(output, [-1, heads * hidden_size_v]) # N * (M * D2)
output = L.concat([x, output], axis=1)
return output
def beam_search():
max_len = layers.fill_constant(
shape=[1], dtype=start_tokens.dtype, value=max_out_len)
step_idx = layers.fill_constant(
shape=[1], dtype=start_tokens.dtype, value=0)
cond = layers.less_than(x=step_idx, y=max_len)
while_op = layers.While(cond)
# array states will be stored for each step.
ids = layers.array_write(start_tokens, step_idx)
scores = layers.array_write(init_scores, step_idx)
# cell states will be overwrited at each step.
# caches contains states of history steps to reduce redundant
# computation in decoder.
caches = [{
"k": layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, 0, d_model],
dtype=enc_output.dtype,
value=0),
"v": layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, 0, d_model],
dtype=enc_output.dtype,
value=0)
} for i in range(n_layer)]
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_scores = layers.array_read(array=scores, i=step_idx)
# sequence_expand can gather sequences according to lod thus can be
# used in beam search to sift states corresponding to selected ids.
pre_src_attn_bias = layers.sequence_expand(
x=trg_src_attn_bias, y=pre_scores)
pre_enc_output = layers.sequence_expand(x=enc_output, y=pre_scores)
pre_caches = [{
"k": layers.sequence_expand(
x=cache["k"], y=pre_scores),
"v": layers.sequence_expand(
x=cache["v"], y=pre_scores),
} for cache in caches]
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_enc_output, # cann't use pre_ids here since it has lod
value=1,
shape=[-1, 1],
dtype=pre_ids.dtype),
y=layers.increment(
x=step_idx, value=1.0, in_place=False),
axis=0)
logits = wrap_decoder(
trg_vocab_size,
max_in_len,
n_layer,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
dropout_rate,
weight_sharing,
dec_inputs=(
pre_ids, pre_pos, None, pre_src_attn_bias, trg_data_shape,
slf_attn_pre_softmax_shape, slf_attn_post_softmax_shape,
src_attn_pre_softmax_shape, src_attn_post_softmax_shape),
enc_output=pre_enc_output,
caches=pre_caches)
topk_scores, topk_indices = layers.topk(
input=layers.softmax(logits), k=beam_size)
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores),
y=layers.reshape(
pre_scores, shape=[-1]),
axis=0)
# beam_search op uses lod to distinguish branches.
topk_indices = layers.lod_reset(topk_indices, pre_ids)
selected_ids, selected_scores = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=beam_size,
end_id=eos_idx)
layers.increment(x=step_idx, value=1.0, in_place=True)
# update states
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.assign(pre_src_attn_bias, trg_src_attn_bias)
layers.assign(pre_enc_output, enc_output)
for i in range(n_layer):
layers.assign(pre_caches[i]["k"], caches[i]["k"])
layers.assign(pre_caches[i]["v"], caches[i]["v"])
layers.assign(
layers.elementwise_add(
x=slf_attn_pre_softmax_shape,
y=attn_pre_softmax_shape_delta),
slf_attn_pre_softmax_shape)
layers.assign(
layers.elementwise_add(
x=slf_attn_post_softmax_shape,
y=attn_post_softmax_shape_delta),
slf_attn_post_softmax_shape)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=beam_size, end_id=eos_idx)
return finished_ids, finished_scores
def inference(self, model, inputs, outputs):
"""
Run inference.
Args:
inputs(dict): Its key is input name(str) and its value is a Variable.
model(object): A generate model. Need to implement `_generation_network` and `_calc_logits`.
Returns:
dict(str:Variable): Its key is output name(str) and its value is a Variable.
"""
# prepare while loop
max_len = layers.fill_constant(
shape=[1], dtype="int64", value=self.max_dec_len, force_cpu=True)
min_len = layers.fill_constant(
shape=[1], dtype="int64", value=self.min_dec_len, force_cpu=True)
step_idx = layers.fill_constant(
shape=[1], dtype="int64", value=0, force_cpu=True)
ids = layers.array_write(layers.reshape(inputs["tgt_ids"], (-1, 1)), step_idx)
pos_biases = layers.array_write(layers.reshape(inputs["tgt_pos"], (-1, 1)), step_idx)
scores = layers.array_write(inputs["init_score"], step_idx)
tgt_generation_mask = layers.array_write(inputs["tgt_generation_mask"], step_idx)
parent_idx = inputs["parent_idx"]
if self.decoding_strategy == "beam_search":
beam_size = self.beam_size
else:
beam_size = 1
eos_penalty = np.zeros(self.vocab_size, dtype="float32")
eos_penalty[self.eos_id] = -1e9
eos_penalty = layers.assign(eos_penalty)
token_penalty = np.zeros(self.vocab_size, dtype="float32")
token_penalty[self.unk_id] = -1e9
if self.mask_id >= 0:
token_penalty[self.mask_id] = -1e9
token_penalty = layers.assign(token_penalty)
# start while loop
cond = layers.less_than(x=step_idx, y=max_len)
while_op = layers.While(cond)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
pos_bias = layers.array_read(array=pos_biases, i=step_idx)
pos_bias = layers.gather(input=pos_bias, index=parent_idx)
tmp_tgt_generation_mask = layers.array_read(tgt_generation_mask, i=step_idx)
dtype = tmp_tgt_generation_mask.dtype
append_mask = layers.fill_constant_batch_size_like(
input=pre_ids,
value=1.0,
shape=[-1, 1, 1],
dtype=dtype)
tmp_tgt_generation_mask = layers.concat([tmp_tgt_generation_mask, append_mask], axis=2)
pre_mask = tmp_tgt_generation_mask = layers.gather(input=tmp_tgt_generation_mask, index=parent_idx)
pre_sent = layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype)
if self.continuous_position:
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype), y=step_idx, axis=0) + pos_bias
else:
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype), y=step_idx, axis=0)
if self.use_role:
pre_role = layers.fill_constant_batch_size_like(
input=pre_mask,
value=0,
shape=[-1, 1, 1],
dtype=pre_ids.dtype)
else:
pre_role = None
dec_out, _ = model._generation_network(
token_ids=pre_ids,
type_ids=pre_sent,
pos_ids=pre_pos,
role_ids=pre_role,
generation_mask=tmp_tgt_generation_mask,
gather_idx=parent_idx)
logits = model._calc_logits(dec_out)
# ignore unk and mask token
if self.ignore_unk:
logits = layers.elementwise_add(logits, token_penalty, axis=1)
# min dec length
min_len_cond = layers.less_than(x=step_idx, y=min_len)
def min_len_penalty():
"""Plus minimum length penalty."""
return layers.elementwise_add(logits, eos_penalty, axis=1)
def no_penalty():
"""No penalty."""
return logits
logits = layers.case([(min_len_cond, min_len_penalty)], default=no_penalty)
# get probs
probs = layers.softmax(logits / self.temperature)
if self.decoding_strategy == "beam_search":
topk_scores, topk_indices = layers.topk(
input=probs, k=beam_size)
else:
if self.decoding_strategy.startswith("sampling"):
sampling_ids = layers.sampling_id(probs, dtype="int")
elif self.decoding_strategy.startswith("topk_sampling"):
topk_probs, _ = layers.topk(input=probs, k=self.topk)
ge_cond = layers.cast(
layers.greater_equal(
probs,
layers.unsqueeze(topk_probs[:, -1], [1])),
"float32")
old_probs = probs
probs = probs * ge_cond / layers.reduce_sum(topk_probs, dim=-1, keep_dim=True)
sampling_ids = layers.sampling_id(probs, dtype="int")
probs = old_probs
else:
raise ValueError(self.decoding_strategy)
sampling_scores = layers.one_hot(
layers.unsqueeze(sampling_ids, [1]), probs.shape[1]
)
sampling_scores = sampling_scores * probs - (1 - sampling_scores) * 1e3
topk_scores, topk_indices = layers.topk(
input=sampling_scores, k=1)
pre_len = layers.cast(step_idx, "float32")
layers.increment(x=step_idx, value=1.0, in_place=True)
cur_len = layers.cast(step_idx, "float32")
# update scores
if self.length_average:
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores * pre_len, axis=0) / cur_len
elif self.length_penalty > 0:
pre_lp = layers.pow((5 + pre_len) / 6, self.length_penalty)
cur_lp = layers.pow((5 + cur_len) / 6, self.length_penalty)
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores * pre_lp, axis=0) / cur_lp
else:
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores, axis=0)
topk_indices = layers.lod_reset(topk_indices, pre_ids)
accu_scores = layers.lod_reset(accu_scores, pre_ids)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=beam_size,
end_id=self.eos_id,
return_parent_idx=True)
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.array_write(pre_mask, i=step_idx, array=tgt_generation_mask)
layers.array_write(pos_bias, i=step_idx, array=pos_biases)
layers.assign(gather_idx, parent_idx)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=beam_size, end_id=self.eos_id)
predictions = {
"finished_ids": finished_ids,
"finished_scores": finished_scores,
"token_ids": inputs["token_ids"],
"data_id": inputs["data_id"]
}
return predictions
def infilling_decode(self):
if self.task_type == "dialog":
emb_num = 4
else:
emb_num = 3
input_shapes = [[-1, self.max_seq_len, 1]] * emb_num + \
[[-1, self.max_seq_len, self.max_seq_len]]
input_dtypes = ['int64'] * emb_num + ['float32']
input_lod_levels = [0] * emb_num + [0]
shapes = input_shapes + [[-1, self.max_seq_len, 1],
[-1, self.max_seq_len, 1], [-1, 1], [-1],
[-1, 1, self.max_seq_len], [-1, 1]]
dtypes = input_dtypes + [
'int64', 'int64', 'float32', 'int32', 'float32', 'int64'
]
lod_levels = input_lod_levels + [2, 2, 2, 0, 0, 0]
inputs = self.to_ternsor(shapes, dtypes, lod_levels)
pyreader = fluid.io.DataLoader.from_generator(feed_list=inputs,
capacity=50,
iterable=False)
emb_ids = {}
for key, value in zip(self.emb_keys, inputs[:emb_num]):
emb_ids[key] = value
input_mask = inputs[emb_num]
tgt_ids, tgt_pos, init_scores, parent_idx, tgt_input_mask, data_ids = inputs[
-6:]
ernie = ErnieModel(emb_ids=emb_ids,
input_mask=input_mask,
config=self.ernie_config,
use_fp16=self.use_fp16,
task_type=self.task_type,
decoding=True,
gather_idx=parent_idx)
max_len = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=self.max_dec_len,
force_cpu=True)
step_idx = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=0,
force_cpu=True)
pos_idx = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=1,
force_cpu=True)
cond = layers.less_than(x=step_idx, y=max_len)
while_op = layers.While(cond)
ids = layers.array_write(layers.reshape(tgt_ids, (-1, 1)), step_idx)
pos_biases = layers.array_write(layers.reshape(tgt_pos, (-1, 1)),
step_idx)
scores = layers.array_write(init_scores, step_idx)
tgt_masks = layers.array_write(tgt_input_mask, step_idx)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
pos_bias = layers.array_read(array=pos_biases, i=step_idx)
pos_bias = layers.gather(input=pos_bias, index=parent_idx)
tmp_mask = layers.array_read(tgt_masks, i=step_idx)
def gen_batch_like(value,
dtype="int64",
shape=[-1, 1, 1],
is_scalar=True):
if is_scalar:
return layers.fill_constant_batch_size_like(
input=parent_idx,
value=value,
shape=shape,
dtype=dtype)
else:
return layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=parent_idx,
value=1,
shape=shape,
dtype=dtype),
y=value,
axis=0)
tmp_mask = layers.gather(input=tmp_mask, index=parent_idx)
append_0_mask = gen_batch_like(0.0, dtype=tmp_mask.dtype)
append_1_mask = gen_batch_like(1.0, dtype=tmp_mask.dtype)
tmp_mask = layers.concat([tmp_mask, append_1_mask], axis=2)
pre_mask = layers.concat([tmp_mask, append_0_mask], axis=2)
cur_mask = layers.concat([tmp_mask, append_1_mask], axis=2)
cur_ids = gen_batch_like(self.attn_id)
pre_pos = gen_batch_like(step_idx, is_scalar=False)
cur_pos = gen_batch_like(pos_idx, is_scalar=False)
if self.continuous_position:
pre_pos = pre_pos + pos_bias
cur_pos = cur_pos + pos_bias
dec_emb_ids = {
"word_embedding": layers.concat([pre_ids, cur_ids], axis=1),
"pos_embedding": layers.concat([pre_pos, cur_pos], axis=1)
}
if self.task_type == "dialog":
role_ids = gen_batch_like(0)
turn_ids = gen_batch_like(0)
dec_emb_ids["role_embedding"] = layers.concat(
[role_ids, role_ids], axis=1)
dec_emb_ids["turn_embedding"] = layers.concat(
[turn_ids, turn_ids], axis=1)
else:
sent_ids = gen_batch_like(self.tgt_type_id)
dec_emb_ids["sent_embedding"] = layers.concat(
[sent_ids, sent_ids], axis=1)
dec_mask = layers.concat([pre_mask, cur_mask], axis=1)
dec_out = ernie.encode(dec_emb_ids,
dec_mask,
parent_idx,
remove_query=True)
fc_out = self.cal_logit(dec_out[:, 1:, :], None)
topk_scores, topk_indices = layers.topk(
input=layers.softmax(fc_out), k=self.beam_size)
pre_lenpen = layers.pow(
(5.0 + layers.cast(step_idx, pre_scores.dtype)) / 6.0,
self.length_penalty)
cur_lenpen = layers.pow(
(5.0 + layers.cast(pos_idx, pre_scores.dtype)) / 6.0,
self.length_penalty)
accu_scores = layers.elementwise_add(x=layers.log(topk_scores),
y=pre_scores * pre_lenpen,
axis=0) / cur_lenpen
topk_indices = layers.lod_reset(topk_indices, pre_ids)
accu_scores = layers.lod_reset(accu_scores, pre_ids)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=self.beam_size,
end_id=self.eos_idx,
return_parent_idx=True)
layers.increment(x=step_idx, value=1.0, in_place=True)
layers.increment(x=pos_idx, value=1.0, in_place=True)
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.array_write(tmp_mask, i=step_idx, array=tgt_masks)
layers.array_write(pos_bias, i=step_idx, array=pos_biases)
layers.assign(gather_idx, parent_idx)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=self.beam_size, end_id=self.eos_idx)
graph_vars = {
"finished_ids": finished_ids,
"finished_scores": finished_scores,
"data_ids": data_ids
}
for k, v in graph_vars.items():
v.persistable = True
return pyreader, graph_vars
def beam_search():
max_len = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=max_out_len,
force_cpu=True)
step_idx = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=0,
force_cpu=True)
cond = layers.less_than(x=step_idx,
y=max_len) # default force_cpu=True
while_op = layers.While(cond)
# array states will be stored for each step.
ids = layers.array_write(layers.reshape(start_tokens, (-1, 1)),
step_idx)
scores = layers.array_write(init_scores, step_idx)
# cell states will be overwrited at each step.
# caches contains states of history steps in decoder self-attention
# and static encoder output projections in encoder-decoder attention
# to reduce redundant computation.
caches = [
{
"k": # for self attention
layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, n_head, 0, d_key],
dtype=enc_output.dtype,
value=0),
"v": # for self attention
layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, n_head, 0, d_value],
dtype=enc_output.dtype,
value=0),
"static_k": # for encoder-decoder attention
layers.create_tensor(dtype=enc_output.dtype),
"static_v": # for encoder-decoder attention
layers.create_tensor(dtype=enc_output.dtype)
} for i in range(n_layer)
]
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
# Since beam_search_op dosen't enforce pre_ids' shape, we can do
# inplace reshape here which actually change the shape of pre_ids.
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
# gather cell states corresponding to selected parent
pre_src_attn_bias = layers.gather(trg_src_attn_bias,
index=parent_idx)
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_src_attn_bias, # cann't use lod tensor here
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype),
y=step_idx,
axis=0)
logits = wrap_decoder(trg_vocab_size,
max_in_len,
n_layer,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
preprocess_cmd,
postprocess_cmd,
weight_sharing,
dec_inputs=(pre_ids, pre_pos, None,
pre_src_attn_bias),
enc_output=enc_output,
caches=caches,
gather_idx=parent_idx,
bos_idx=bos_idx)
# intra-beam topK
topk_scores, topk_indices = layers.topk(
input=layers.softmax(logits), k=beam_size)
accu_scores = layers.elementwise_add(x=layers.log(topk_scores),
y=pre_scores,
axis=0)
# beam_search op uses lod to differentiate branches.
accu_scores = layers.lod_reset(accu_scores, pre_ids)
# topK reduction across beams, also contain special handle of
# end beams and end sentences(batch reduction)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=beam_size,
end_id=eos_idx,
return_parent_idx=True)
layers.increment(x=step_idx, value=1.0, in_place=True)
# cell states(caches) have been updated in wrap_decoder,
# only need to update beam search states here.
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.assign(gather_idx, parent_idx)
layers.assign(pre_src_attn_bias, trg_src_attn_bias)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=beam_size, end_id=eos_idx)
return finished_ids, finished_scores
def beam_search():
"""Beam search function"""
max_len = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=self.max_out_len,
force_cpu=True)
min_len = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=self.min_out_len)
neg_inf = layers.fill_constant(shape=[1],
dtype='float32',
value=-INF)
step_idx = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=0,
force_cpu=True)
step_next_idx = layers.fill_constant(shape=[1],
dtype=start_tokens.dtype,
value=1,
force_cpu=True)
cond = layers.less_than(x=step_idx,
y=max_len) # default force_cpu=True
while_op = layers.While(cond)
# array states will be stored for each step.
ids = layers.array_write(layers.reshape(start_tokens, (-1, 1)),
step_idx)
scores = layers.array_write(init_scores, step_idx)
# cell states will be overwrited at each step.
# caches contains states of history steps in decoder self-attention
# and static encoder output projections in encoder-decoder attention
# to reduce redundant computation.
caches = [
{
"k": # for self attention
layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, self._n_head, 0, self._emb_size // self._n_head],
dtype=enc_words_output.dtype,
value=0),
"v": # for self attention
layers.fill_constant_batch_size_like(
input=start_tokens,
shape=[-1, self._n_head, 0, self._emb_size // self._n_head],
dtype=enc_words_output.dtype,
value=0),
"static_k_word": # for encoder-decoder attention
layers.create_tensor(dtype=enc_words_output.dtype),
"static_v_word": # for encoder-decoder attention
layers.create_tensor(dtype=enc_words_output.dtype),
"static_k_sent": # for encoder-decoder attention
layers.create_tensor(dtype=enc_sents_output.dtype),
"static_v_sent": # for encoder-decoder attention
layers.create_tensor(dtype=enc_sents_output.dtype)
} for i in range(self._dec_n_layer)
]
trigram_blocking = TrigramBlocking(start_tokens,
self.tokenizer,
use_fp16=self._use_fp16,
beam_size=self.beam_size)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
# Since beam_search_op dosen't enforce pre_ids' shape, we can do
# inplace reshape here which actually change the shape of pre_ids.
# pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
# gather cell states corresponding to selected parent
pre_src_words_attn_bias = layers.gather(
tgt_src_words_attn_bias, index=parent_idx)
pre_src_sents_attn_bias = layers.gather(
tgt_src_sents_attn_bias, index=parent_idx)
pre_graph_attn_bias = layers.gather(graph_attn_bias,
index=parent_idx)
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=
pre_src_sents_attn_bias, # cann't use lod tensor here
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype),
y=step_idx,
axis=0)
logits = self.decode(
dec_input=(pre_ids, pre_pos, None, pre_src_words_attn_bias,
pre_src_sents_attn_bias, pre_graph_attn_bias),
enc_words_output=enc_words_output,
enc_sents_output=enc_sents_output,
caches=caches,
gather_idx=parent_idx)
# prevent generating end token if length less than min_out_len
eos_index = layers.fill_constant(
shape=[layers.shape(logits)[0]],
dtype='int64',
value=self.eos_idx)
eos_index = fluid.one_hot(eos_index, depth=self.voc_size)
less_cond = layers.cast(layers.less_than(x=step_idx,
y=min_len),
dtype='float32')
less_val = layers.elementwise_mul(less_cond, neg_inf)
eos_val = layers.elementwise_mul(eos_index, less_val, axis=0)
revised_logits = layers.elementwise_add(logits,
eos_val,
axis=0)
# topK reduction across beams, also contain special handle of
# end beams and end sentences(batch reduction)
topk_scores, topk_indices = layers.topk(
input=layers.softmax(revised_logits), k=self.beam_size)
# Roll-Back previous-scores for length-penalty
# previous-scores has been length-penaltied, before this timestep length-penalty, need roll-back
# because of doing this, we need store the length-penaltied score in `scores`
# while calculating use the un-penaltied score
# -> safe for step_idx == 0 (initialization state), because previous-score == 0
pre_timestep_length_penalty = fluid.layers.pow(
((5.0 + fluid.layers.cast(step_idx, pre_scores.dtype)) /
6.0), self.len_penalty)
pre_scores_wo_len_penalty = fluid.layers.elementwise_mul(
pre_scores, pre_timestep_length_penalty)
# calc trigram-blocking delta scores for current alive sequence
if self.block_trigram:
trigram_blocking.update_seq(pre_ids, parent_idx)
trigram_blocking.expand_cand_seq(topk_indices)
fluid.layers.py_func(
func=trigram_blocking.blocking_forward,
x=[
trigram_blocking.cand_seq,
trigram_blocking.id2is_full_token
],
out=trigram_blocking.delta_score_out,
backward_func=None)
layers.Print(trigram_blocking.delta_score_out,
summarize=100,
message="trigram_blocking.delta_score_out")
pre_scores_wo_len_penalty = fluid.layers.elementwise_add(
x=trigram_blocking.delta_score_out,
y=pre_scores_wo_len_penalty,
axis=0)
# => [N, topk]
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores),
y=pre_scores_wo_len_penalty,
axis=0)
cur_timestep_length_penalty = layers.pow(
((5.0 + layers.cast(step_next_idx, accu_scores.dtype)) /
6.0), self.len_penalty)
curr_scores = layers.elementwise_div(
accu_scores, cur_timestep_length_penalty)
# beam_search op uses lod to differentiate branches.
curr_scores = layers.lod_reset(curr_scores, pre_ids)
topk_indices = layers.lod_reset(topk_indices, pre_ids)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=curr_scores,
beam_size=self.beam_size,
end_id=self.eos_idx,
return_parent_idx=True)
layers.increment(x=step_idx, value=1.0, in_place=True)
layers.increment(x=step_next_idx, value=1.0, in_place=True)
# cell states(caches) have been updated in wrap_decoder,
# only need to update beam search states here.
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.assign(gather_idx, parent_idx)
layers.assign(pre_src_words_attn_bias, tgt_src_words_attn_bias)
layers.assign(pre_src_sents_attn_bias, tgt_src_sents_attn_bias)
layers.assign(pre_graph_attn_bias, graph_attn_bias)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(
layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=self.beam_size, end_id=self.eos_idx)
return finished_ids, finished_scores
def rc_model(hidden_size, vocab, args):
"""This function build the whole BiDAF network"""
emb_shape = [vocab.size(), vocab.embed_dim]
start_labels = layers.data(name="start_lables",
shape=[1],
dtype='float32',
lod_level=1)
end_labels = layers.data(name="end_lables",
shape=[1],
dtype='float32',
lod_level=1)
# stage 1:setup input data, embedding table & encode
q_id0 = get_data('q_id0', 1, args)
q_ids = get_data('q_ids', 2, args)
p_ids_name = 'p_ids'
p_ids = get_data('p_ids', 2, args)
p_embs = embedding(p_ids, emb_shape, args)
q_embs = embedding(q_ids, emb_shape, args)
drnn = layers.DynamicRNN()
with drnn.block():
p_emb = drnn.step_input(p_embs)
q_emb = drnn.step_input(q_embs)
p_enc = encoder(p_emb, 'p_enc', hidden_size, args)
q_enc = encoder(q_emb, 'q_enc', hidden_size, args)
# stage 2:match
g_i = attn_flow(q_enc, p_enc, p_ids_name, args)
# stage 3:fusion
m_i = fusion(g_i, args)
drnn.output(m_i, q_enc)
ms, q_encs = drnn()
p_vec = layers.lod_reset(x=ms, y=start_labels)
q_vec = layers.lod_reset(x=q_encs, y=q_id0)
# stage 4:decode
start_probs, end_probs = point_network_decoder(p_vec=p_vec,
q_vec=q_vec,
hidden_size=hidden_size,
args=args)
# calculate model loss
cost0 = layers.sequence_pool(
layers.cross_entropy(input=start_probs,
label=start_labels,
soft_label=True), 'sum')
cost1 = layers.sequence_pool(
layers.cross_entropy(input=end_probs,
label=end_labels,
soft_label=True), 'sum')
cost0 = layers.mean(cost0)
cost1 = layers.mean(cost1)
cost = cost0 + cost1
cost.persistable = True
feeding_list = ["q_ids", "start_lables", "end_lables", "p_ids", "q_id0"]
return cost, start_probs, end_probs, ms, feeding_list
def topk_pool(gw, score, graph_id, ratio):
"""Implementation of topk pooling, where k means pooling ratio.
Args:
gw: Graph wrapper object.
score: The attention score of all nodes, which is used to select
important nodes.
graph_id: The graphs that the nodes belong to.
ratio: The pooling ratio of nodes we want to select.
Return:
perm: The index of nodes we choose.
ratio_length: The selected node numbers of each graph.
"""
graph_lod = gw.graph_lod
graph_nodes = gw.num_nodes
num_graph = gw.num_graph
num_nodes = L.ones(shape=[graph_nodes], dtype="float32")
num_nodes = L.lod_reset(num_nodes, graph_lod)
num_nodes_per_graph = L.sequence_pool(num_nodes, pool_type='sum')
max_num_nodes = L.reduce_max(num_nodes_per_graph, dim=0)
max_num_nodes = L.cast(max_num_nodes, dtype="int32")
index = L.arange(0, gw.num_nodes, dtype="int64")
offset = L.gather(graph_lod, graph_id, overwrite=False)
index = (index - offset) + (graph_id * max_num_nodes)
index.stop_gradient = True
# padding
dense_score = L.fill_constant(shape=[num_graph * max_num_nodes],
dtype="float32",
value=-999999)
index = L.reshape(index, shape=[-1])
dense_score = L.scatter(dense_score, index, updates=score)
num_graph = L.cast(num_graph, dtype="int32")
dense_score = L.reshape(dense_score, shape=[num_graph, max_num_nodes])
# record the sorted index
_, sort_index = L.argsort(dense_score, axis=-1, descending=True)
# recover the index range
graph_lod = graph_lod[:-1]
graph_lod = L.reshape(graph_lod, shape=[-1, 1])
graph_lod = L.cast(graph_lod, dtype="int64")
sort_index = L.elementwise_add(sort_index, graph_lod, axis=-1)
sort_index = L.reshape(sort_index, shape=[-1, 1])
# use sequence_slice to choose selected node index
pad_lod = L.arange(0, (num_graph + 1) * max_num_nodes,
step=max_num_nodes,
dtype="int32")
sort_index = L.lod_reset(sort_index, pad_lod)
ratio_length = L.ceil(num_nodes_per_graph * ratio)
ratio_length = L.cast(ratio_length, dtype="int64")
ratio_length = L.reshape(ratio_length, shape=[-1, 1])
offset = L.zeros(shape=[num_graph, 1], dtype="int64")
choose_index = L.sequence_slice(input=sort_index,
offset=offset,
length=ratio_length)
perm = L.reshape(choose_index, shape=[-1])
return perm, ratio_length
def gru_attention_infer(self, decoder_boot, max_length, char_num,
word_vector_dim, encoded_vector, encoded_proj,
decoder_size):
init_state = decoder_boot
beam_size = 1
array_len = layers.fill_constant(
shape=[1], dtype='int64', value=max_length)
counter = layers.zeros(shape=[1], dtype='int64', force_cpu=True)
# fill the first element with init_state
state_array = layers.create_array('float32')
layers.array_write(init_state, array=state_array, i=counter)
# ids, scores as memory
ids_array = layers.create_array('int64')
scores_array = layers.create_array('float32')
rois_shape = layers.shape(init_state)
batch_size = layers.slice(
rois_shape, axes=[0], starts=[0], ends=[1]) + 1
lod_level = layers.range(
start=0, end=batch_size, step=1, dtype=batch_size.dtype)
init_ids = layers.fill_constant_batch_size_like(
input=init_state, shape=[-1, 1], value=0, dtype='int64')
init_ids = layers.lod_reset(init_ids, lod_level)
init_ids = layers.lod_append(init_ids, lod_level)
init_scores = layers.fill_constant_batch_size_like(
input=init_state, shape=[-1, 1], value=1, dtype='float32')
init_scores = layers.lod_reset(init_scores, init_ids)
layers.array_write(init_ids, array=ids_array, i=counter)
layers.array_write(init_scores, array=scores_array, i=counter)
full_ids = fluid.layers.fill_constant_batch_size_like(
input=init_state, shape=[-1, 1], dtype='int64', value=1)
cond = layers.less_than(x=counter, y=array_len)
while_op = layers.While(cond=cond)
with while_op.block():
pre_ids = layers.array_read(array=ids_array, i=counter)
pre_state = layers.array_read(array=state_array, i=counter)
pre_score = layers.array_read(array=scores_array, i=counter)
pre_ids_emb = layers.embedding(
input=pre_ids,
size=[char_num, word_vector_dim],
dtype='float32')
context = self.simple_attention(encoded_vector, encoded_proj,
pre_state, decoder_size)
# expand the recursive_sequence_lengths of pre_state
# to be the same with pre_score
pre_state_expanded = layers.sequence_expand(pre_state, pre_score)
context_expanded = layers.sequence_expand(context, pre_score)
fc_1 = layers.fc(input=context_expanded,
size=decoder_size * 3,
bias_attr=False,
name="rnn_fc1")
fc_2 = layers.fc(input=pre_ids_emb,
size=decoder_size * 3,
bias_attr=False,
name="rnn_fc2")
decoder_inputs = fc_1 + fc_2
current_state, _, _ = layers.gru_unit(
input=decoder_inputs,
hidden=pre_state_expanded,
size=decoder_size * 3)
current_state_with_lod = layers.lod_reset(
x=current_state, y=pre_score)
# use score to do beam search
current_score = layers.fc(input=current_state_with_lod,
size=char_num,
bias_attr=True,
act='softmax',
name="rnn_out_fc")
topk_scores, topk_indices = layers.topk(current_score, k=beam_size)
new_ids = fluid.layers.concat([full_ids, topk_indices], axis=1)
fluid.layers.assign(new_ids, full_ids)
layers.increment(x=counter, value=1, in_place=True)
# update the memories
layers.array_write(current_state, array=state_array, i=counter)
layers.array_write(topk_indices, array=ids_array, i=counter)
layers.array_write(topk_scores, array=scores_array, i=counter)
# update the break condition:
# up to the max length or all candidates of
# source sentences have ended.
length_cond = layers.less_than(x=counter, y=array_len)
finish_cond = layers.logical_not(layers.is_empty(x=topk_indices))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
return full_ids
def graph_pooling(node_feat, graph_lod, pool_type='sum'):
"""graph pooling layers for nodes"""
node_feat = L.lod_reset(node_feat, graph_lod)
graph_feat = L.sequence_pool(node_feat, pool_type)
return graph_feat
def rc_model(hidden_size, vocab, args):
emb_shape = [vocab.size(), vocab.embed_dim]
start_labels = layers.data(
name="start_lables", shape=[1], dtype='float32', lod_level=1)
end_labels = layers.data(
name="end_lables", shape=[1], dtype='float32', lod_level=1)
vocab_size=52445
# stage 1:encode
q_id0 = get_data('q_id0', 1, args)
q_ids = get_data('q_ids', 2, args)
p_ids_name = 'p_ids'
p_ids = get_data('p_ids', 2, args)
q_ids_elmo = get_data('q_ids_elmo', 2, args)
p_ids_elmo = get_data('p_ids_elmo', 2, args)
#layers.Print(p_ids_elmo, message='p_ids_elmo', summarize=10)
#layers.Print(p_ids, message='p_ids', summarize=10)
#layers.Print(q_ids_elmo, message='q_ids_elmo', summarize=10)
#layers.Print(q_ids, message='q_ids', summarize=10)
p_embs = embedding(p_ids, emb_shape, args)
q_embs = embedding(q_ids, emb_shape, args)
if args.elmo==True:
q_embs_elmo = emb(q_ids_elmo)
p_embs_elmo = emb(p_ids_elmo)
drnn = layers.DynamicRNN()
with drnn.block():
p_emb = drnn.step_input(p_embs)
q_emb = drnn.step_input(q_embs)
if args.elmo==True:
q_emb_elmo = drnn.step_input(q_embs_elmo)
p_emb_elmo = drnn.step_input(p_embs_elmo)
p_encs_elmo= elmo_encoder(p_emb_elmo)
q_encs_elmo= elmo_encoder(q_emb_elmo)
#layers.Print(p_encs_elmo, message='p_encs_elmo', summarize=10)
#layers.Print(q_encs_elmo, message='q_encs_elmo', summarize=10)
#layers.Print(p_emb, message='p_emb', summarize=10)
p_emb=layers.concat(input=[p_emb, p_emb_elmo], axis=1)
q_emb=layers.concat(input=[q_emb, q_emb_elmo], axis=1)
p_enc = encoder(p_emb,'p_enc', hidden_size, args)
q_enc = encoder(q_emb, 'q_enc', hidden_size, args)
g_i = attn_flow(q_enc, p_enc, p_ids_name, args)
# stage 3:fusion
m_i = fusion(g_i, args)
drnn.output(m_i, q_enc)
ms, q_encs = drnn()
p_vec = layers.lod_reset(x=ms, y=start_labels)
q_vec = layers.lod_reset(x=q_encs, y=q_id0)
# stage 4:decode
start_probs, end_probs = point_network_decoder(
p_vec=p_vec, q_vec=q_vec, hidden_size=hidden_size, args=args)
cost0 = layers.sequence_pool(
layers.cross_entropy(
input=start_probs, label=start_labels, soft_label=True),
'sum')
cost1 = layers.sequence_pool(
layers.cross_entropy(
input=end_probs, label=end_labels, soft_label=True),
'sum')
cost0 = layers.mean(cost0)
cost1 = layers.mean(cost1)
cost = cost0 + cost1
cost.persistable = True
feeding_list=[]
if args.elmo==True:
feeding_list = ["q_ids", "start_lables", "end_lables", "p_ids", "q_id0","q_ids_elmo","p_ids_elmo"]
else:
feeding_list = ["q_ids", "start_lables", "end_lables", "p_ids", "q_id0"]
return cost, start_probs, end_probs, ms, feeding_list
def attn_flow(q_enc, p_enc, p_ids_name):
tag = p_ids_name + "::"
drnn = layers.DynamicRNN()
with drnn.block():
h_cur = drnn.step_input(p_enc)
u_all = drnn.static_input(q_enc)
h_expd = layers.sequence_expand(x=h_cur, y=u_all)
s_t_ = layers.elementwise_mul(x=u_all, y=h_expd, axis=0)
s_t1 = layers.reduce_sum(input=s_t_, dim=1)
s_t = layers.sequence_softmax(input=s_t1)
u_expr = layers.elementwise_mul(x=u_all, y=s_t, axis=0)
u_expr = layers.sequence_pool(input=u_expr, pool_type='sum')
if args.debug == True:
'''
layers.Print(h_expd, message='h_expd')
layers.Print(h_cur, message='h_cur')
layers.Print(u_all, message='u_all')
layers.Print(s_t, message='s_t')
layers.Print(s_t_, message='s_t_')
layers.Print(u_expr, message='u_expr')
'''
drnn.output(u_expr)
U_expr = drnn()
#'''
drnn2 = layers.DynamicRNN()
with drnn2.block():
h_cur = drnn2.step_input(p_enc)
u_all = drnn2.static_input(q_enc)
h_expd = layers.sequence_expand(x=h_cur, y=u_all)
s_t_ = layers.elementwise_mul(x=u_all, y=h_expd, axis=0)
s_t2 = layers.reduce_sum(input=s_t_, dim=1, keep_dim=True)
b_t = layers.sequence_pool(input=s_t2, pool_type='max')
if args.debug == True:
'''
layers.Print(s_t2, message='s_t2')
layers.Print(b_t, message='b_t')
'''
drnn2.output(b_t)
b = drnn2()
b_norm = layers.sequence_softmax(input=b)
h_expr = layers.elementwise_mul(x=p_enc, y=b_norm, axis=0)
h_expr = layers.sequence_pool(input=h_expr, pool_type='sum')
H_expr = layers.sequence_expand(x=h_expr, y=p_enc)
H_expr = layers.lod_reset(x=H_expr, y=p_enc)
h_u = layers.elementwise_mul(x=H_expr, y=U_expr, axis=0)
h_h = layers.elementwise_mul(x=H_expr, y=p_enc, axis=0)
g = layers.concat(input=[H_expr, U_expr, h_u, h_h], axis = 1)
#fusion
m = bi_lstm_encoder(input_seq=g, gate_size=embedding_dim)
if args.debug == True:
layers.Print(U_expr, message=tag + 'U_expr')
layers.Print(H_expr, message=tag + 'H_expr')
layers.Print(b, message=tag + 'b')
layers.Print(b_norm, message=tag + 'b_norm')
layers.Print(g, message=tag +'g')
layers.Print(m, message=tag + 'm')
layers.Print(h_h, message=tag + 'h_h')
layers.Print(q_enc, message=tag + 'q_enc')
layers.Print(p_enc, message=tag + 'p_enc')
return m, g
def decode(context, is_sparse):
init_state = context
array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)
counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
# fill the first element with init_state
state_array = pd.create_array('float32')
pd.array_write(init_state, array=state_array, i=counter)
# ids, scores as memory
ids_array = pd.create_array('int64')
scores_array = pd.create_array('float32')
init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2)
init_scores = pd.data(
name="init_scores", shape=[1], dtype="float32", lod_level=2)
pd.array_write(init_ids, array=ids_array, i=counter)
pd.array_write(init_scores, array=scores_array, i=counter)
cond = pd.less_than(x=counter, y=array_len)
while_op = pd.While(cond=cond)
with while_op.block():
pre_ids = pd.array_read(array=ids_array, i=counter)
pre_state = pd.array_read(array=state_array, i=counter)
pre_score = pd.array_read(array=scores_array, i=counter)
# expand the lod of pre_state to be the same with pre_score
pre_state_expanded = pd.sequence_expand(pre_state, pre_score)
pre_ids_emb = pd.embedding(
input=pre_ids,
size=[dict_size, word_dim],
dtype='float32',
is_sparse=is_sparse)
# use rnn unit to update rnn
current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb],
size=decoder_size,
act='tanh')
current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score)
# use score to do beam search
current_score = pd.fc(input=current_state_with_lod,
size=target_dict_dim,
act='softmax')
topk_scores, topk_indices = pd.topk(current_score, k=topk_size)
selected_ids, selected_scores = pd.beam_search(
pre_ids, topk_indices, topk_scores, beam_size, end_id=10, level=0)
pd.increment(x=counter, value=1, in_place=True)
# update the memories
pd.array_write(current_state, array=state_array, i=counter)
pd.array_write(selected_ids, array=ids_array, i=counter)
pd.array_write(selected_scores, array=scores_array, i=counter)
pd.less_than(x=counter, y=array_len, cond=cond)
translation_ids, translation_scores = pd.beam_search_decode(
ids=ids_array, scores=scores_array)
# return init_ids, init_scores
return translation_ids, translation_scores
def _do_beam_search(trg_vocab_size, max_in_len, n_layer, n_head, d_key,
d_value, d_model, d_inner_hid, prepostprocess_dropout,
attention_dropout, relu_dropout, preprocess_cmd,
postprocess_cmd, weight_sharing, beam_size, max_len,
bos_idx, eos_idx, ids, scores, parent_idx,
trg_src_attn_bias, caches, enc_output, step_idx):
"""
do beam search
"""
cond = layers.less_than(x=step_idx, y=max_len) # default force_cpu=True
while_op = layers.While(cond)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
# Since beam_search_op dosen't enforce pre_ids' shape, we can do
# inplace reshape here which actually change the shape of pre_ids.
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
# gather cell states corresponding to selected parent
pre_src_attn_bias = layers.gather(trg_src_attn_bias, index=parent_idx)
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_src_attn_bias, # cann't use lod tensor here
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype),
y=step_idx,
axis=0)
logits = wrap_decoder(trg_vocab_size,
max_in_len,
n_layer,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
preprocess_cmd,
postprocess_cmd,
weight_sharing,
dec_inputs=(pre_ids, pre_pos, None,
pre_src_attn_bias),
enc_output=enc_output,
caches=caches,
gather_idx=parent_idx,
bos_idx=bos_idx)
# intra-beam topK
topk_scores, topk_indices = layers.topk(input=layers.softmax(logits),
k=beam_size)
accu_scores = layers.elementwise_add(x=layers.log(topk_scores),
y=pre_scores,
axis=0)
# beam_search op uses lod to differentiate branches.
accu_scores = layers.lod_reset(accu_scores, pre_ids)
# topK reduction across beams, also contain special handle of
# end beams and end sentences(batch reduction)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=beam_size,
end_id=eos_idx,
return_parent_idx=True)
layers.increment(x=step_idx, value=1.0, in_place=True)
# cell states(caches) have been updated in wrap_decoder,
# only need to update beam search states here.
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.assign(gather_idx, parent_idx)
layers.assign(pre_src_attn_bias, trg_src_attn_bias)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
def decoder_decode(context, is_sparse):
init_state = context
array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)
counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
# fill the first element with init_state
state_array = pd.create_array('float32')
pd.array_write(init_state, array=state_array, i=counter)
# ids, scores as memory
ids_array = pd.create_array('int64')
scores_array = pd.create_array('float32')
init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2)
init_scores = pd.data(
name="init_scores", shape=[1], dtype="float32", lod_level=2)
pd.array_write(init_ids, array=ids_array, i=counter)
pd.array_write(init_scores, array=scores_array, i=counter)
cond = pd.less_than(x=counter, y=array_len)
while_op = pd.While(cond=cond)
with while_op.block():
pre_ids = pd.array_read(array=ids_array, i=counter)
pre_state = pd.array_read(array=state_array, i=counter)
pre_score = pd.array_read(array=scores_array, i=counter)
# expand the recursive_sequence_lengths of pre_state to be the same with pre_score
pre_state_expanded = pd.sequence_expand(pre_state, pre_score)
pre_ids_emb = pd.embedding(
input=pre_ids,
size=[dict_size, word_dim],
dtype='float32',
is_sparse=is_sparse)
# use rnn unit to update rnn
current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb],
size=decoder_size,
act='tanh')
current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score)
# use score to do beam search
current_score = pd.fc(input=current_state_with_lod,
size=target_dict_dim,
act='softmax')
topk_scores, topk_indices = pd.topk(current_score, k=beam_size)
# calculate accumulated scores after topk to reduce computation cost
accu_scores = pd.elementwise_add(
x=pd.log(topk_scores), y=pd.reshape(
pre_score, shape=[-1]), axis=0)
selected_ids, selected_scores = pd.beam_search(
pre_ids,
pre_score,
topk_indices,
accu_scores,
beam_size,
end_id=10,
level=0)
pd.increment(x=counter, value=1, in_place=True)
# update the memories
pd.array_write(current_state, array=state_array, i=counter)
pd.array_write(selected_ids, array=ids_array, i=counter)
pd.array_write(selected_scores, array=scores_array, i=counter)
# update the break condition: up to the max length or all candidates of
# source sentences have ended.
length_cond = pd.less_than(x=counter, y=array_len)
finish_cond = pd.logical_not(pd.is_empty(x=selected_ids))
pd.logical_and(x=length_cond, y=finish_cond, out=cond)
translation_ids, translation_scores = pd.beam_search_decode(
ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10)
# return init_ids, init_scores
return translation_ids, translation_scores
def decoder_decode(context, is_sparse):
init_state = context
array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)
counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
# fill the first element with init_state
state_array = pd.create_array('float32')
pd.array_write(init_state, array=state_array, i=counter)
# ids, scores as memory
ids_array = pd.create_array('int64')
scores_array = pd.create_array('float32')
init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2)
init_scores = pd.data(name="init_scores",
shape=[1],
dtype="float32",
lod_level=2)
pd.array_write(init_ids, array=ids_array, i=counter)
pd.array_write(init_scores, array=scores_array, i=counter)
cond = pd.less_than(x=counter, y=array_len)
while_op = pd.While(cond=cond)
with while_op.block():
pre_ids = pd.array_read(array=ids_array, i=counter)
pre_state = pd.array_read(array=state_array, i=counter)
pre_score = pd.array_read(array=scores_array, i=counter)
# expand the lod of pre_state to be the same with pre_score
pre_state_expanded = pd.sequence_expand(pre_state, pre_score)
pre_ids_emb = pd.embedding(input=pre_ids,
size=[dict_size, word_dim],
dtype='float32',
is_sparse=is_sparse)
# use rnn unit to update rnn
current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb],
size=decoder_size,
act='tanh')
current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score)
# use score to do beam search
current_score = pd.fc(input=current_state_with_lod,
size=target_dict_dim,
act='softmax')
topk_scores, topk_indices = pd.topk(current_score, k=50)
selected_ids, selected_scores = pd.beam_search(pre_ids,
topk_indices,
topk_scores,
beam_size,
end_id=10,
level=0)
pd.increment(x=counter, value=1, in_place=True)
# update the memories
pd.array_write(current_state, array=state_array, i=counter)
pd.array_write(selected_ids, array=ids_array, i=counter)
pd.array_write(selected_scores, array=scores_array, i=counter)
pd.less_than(x=counter, y=array_len, cond=cond)
translation_ids, translation_scores = pd.beam_search_decode(
ids=ids_array, scores=scores_array)
# return init_ids, init_scores
return translation_ids, translation_scores
def fast_decode(self):
"""create model for inference"""
if self.task_type == "dialog":
emb_num = 4
else:
emb_num = 3
input_shapes = [[-1, self.max_seq_len, 1]] * emb_num + \
[[-1, self.max_seq_len, self.max_seq_len]]
input_dtypes = ['int64'] * emb_num + ['float32']
input_lod_levels = [0] * emb_num + [0]
shapes = input_shapes + [[-1, 1, 1], [-1, 1, 1], [-1, 1], [-1],
[-1, 1, self.max_seq_len], [-1, 1]]
dtypes = input_dtypes + [
'int64', 'int64', 'float32', 'int32', 'float32', 'int64'
]
lod_levels = input_lod_levels + [2, 2, 2, 0, 0, 0]
inputs = self.to_tensor(shapes, dtypes, lod_levels)
pyreader = fluid.io.DataLoader.from_generator(feed_list=inputs,
capacity=70,
iterable=False)
emb_ids = {}
for key, value in zip(self.emb_keys, inputs[:emb_num]):
emb_ids[key] = value
input_mask = inputs[emb_num]
tgt_ids, tgt_pos, init_scores, parent_idx, tgt_input_mask, data_ids = inputs[
-6:]
unimo = UNIMOModel(emb_ids=emb_ids,
input_mask=input_mask,
config=self.gene_config,
task_type=self.task_type,
decoding=True,
gather_idx=parent_idx)
max_len = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=self.max_out_len,
force_cpu=True)
min_len = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=self.min_out_len,
force_cpu=True)
neg_inf = layers.fill_constant(shape=[1], dtype='float32', value=-1e18)
step_idx = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=0,
force_cpu=True)
step_next_idx = layers.fill_constant(shape=[1],
dtype=tgt_ids.dtype,
value=1,
force_cpu=True)
cond = layers.less_than(x=step_idx, y=max_len)
while_op = layers.While(cond)
ids = layers.array_write(layers.reshape(tgt_ids, (-1, 1)), step_idx)
pos_biases = layers.array_write(tgt_pos, step_idx)
scores = layers.array_write(init_scores, step_idx)
tgt_masks = layers.array_write(tgt_input_mask, step_idx)
trigram_blocking = TrigramBlocking(tgt_ids,
self.tokenizer,
beam_size=self.beam_size)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
pos_bias = layers.array_read(array=pos_biases, i=step_idx)
pos_bias = layers.gather(input=pos_bias, index=parent_idx)
def gen_batch_like(value,
dtype="int64",
shape=[-1, 1, 1],
is_scalar=True):
"""generate batch"""
if is_scalar:
return layers.fill_constant_batch_size_like(
input=parent_idx,
value=value,
shape=shape,
dtype=dtype)
else:
return layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=parent_idx,
value=1,
shape=shape,
dtype=dtype),
y=value,
axis=0)
tmp_mask = layers.array_read(tgt_masks, i=step_idx)
tmp_mask = layers.gather(input=tmp_mask, index=parent_idx)
append_1_mask = gen_batch_like(1.0, dtype=tmp_mask.dtype)
pre_mask = layers.concat([tmp_mask, append_1_mask], axis=2)
pre_pos = gen_batch_like(step_idx, is_scalar=False)
pre_pos = pre_pos + pos_bias ####################### pos start from 2
pre_sent = gen_batch_like(self.tgt_type_id, dtype=pre_ids.dtype)
dec_emb_ids = {"word_embedding": pre_ids, "pos_embedding": pre_pos}
if self.task_type == "dialog":
role_ids = gen_batch_like(0)
turn_ids = gen_batch_like(0)
dec_emb_ids["role_embedding"] = role_ids
dec_emb_ids["turn_embedding"] = turn_ids
else:
dec_emb_ids["sent_embedding"] = pre_sent
dec_out = unimo.encode(emb_ids=dec_emb_ids,
input_mask=pre_mask,
gather_idx=parent_idx)
fc_out = self.cal_logit(dec_out, None)
# prevent generating end token if length less than min_out_len
eos_index = layers.fill_constant(shape=[layers.shape(fc_out)[0]],
dtype='int64',
value=self.eos_id)
eos_index = fluid.one_hot(eos_index, depth=self.vocab_size)
less_cond = layers.cast(layers.less_than(x=step_idx, y=min_len),
dtype='float32')
less_val = layers.elementwise_mul(less_cond, neg_inf)
eos_val = layers.elementwise_mul(eos_index, less_val, axis=0)
revised_logits = layers.elementwise_add(fc_out, eos_val, axis=0)
# topK reduction across beams, also contain special handle of
# end beams and end sentences(batch reduction)
topk_scores, topk_indices = layers.topk(
input=layers.softmax(revised_logits), k=self.beam_size)
# Roll-Back previous-scores for length-penalty
# previous-scores has been length-penaltied, before this timestep length-penalty, need roll-back
# because of doing this, we need store the length-penaltied score in `scores`
# while calculating use the un-penaltied score
# -> safe for step_idx == 0 (initialization state), because previous-score == 0
pre_timestep_length_penalty = fluid.layers.pow(
((5.0 + fluid.layers.cast(step_idx, pre_scores.dtype)) / 6.0),
self.length_penalty)
pre_scores_wo_len_penalty = fluid.layers.elementwise_mul(
pre_scores, pre_timestep_length_penalty)
# calc trigram-blocking delta scores for current alive sequence
if self.block_trigram:
trigram_blocking.update_seq(pre_ids, parent_idx)
trigram_blocking.expand_cand_seq(topk_indices)
fluid.layers.py_func(func=trigram_blocking.blocking_forward,
x=[
trigram_blocking.cand_seq,
trigram_blocking.id2is_full_token
],
out=trigram_blocking.delta_score_out,
backward_func=None)
pre_scores_wo_len_penalty = fluid.layers.elementwise_add(
x=trigram_blocking.delta_score_out,
y=pre_scores_wo_len_penalty,
axis=0)
# => [N, topk]
accu_scores = layers.elementwise_add(x=layers.log(topk_scores),
y=pre_scores_wo_len_penalty,
axis=0)
cur_timestep_length_penalty = layers.pow(
((5.0 + layers.cast(step_next_idx, accu_scores.dtype)) / 6.0),
self.length_penalty)
curr_scores = layers.elementwise_div(accu_scores,
cur_timestep_length_penalty)
# beam_search op uses lod to differentiate branches.
curr_scores = layers.lod_reset(curr_scores, pre_ids)
topk_indices = layers.lod_reset(topk_indices, pre_ids)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=curr_scores,
beam_size=self.beam_size,
end_id=self.eos_id,
return_parent_idx=True)
layers.increment(x=step_idx, value=1.0, in_place=True)
layers.increment(x=step_next_idx, value=1.0, in_place=True)
# cell states(caches) have been updated in wrap_decoder,
# only need to update beam search states here.
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.array_write(pre_mask, i=step_idx, array=tgt_masks)
layers.array_write(pos_bias, i=step_idx, array=pos_biases)
layers.assign(gather_idx, parent_idx)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=self.beam_size, end_id=self.eos_id)
graph_vars = {
"finished_ids": finished_ids,
"finished_scores": finished_scores,
"data_ids": data_ids
}
for k, v in graph_vars.items():
v.persistable = True
return pyreader, graph_vars
