def test_error(self):
startup_program = Program()
main_program = Program()
use_cuda = core.is_compiled_with_cuda()
with program_guard(main_program, startup_program):
def fn_1(opt, avg_loss):
opt.minimize(avg_loss)
def fn_2(opt, avg_loss):
opt.minimize(avg_loss)
x = fluid.layers.data("X", [10], 'float32')
hidden = layers.fc(x, 5)
avg_loss = layers.mean(hidden)
adam = optimizer.Adam(learning_rate=LR)
sgd = optimizer.SGD(learning_rate=LR)
cond = layers.fill_constant([1], 'bool', True)
layers.case([(cond, lambda: fn_1(adam, avg_loss))],
lambda: fn_2(sgd, avg_loss))
cpu_place = fluid.CPUPlace()
cuda_place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
for place in [cpu_place, cuda_place]:
exe = fluid.Executor(place)
exe.run(startup_program)
np.random.seed(SEED)
# NOTE(liym27):
# This test needs to run in multi cards to test NotImplementedError.
# Here, move this test from RUN_TYPE=DIST in tests/unittests/CMakeList.txt,
# to use multi cards ** only on CPU ** not GPU to reduce CI time.
os.environ['CPU_NUM'] = str(2)
pe_exe = fluid.ParallelExecutor(use_cuda=use_cuda,
main_program=main_program,
loss_name=avg_loss.name)
num_devices = pe_exe.device_count
def not_implemented_error():
pe_exe.run(feed={
'X':
np.random.random(size=[64, 10]).astype('float32'),
},
fetch_list=[avg_loss.name])
if num_devices > 1:
self.assertRaises(NotImplementedError, not_implemented_error)
def test_return_single_var(self):
def fn_1():
return layers.fill_constant(shape=[4, 2], dtype='int32', value=1)
def fn_2():
return layers.fill_constant(shape=[4, 2], dtype='int32', value=2)
def fn_3():
return layers.fill_constant(shape=[4, 3], dtype='int32', value=3)
main_program = Program()
startup_program = Program()
with program_guard(main_program, startup_program):
x = layers.fill_constant(shape=[1], dtype='float32', value=0.3)
y = layers.fill_constant(shape=[1], dtype='float32', value=0.1)
z = layers.fill_constant(shape=[1], dtype='float32', value=0.2)
pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1
pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3
# call fn_1
out_0 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)],
default=fn_3)
# call fn_2
out_1 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)],
default=fn_3)
# call default fn_3
out_2 = layers.case(pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)),
default=fn_3)
# no default, call fn_2
out_3 = layers.case(pred_fn_pairs=[(pred_1, fn_2)])
# no default, call fn_2. but pred_2 is false
out_4 = layers.case(pred_fn_pairs=[(pred_2, fn_2)])
place = fluid.CUDAPlace(
0) if core.is_compiled_with_cuda() else fluid.CPUPlace()
exe = fluid.Executor(place)
res = exe.run(main_program,
fetch_list=[out_0, out_1, out_2, out_3, out_4])
self.assertTrue(np.allclose(res[0], 1))
self.assertTrue(np.allclose(res[1], 2))
self.assertTrue(np.allclose(res[2], 3))
self.assertTrue(np.allclose(res[3], 2))
self.assertTrue(np.allclose(res[4], 2))
def test_optimizer_in_case(self):
BATCH_SIZE = 1
INPUT_SIZE = 784
EPOCH_NUM = 2
x = fluid.data(name='x',
shape=[BATCH_SIZE, INPUT_SIZE],
dtype='float32')
y = fluid.data(name='y',
shape=[BATCH_SIZE, INPUT_SIZE],
dtype='float32')
switch_id = fluid.data(name='switch_id', shape=[1], dtype='int32')
one = layers.fill_constant(shape=[1], dtype='int32', value=1)
adam = optimizer.Adam(learning_rate=0.001)
adagrad = optimizer.Adagrad(learning_rate=0.001)
def fn_1():
sum = layers.elementwise_mul(x, y)
loss = layers.mean(sum, name="f_1_loss")
adam.minimize(loss)
def fn_2():
sum = layers.elementwise_mul(x, y)
loss = layers.mean(sum, name="f_2_loss")
adagrad.minimize(loss)
layers.case(pred_fn_pairs=[(switch_id == one, fn_1)], default=fn_2)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
for epoch in range(EPOCH_NUM):
np.random.seed(epoch)
feed_image = np.random.random(
size=[BATCH_SIZE, INPUT_SIZE]).astype('float32')
main_program = fluid.default_main_program()
out = exe.run(main_program,
feed={
'x': feed_image,
'y': feed_image,
'switch_id': np.array([epoch]).astype('int32')
},
fetch_list=[])
def fn_3():
var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5)
var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6)
out = layers.case(pred_fn_pairs=[
(var_5 < var_6, partial(fn_2, x=3)),
(var_5 == var_6,
partial(
layers.fill_constant, shape=[2], dtype='int32', value=7))
])
return out
def test_return_var_tuple(self):
def fn_1():
return layers.fill_constant(shape=[1, 2], dtype='int32',
value=1), layers.fill_constant(
shape=[2, 3],
dtype='float32',
value=2)
def fn_2():
return layers.fill_constant(shape=[3, 4], dtype='int32',
value=3), layers.fill_constant(
shape=[4, 5],
dtype='float32',
value=4)
def fn_3():
return layers.fill_constant(shape=[5], dtype='int32',
value=5), layers.fill_constant(
shape=[5, 6],
dtype='float32',
value=6)
main_program = Program()
startup_program = Program()
with program_guard(main_program, startup_program):
x = layers.fill_constant(shape=[1], dtype='float32', value=1)
y = layers.fill_constant(shape=[1], dtype='float32', value=1)
z = layers.fill_constant(shape=[1], dtype='float32', value=3)
pred_1 = layers.equal(x, y) # true
pred_2 = layers.equal(x, z) # false
out = layers.case(((pred_1, fn_1), (pred_2, fn_2)), fn_3)
place = fluid.CUDAPlace(
0) if core.is_compiled_with_cuda() else fluid.CPUPlace()
exe = fluid.Executor(place)
ret = exe.run(main_program, fetch_list=out)
self.assertTrue(
np.allclose(np.asarray(ret[0]), np.full((1, 2), 1, np.int32)))
self.assertTrue(
np.allclose(np.asarray(ret[1]), np.full((2, 3), 2,
np.float32)))
def inference(self, model, inputs, outputs):
"""
Run inference.
Args:
inputs(dict): Its key is input name(str) and its value is a Variable.
model(object): A generate model. Need to implement `_generation_network` and `_calc_logits`.
Returns:
dict(str:Variable): Its key is output name(str) and its value is a Variable.
"""
# prepare while loop
max_len = layers.fill_constant(
shape=[1], dtype="int64", value=self.max_dec_len, force_cpu=True)
min_len = layers.fill_constant(
shape=[1], dtype="int64", value=self.min_dec_len, force_cpu=True)
step_idx = layers.fill_constant(
shape=[1], dtype="int64", value=0, force_cpu=True)
ids = layers.array_write(layers.reshape(inputs["tgt_ids"], (-1, 1)), step_idx)
pos_biases = layers.array_write(layers.reshape(inputs["tgt_pos"], (-1, 1)), step_idx)
scores = layers.array_write(inputs["init_score"], step_idx)
tgt_generation_mask = layers.array_write(inputs["tgt_generation_mask"], step_idx)
parent_idx = inputs["parent_idx"]
if self.decoding_strategy == "beam_search":
beam_size = self.beam_size
else:
beam_size = 1
eos_penalty = np.zeros(self.vocab_size, dtype="float32")
eos_penalty[self.eos_id] = -1e9
eos_penalty = layers.assign(eos_penalty)
token_penalty = np.zeros(self.vocab_size, dtype="float32")
token_penalty[self.unk_id] = -1e9
if self.mask_id >= 0:
token_penalty[self.mask_id] = -1e9
token_penalty = layers.assign(token_penalty)
# start while loop
cond = layers.less_than(x=step_idx, y=max_len)
while_op = layers.While(cond)
with while_op.block():
pre_ids = layers.array_read(array=ids, i=step_idx)
pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True)
pre_scores = layers.array_read(array=scores, i=step_idx)
pos_bias = layers.array_read(array=pos_biases, i=step_idx)
pos_bias = layers.gather(input=pos_bias, index=parent_idx)
tmp_tgt_generation_mask = layers.array_read(tgt_generation_mask, i=step_idx)
dtype = tmp_tgt_generation_mask.dtype
append_mask = layers.fill_constant_batch_size_like(
input=pre_ids,
value=1.0,
shape=[-1, 1, 1],
dtype=dtype)
tmp_tgt_generation_mask = layers.concat([tmp_tgt_generation_mask, append_mask], axis=2)
pre_mask = tmp_tgt_generation_mask = layers.gather(input=tmp_tgt_generation_mask, index=parent_idx)
pre_sent = layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype)
if self.continuous_position:
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype), y=step_idx, axis=0) + pos_bias
else:
pre_pos = layers.elementwise_mul(
x=layers.fill_constant_batch_size_like(
input=pre_mask,
value=1,
shape=[-1, 1, 1],
dtype=pre_ids.dtype), y=step_idx, axis=0)
if self.use_role:
pre_role = layers.fill_constant_batch_size_like(
input=pre_mask,
value=0,
shape=[-1, 1, 1],
dtype=pre_ids.dtype)
else:
pre_role = None
dec_out, _ = model._generation_network(
token_ids=pre_ids,
type_ids=pre_sent,
pos_ids=pre_pos,
role_ids=pre_role,
generation_mask=tmp_tgt_generation_mask,
gather_idx=parent_idx)
logits = model._calc_logits(dec_out)
# ignore unk and mask token
if self.ignore_unk:
logits = layers.elementwise_add(logits, token_penalty, axis=1)
# min dec length
min_len_cond = layers.less_than(x=step_idx, y=min_len)
def min_len_penalty():
"""Plus minimum length penalty."""
return layers.elementwise_add(logits, eos_penalty, axis=1)
def no_penalty():
"""No penalty."""
return logits
logits = layers.case([(min_len_cond, min_len_penalty)], default=no_penalty)
# get probs
probs = layers.softmax(logits / self.temperature)
if self.decoding_strategy == "beam_search":
topk_scores, topk_indices = layers.topk(
input=probs, k=beam_size)
else:
if self.decoding_strategy.startswith("sampling"):
sampling_ids = layers.sampling_id(probs, dtype="int")
elif self.decoding_strategy.startswith("topk_sampling"):
topk_probs, _ = layers.topk(input=probs, k=self.topk)
ge_cond = layers.cast(
layers.greater_equal(
probs,
layers.unsqueeze(topk_probs[:, -1], [1])),
"float32")
old_probs = probs
probs = probs * ge_cond / layers.reduce_sum(topk_probs, dim=-1, keep_dim=True)
sampling_ids = layers.sampling_id(probs, dtype="int")
probs = old_probs
else:
raise ValueError(self.decoding_strategy)
sampling_scores = layers.one_hot(
layers.unsqueeze(sampling_ids, [1]), probs.shape[1]
)
sampling_scores = sampling_scores * probs - (1 - sampling_scores) * 1e3
topk_scores, topk_indices = layers.topk(
input=sampling_scores, k=1)
pre_len = layers.cast(step_idx, "float32")
layers.increment(x=step_idx, value=1.0, in_place=True)
cur_len = layers.cast(step_idx, "float32")
# update scores
if self.length_average:
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores * pre_len, axis=0) / cur_len
elif self.length_penalty > 0:
pre_lp = layers.pow((5 + pre_len) / 6, self.length_penalty)
cur_lp = layers.pow((5 + cur_len) / 6, self.length_penalty)
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores * pre_lp, axis=0) / cur_lp
else:
accu_scores = layers.elementwise_add(
x=layers.log(topk_scores), y=pre_scores, axis=0)
topk_indices = layers.lod_reset(topk_indices, pre_ids)
accu_scores = layers.lod_reset(accu_scores, pre_ids)
selected_ids, selected_scores, gather_idx = layers.beam_search(
pre_ids=pre_ids,
pre_scores=pre_scores,
ids=topk_indices,
scores=accu_scores,
beam_size=beam_size,
end_id=self.eos_id,
return_parent_idx=True)
layers.array_write(selected_ids, i=step_idx, array=ids)
layers.array_write(selected_scores, i=step_idx, array=scores)
layers.array_write(pre_mask, i=step_idx, array=tgt_generation_mask)
layers.array_write(pos_bias, i=step_idx, array=pos_biases)
layers.assign(gather_idx, parent_idx)
length_cond = layers.less_than(x=step_idx, y=max_len)
finish_cond = layers.logical_not(layers.is_empty(x=selected_ids))
layers.logical_and(x=length_cond, y=finish_cond, out=cond)
finished_ids, finished_scores = layers.beam_search_decode(
ids, scores, beam_size=beam_size, end_id=self.eos_id)
predictions = {
"finished_ids": finished_ids,
"finished_scores": finished_scores,
"token_ids": inputs["token_ids"],
"data_id": inputs["data_id"]
}
return predictions
def type_error_default():
layers.case(pred_fn_pairs=[(pred_1, fn_1)], default=fn_1())
def type_error_fn():
layers.case(pred_fn_pairs=[(pred_1, 2)], default=fn_1)
def type_error_pred():
layers.case(pred_fn_pairs=[(1, fn_1)], default=fn_1)
def type_error_pred_fn_1():
layers.case(pred_fn_pairs=[1], default=fn_1)
def test_nested_case(self):
def fn_1(x=1):
var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5)
var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6)
out = layers.case(pred_fn_pairs=[
(var_5 < var_6,
partial(
layers.fill_constant, shape=[1], dtype='int32', value=x)),
(var_5 == var_6,
partial(
layers.fill_constant, shape=[2], dtype='int32', value=x))
])
return out
def fn_2(x=2):
var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5)
var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6)
out = layers.case(pred_fn_pairs=[
(var_5 < var_6, partial(fn_1, x=x)),
(var_5 == var_6,
partial(
layers.fill_constant, shape=[2], dtype='int32', value=x))
])
return out
def fn_3():
var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5)
var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6)
out = layers.case(pred_fn_pairs=[
(var_5 < var_6, partial(fn_2, x=3)),
(var_5 == var_6,
partial(
layers.fill_constant, shape=[2], dtype='int32', value=7))
])
return out
main_program = Program()
startup_program = Program()
with program_guard(main_program, startup_program):
x = layers.fill_constant(shape=[1], dtype='float32', value=0.3)
y = layers.fill_constant(shape=[1], dtype='float32', value=0.1)
z = layers.fill_constant(shape=[1], dtype='float32', value=0.2)
pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1
pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3
out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)],
default=fn_3)
out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)],
default=fn_3)
out_3 = layers.case(pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)],
default=fn_3)
place = fluid.CUDAPlace(
0) if core.is_compiled_with_cuda() else fluid.CPUPlace()
exe = fluid.Executor(place)
res = exe.run(main_program, fetch_list=[out_1, out_2, out_3])
self.assertTrue(np.allclose(res[0], 1))
self.assertTrue(np.allclose(res[1], 2))
self.assertTrue(np.allclose(res[2], 3))
def static(train_data,
loss_in_switch=True,
use_cuda=False,
use_parallel_exe=False):
startup_program = Program()
main_program = Program()
startup_program.random_seed = SEED
main_program.random_seed = SEED
with program_guard(main_program, startup_program):
def double_fc_net(image):
hidden = layers.fc(
image,
size=FC_SIZE,
act='relu',
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=0.99)),
bias_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=0.5)),
name="hidden")
prediction = layers.fc(
hidden,
size=CLASS_NUM,
act='softmax',
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=1.2)),
bias_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=0.8)),
name="prediction")
return hidden, prediction
def fn_1(opt, avg_loss=None, pred=None, label=None):
if avg_loss is None:
loss = layers.cross_entropy(input=pred, label=label)
avg_loss = layers.mean(loss, name='mean_cross_entropy_loss')
opt.minimize(avg_loss)
return avg_loss
def fn_2(opt, avg_loss=None, pred=None, label=None):
if avg_loss is None:
loss = layers.softmax_with_cross_entropy(logits=pred,
label=label)
avg_loss = layers.mean(loss, name='mean_softmax_loss')
opt.minimize(avg_loss)
return avg_loss
image = fluid.data('image', [BATCH_SIZE, INPUT_SIZE], 'float32')
label = fluid.data('label', [BATCH_SIZE, 1], 'int64')
hidden, prediction = double_fc_net(image)
adam = optimizer.Adam(learning_rate=LR)
sgd = optimizer.SGD(learning_rate=LR)
id = fluid.data('id', [1], 'int32')
two = layers.fill_constant([1], 'int32', 2)
mod_two = layers.elementwise_mod(id, two) == 0
if loss_in_switch:
avg_loss = layers.case(
[(mod_two, lambda: fn_1(adam, None, prediction, label))],
lambda: fn_2(sgd, None, prediction, label))
else:
loss_1 = layers.cross_entropy(input=prediction, label=label)
avg_loss_1 = layers.mean(loss_1)
loss_2 = layers.softmax_with_cross_entropy(logits=prediction,
label=label)
avg_loss_2 = layers.mean(loss_2)
avg_loss = layers.case([(mod_two, lambda: fn_1(adam, avg_loss_1))],
lambda: fn_2(sgd, avg_loss_2))
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_program)
for epoch in range(EPOCH_NUM):
feed_image, feed_label = train_data[epoch]
fetch_list = [hidden, prediction, avg_loss]
feed = {
'image': feed_image,
'label': feed_label,
'id': np.array([epoch]).astype('int32')
}
out = exe.run(main_program, feed=feed, fetch_list=fetch_list)
out_hidden, out_pred, loss = out
return out_hidden, out_pred, loss
def fn_1():
return layers.fill_constant(shape=[1, 2], dtype='float32', value=1)
def fn_2():
return layers.fill_constant(shape=[2, 2], dtype='int32', value=2)
def fn_3():
return layers.fill_constant(shape=[3], dtype='int32', value=3)
main_program = fluid.default_main_program()
startup_program = fluid.default_startup_program()
with fluid.program_guard(main_program, startup_program):
x = layers.fill_constant(shape=[1], dtype='float32', value=0.3)
y = layers.fill_constant(shape=[1], dtype='float32', value=0.1)
z = layers.fill_constant(shape=[1], dtype='float32', value=0.2)
pred_1 = layers.less_than(z, x)
pred_2 = layers.less_than(x, y)
pred_3 = layers.equal(x, y)
out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)],
default=fn_3)
out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)])
exe = fluid.Executor(fluid.CPUPlace())
res_1, res_2 = exe.run(main_program, fetch_list=[out_1, out_2])
print(res_1)
print(res_2)
