def test_error(self): startup_program = Program() main_program = Program() use_cuda = core.is_compiled_with_cuda() with program_guard(main_program, startup_program): def fn_1(opt, avg_loss): opt.minimize(avg_loss) def fn_2(opt, avg_loss): opt.minimize(avg_loss) x = fluid.layers.data("X", [10], 'float32') hidden = layers.fc(x, 5) avg_loss = layers.mean(hidden) adam = optimizer.Adam(learning_rate=LR) sgd = optimizer.SGD(learning_rate=LR) cond = layers.fill_constant([1], 'bool', True) layers.case([(cond, lambda: fn_1(adam, avg_loss))], lambda: fn_2(sgd, avg_loss)) cpu_place = fluid.CPUPlace() cuda_place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() for place in [cpu_place, cuda_place]: exe = fluid.Executor(place) exe.run(startup_program) np.random.seed(SEED) # NOTE(liym27): # This test needs to run in multi cards to test NotImplementedError. # Here, move this test from RUN_TYPE=DIST in tests/unittests/CMakeList.txt, # to use multi cards ** only on CPU ** not GPU to reduce CI time. os.environ['CPU_NUM'] = str(2) pe_exe = fluid.ParallelExecutor(use_cuda=use_cuda, main_program=main_program, loss_name=avg_loss.name) num_devices = pe_exe.device_count def not_implemented_error(): pe_exe.run(feed={ 'X': np.random.random(size=[64, 10]).astype('float32'), }, fetch_list=[avg_loss.name]) if num_devices > 1: self.assertRaises(NotImplementedError, not_implemented_error)
def test_return_single_var(self): def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) def fn_2(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=2) def fn_3(): return layers.fill_constant(shape=[4, 3], dtype='int32', value=3) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): x = layers.fill_constant(shape=[1], dtype='float32', value=0.3) y = layers.fill_constant(shape=[1], dtype='float32', value=0.1) z = layers.fill_constant(shape=[1], dtype='float32', value=0.2) pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 # call fn_1 out_0 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)], default=fn_3) # call fn_2 out_1 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3) # call default fn_3 out_2 = layers.case(pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)), default=fn_3) # no default, call fn_2 out_3 = layers.case(pred_fn_pairs=[(pred_1, fn_2)]) # no default, call fn_2. but pred_2 is false out_4 = layers.case(pred_fn_pairs=[(pred_2, fn_2)]) place = fluid.CUDAPlace( 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=[out_0, out_1, out_2, out_3, out_4]) self.assertTrue(np.allclose(res[0], 1)) self.assertTrue(np.allclose(res[1], 2)) self.assertTrue(np.allclose(res[2], 3)) self.assertTrue(np.allclose(res[3], 2)) self.assertTrue(np.allclose(res[4], 2))
def test_optimizer_in_case(self): BATCH_SIZE = 1 INPUT_SIZE = 784 EPOCH_NUM = 2 x = fluid.data(name='x', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32') y = fluid.data(name='y', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32') switch_id = fluid.data(name='switch_id', shape=[1], dtype='int32') one = layers.fill_constant(shape=[1], dtype='int32', value=1) adam = optimizer.Adam(learning_rate=0.001) adagrad = optimizer.Adagrad(learning_rate=0.001) def fn_1(): sum = layers.elementwise_mul(x, y) loss = layers.mean(sum, name="f_1_loss") adam.minimize(loss) def fn_2(): sum = layers.elementwise_mul(x, y) loss = layers.mean(sum, name="f_2_loss") adagrad.minimize(loss) layers.case(pred_fn_pairs=[(switch_id == one, fn_1)], default=fn_2) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) for epoch in range(EPOCH_NUM): np.random.seed(epoch) feed_image = np.random.random( size=[BATCH_SIZE, INPUT_SIZE]).astype('float32') main_program = fluid.default_main_program() out = exe.run(main_program, feed={ 'x': feed_image, 'y': feed_image, 'switch_id': np.array([epoch]).astype('int32') }, fetch_list=[])
def fn_3(): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) out = layers.case(pred_fn_pairs=[ (var_5 < var_6, partial(fn_2, x=3)), (var_5 == var_6, partial( layers.fill_constant, shape=[2], dtype='int32', value=7)) ]) return out
def test_return_var_tuple(self): def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='int32', value=1), layers.fill_constant( shape=[2, 3], dtype='float32', value=2) def fn_2(): return layers.fill_constant(shape=[3, 4], dtype='int32', value=3), layers.fill_constant( shape=[4, 5], dtype='float32', value=4) def fn_3(): return layers.fill_constant(shape=[5], dtype='int32', value=5), layers.fill_constant( shape=[5, 6], dtype='float32', value=6) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): x = layers.fill_constant(shape=[1], dtype='float32', value=1) y = layers.fill_constant(shape=[1], dtype='float32', value=1) z = layers.fill_constant(shape=[1], dtype='float32', value=3) pred_1 = layers.equal(x, y) # true pred_2 = layers.equal(x, z) # false out = layers.case(((pred_1, fn_1), (pred_2, fn_2)), fn_3) place = fluid.CUDAPlace( 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) self.assertTrue( np.allclose(np.asarray(ret[0]), np.full((1, 2), 1, np.int32))) self.assertTrue( np.allclose(np.asarray(ret[1]), np.full((2, 3), 2, np.float32)))
def inference(self, model, inputs, outputs): """ Run inference. Args: inputs(dict): Its key is input name(str) and its value is a Variable. model(object): A generate model. Need to implement `_generation_network` and `_calc_logits`. Returns: dict(str:Variable): Its key is output name(str) and its value is a Variable. """ # prepare while loop max_len = layers.fill_constant( shape=[1], dtype="int64", value=self.max_dec_len, force_cpu=True) min_len = layers.fill_constant( shape=[1], dtype="int64", value=self.min_dec_len, force_cpu=True) step_idx = layers.fill_constant( shape=[1], dtype="int64", value=0, force_cpu=True) ids = layers.array_write(layers.reshape(inputs["tgt_ids"], (-1, 1)), step_idx) pos_biases = layers.array_write(layers.reshape(inputs["tgt_pos"], (-1, 1)), step_idx) scores = layers.array_write(inputs["init_score"], step_idx) tgt_generation_mask = layers.array_write(inputs["tgt_generation_mask"], step_idx) parent_idx = inputs["parent_idx"] if self.decoding_strategy == "beam_search": beam_size = self.beam_size else: beam_size = 1 eos_penalty = np.zeros(self.vocab_size, dtype="float32") eos_penalty[self.eos_id] = -1e9 eos_penalty = layers.assign(eos_penalty) token_penalty = np.zeros(self.vocab_size, dtype="float32") token_penalty[self.unk_id] = -1e9 if self.mask_id >= 0: token_penalty[self.mask_id] = -1e9 token_penalty = layers.assign(token_penalty) # start while loop cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond) with while_op.block(): pre_ids = layers.array_read(array=ids, i=step_idx) pre_ids = layers.reshape(pre_ids, (-1, 1, 1), inplace=True) pre_scores = layers.array_read(array=scores, i=step_idx) pos_bias = layers.array_read(array=pos_biases, i=step_idx) pos_bias = layers.gather(input=pos_bias, index=parent_idx) tmp_tgt_generation_mask = layers.array_read(tgt_generation_mask, i=step_idx) dtype = tmp_tgt_generation_mask.dtype append_mask = layers.fill_constant_batch_size_like( input=pre_ids, value=1.0, shape=[-1, 1, 1], dtype=dtype) tmp_tgt_generation_mask = layers.concat([tmp_tgt_generation_mask, append_mask], axis=2) pre_mask = tmp_tgt_generation_mask = layers.gather(input=tmp_tgt_generation_mask, index=parent_idx) pre_sent = layers.fill_constant_batch_size_like( input=pre_mask, value=1, shape=[-1, 1, 1], dtype=pre_ids.dtype) if self.continuous_position: pre_pos = layers.elementwise_mul( x=layers.fill_constant_batch_size_like( input=pre_mask, value=1, shape=[-1, 1, 1], dtype=pre_ids.dtype), y=step_idx, axis=0) + pos_bias else: pre_pos = layers.elementwise_mul( x=layers.fill_constant_batch_size_like( input=pre_mask, value=1, shape=[-1, 1, 1], dtype=pre_ids.dtype), y=step_idx, axis=0) if self.use_role: pre_role = layers.fill_constant_batch_size_like( input=pre_mask, value=0, shape=[-1, 1, 1], dtype=pre_ids.dtype) else: pre_role = None dec_out, _ = model._generation_network( token_ids=pre_ids, type_ids=pre_sent, pos_ids=pre_pos, role_ids=pre_role, generation_mask=tmp_tgt_generation_mask, gather_idx=parent_idx) logits = model._calc_logits(dec_out) # ignore unk and mask token if self.ignore_unk: logits = layers.elementwise_add(logits, token_penalty, axis=1) # min dec length min_len_cond = layers.less_than(x=step_idx, y=min_len) def min_len_penalty(): """Plus minimum length penalty.""" return layers.elementwise_add(logits, eos_penalty, axis=1) def no_penalty(): """No penalty.""" return logits logits = layers.case([(min_len_cond, min_len_penalty)], default=no_penalty) # get probs probs = layers.softmax(logits / self.temperature) if self.decoding_strategy == "beam_search": topk_scores, topk_indices = layers.topk( input=probs, k=beam_size) else: if self.decoding_strategy.startswith("sampling"): sampling_ids = layers.sampling_id(probs, dtype="int") elif self.decoding_strategy.startswith("topk_sampling"): topk_probs, _ = layers.topk(input=probs, k=self.topk) ge_cond = layers.cast( layers.greater_equal( probs, layers.unsqueeze(topk_probs[:, -1], [1])), "float32") old_probs = probs probs = probs * ge_cond / layers.reduce_sum(topk_probs, dim=-1, keep_dim=True) sampling_ids = layers.sampling_id(probs, dtype="int") probs = old_probs else: raise ValueError(self.decoding_strategy) sampling_scores = layers.one_hot( layers.unsqueeze(sampling_ids, [1]), probs.shape[1] ) sampling_scores = sampling_scores * probs - (1 - sampling_scores) * 1e3 topk_scores, topk_indices = layers.topk( input=sampling_scores, k=1) pre_len = layers.cast(step_idx, "float32") layers.increment(x=step_idx, value=1.0, in_place=True) cur_len = layers.cast(step_idx, "float32") # update scores if self.length_average: accu_scores = layers.elementwise_add( x=layers.log(topk_scores), y=pre_scores * pre_len, axis=0) / cur_len elif self.length_penalty > 0: pre_lp = layers.pow((5 + pre_len) / 6, self.length_penalty) cur_lp = layers.pow((5 + cur_len) / 6, self.length_penalty) accu_scores = layers.elementwise_add( x=layers.log(topk_scores), y=pre_scores * pre_lp, axis=0) / cur_lp else: accu_scores = layers.elementwise_add( x=layers.log(topk_scores), y=pre_scores, axis=0) topk_indices = layers.lod_reset(topk_indices, pre_ids) accu_scores = layers.lod_reset(accu_scores, pre_ids) selected_ids, selected_scores, gather_idx = layers.beam_search( pre_ids=pre_ids, pre_scores=pre_scores, ids=topk_indices, scores=accu_scores, beam_size=beam_size, end_id=self.eos_id, return_parent_idx=True) layers.array_write(selected_ids, i=step_idx, array=ids) layers.array_write(selected_scores, i=step_idx, array=scores) layers.array_write(pre_mask, i=step_idx, array=tgt_generation_mask) layers.array_write(pos_bias, i=step_idx, array=pos_biases) layers.assign(gather_idx, parent_idx) length_cond = layers.less_than(x=step_idx, y=max_len) finish_cond = layers.logical_not(layers.is_empty(x=selected_ids)) layers.logical_and(x=length_cond, y=finish_cond, out=cond) finished_ids, finished_scores = layers.beam_search_decode( ids, scores, beam_size=beam_size, end_id=self.eos_id) predictions = { "finished_ids": finished_ids, "finished_scores": finished_scores, "token_ids": inputs["token_ids"], "data_id": inputs["data_id"] } return predictions
def type_error_default(): layers.case(pred_fn_pairs=[(pred_1, fn_1)], default=fn_1())
def type_error_fn(): layers.case(pred_fn_pairs=[(pred_1, 2)], default=fn_1)
def type_error_pred(): layers.case(pred_fn_pairs=[(1, fn_1)], default=fn_1)
def type_error_pred_fn_1(): layers.case(pred_fn_pairs=[1], default=fn_1)
def test_nested_case(self): def fn_1(x=1): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) out = layers.case(pred_fn_pairs=[ (var_5 < var_6, partial( layers.fill_constant, shape=[1], dtype='int32', value=x)), (var_5 == var_6, partial( layers.fill_constant, shape=[2], dtype='int32', value=x)) ]) return out def fn_2(x=2): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) out = layers.case(pred_fn_pairs=[ (var_5 < var_6, partial(fn_1, x=x)), (var_5 == var_6, partial( layers.fill_constant, shape=[2], dtype='int32', value=x)) ]) return out def fn_3(): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) out = layers.case(pred_fn_pairs=[ (var_5 < var_6, partial(fn_2, x=3)), (var_5 == var_6, partial( layers.fill_constant, shape=[2], dtype='int32', value=7)) ]) return out main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): x = layers.fill_constant(shape=[1], dtype='float32', value=0.3) y = layers.fill_constant(shape=[1], dtype='float32', value=0.1) z = layers.fill_constant(shape=[1], dtype='float32', value=0.2) pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3) out_3 = layers.case(pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)], default=fn_3) place = fluid.CUDAPlace( 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=[out_1, out_2, out_3]) self.assertTrue(np.allclose(res[0], 1)) self.assertTrue(np.allclose(res[1], 2)) self.assertTrue(np.allclose(res[2], 3))
def static(train_data, loss_in_switch=True, use_cuda=False, use_parallel_exe=False): startup_program = Program() main_program = Program() startup_program.random_seed = SEED main_program.random_seed = SEED with program_guard(main_program, startup_program): def double_fc_net(image): hidden = layers.fc( image, size=FC_SIZE, act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.99)), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.5)), name="hidden") prediction = layers.fc( hidden, size=CLASS_NUM, act='softmax', param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=1.2)), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.8)), name="prediction") return hidden, prediction def fn_1(opt, avg_loss=None, pred=None, label=None): if avg_loss is None: loss = layers.cross_entropy(input=pred, label=label) avg_loss = layers.mean(loss, name='mean_cross_entropy_loss') opt.minimize(avg_loss) return avg_loss def fn_2(opt, avg_loss=None, pred=None, label=None): if avg_loss is None: loss = layers.softmax_with_cross_entropy(logits=pred, label=label) avg_loss = layers.mean(loss, name='mean_softmax_loss') opt.minimize(avg_loss) return avg_loss image = fluid.data('image', [BATCH_SIZE, INPUT_SIZE], 'float32') label = fluid.data('label', [BATCH_SIZE, 1], 'int64') hidden, prediction = double_fc_net(image) adam = optimizer.Adam(learning_rate=LR) sgd = optimizer.SGD(learning_rate=LR) id = fluid.data('id', [1], 'int32') two = layers.fill_constant([1], 'int32', 2) mod_two = layers.elementwise_mod(id, two) == 0 if loss_in_switch: avg_loss = layers.case( [(mod_two, lambda: fn_1(adam, None, prediction, label))], lambda: fn_2(sgd, None, prediction, label)) else: loss_1 = layers.cross_entropy(input=prediction, label=label) avg_loss_1 = layers.mean(loss_1) loss_2 = layers.softmax_with_cross_entropy(logits=prediction, label=label) avg_loss_2 = layers.mean(loss_2) avg_loss = layers.case([(mod_two, lambda: fn_1(adam, avg_loss_1))], lambda: fn_2(sgd, avg_loss_2)) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_program) for epoch in range(EPOCH_NUM): feed_image, feed_label = train_data[epoch] fetch_list = [hidden, prediction, avg_loss] feed = { 'image': feed_image, 'label': feed_label, 'id': np.array([epoch]).astype('int32') } out = exe.run(main_program, feed=feed, fetch_list=fetch_list) out_hidden, out_pred, loss = out return out_hidden, out_pred, loss
def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='float32', value=1) def fn_2(): return layers.fill_constant(shape=[2, 2], dtype='int32', value=2) def fn_3(): return layers.fill_constant(shape=[3], dtype='int32', value=3) main_program = fluid.default_main_program() startup_program = fluid.default_startup_program() with fluid.program_guard(main_program, startup_program): x = layers.fill_constant(shape=[1], dtype='float32', value=0.3) y = layers.fill_constant(shape=[1], dtype='float32', value=0.1) z = layers.fill_constant(shape=[1], dtype='float32', value=0.2) pred_1 = layers.less_than(z, x) pred_2 = layers.less_than(x, y) pred_3 = layers.equal(x, y) out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)]) exe = fluid.Executor(fluid.CPUPlace()) res_1, res_2 = exe.run(main_program, fetch_list=[out_1, out_2]) print(res_1) print(res_2)