def dynamic(train_data, use_cuda=False, use_parallel_exe=False): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = SEED fluid.default_main_program().random_seed = SEED dy_layer = DygraphLayer() adam = fluid.optimizer.Adam(learning_rate=LR, parameter_list=dy_layer.parameters()) sgd = fluid.optimizer.SGD(learning_rate=LR, parameter_list=dy_layer.parameters()) for epoch in range(EPOCH_NUM): image_data, label = train_data[epoch] var_input = fluid.dygraph.to_variable(image_data) var_label = fluid.dygraph.to_variable(label) hidden, prediction = dy_layer(var_input) if epoch % 2 == 0: cross_entropy_loss = layers.cross_entropy( prediction, var_label) loss = layers.mean(cross_entropy_loss) loss.backward() adam.minimize(loss) else: softmax_loss = layers.softmax_with_cross_entropy( prediction, var_label) loss = layers.mean(softmax_loss) loss.backward() sgd.minimize(loss) dy_layer.clear_gradients() return hidden.numpy(), prediction.numpy(), loss.numpy()
def rc_model(hidden_size, vocab, args): emb_shape = [vocab.size(), vocab.embed_dim] start_labels = layers.data(name="start_lables", shape=[1], dtype='float32', lod_level=1) end_labels = layers.data(name="end_lables", shape=[1], dtype='float32', lod_level=1) # stage 1:encode q_id0 = get_data('q_id0', 1, args) q_ids = get_data('q_ids', 2, args) p_ids_name = 'p_ids' p_ids = get_data('p_ids', 2, args) p_embs = embedding(p_ids, emb_shape, args) q_embs = embedding(q_ids, emb_shape, args) drnn = layers.DynamicRNN() with drnn.block(): p_emb = drnn.step_input(p_embs) q_emb = drnn.step_input(q_embs) p_enc = encoder(p_emb, 'p_enc', hidden_size, args) q_enc = encoder(q_emb, 'q_enc', hidden_size, args) # stage 2:match g_i = attn_flow(q_enc, p_enc, p_ids_name, args) # stage 3:fusion m_i = fusion(g_i, args) drnn.output(m_i, q_enc) ms, q_encs = drnn() p_vec = layers.lod_reset(x=ms, y=start_labels) q_vec = layers.lod_reset(x=q_encs, y=q_id0) # stage 4:decode start_probs, end_probs = point_network_decoder(p_vec=p_vec, q_vec=q_vec, hidden_size=hidden_size, args=args) cost0 = layers.sequence_pool( layers.cross_entropy(input=start_probs, label=start_labels, soft_label=True), 'sum') cost1 = layers.sequence_pool( layers.cross_entropy(input=end_probs, label=end_labels, soft_label=True), 'sum') cost0 = layers.mean(cost0) cost1 = layers.mean(cost1) cost = cost0 + cost1 cost.persistable = True feeding_list = ["q_ids", "start_lables", "end_lables", "p_ids", "q_id0"] return cost, start_probs, end_probs, ms, feeding_list
def setUp(self): self.setup_program() self.data_field = {"x", "h_boot"} self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) with fluid.program_guard(self.main_program, self.startup_program): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', append_batch_size=False) x.stop_gradient = False h_boot = layers.data( shape=[self.input_dim], dtype='float32', name='h_boot') h_boot.stop_gradient = False forward_only_rnn = layers.StaticRNN() with forward_only_rnn.step(): h_pre = forward_only_rnn.memory(init=h_boot) x_t = forward_only_rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( x=h_pre, y=x_t), scale=self.py_rnn.scale) forward_only_rnn.update_memory(h_pre, h) forward_only_rnn.output(h) forward_only_output = forward_only_rnn() forward_only_output.stop_gradient = True self.forward_only_output = layers.mean(forward_only_output) rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( x=h_pre, y=x_t), scale=self.py_rnn.scale) rnn.update_memory(h_pre, h) rnn.output(h) self.output = layers.mean(rnn())
def test_nce(self): window_size = 5 words = [] for i in range(window_size): words.append( layers.data(name='word_{0}'.format(i), shape=[1], dtype='int64')) dict_size = 10000 label_word = int(window_size // 2) + 1 embs = [] for i in range(window_size): if i == label_word: continue emb = layers.embedding(input=words[i], size=[dict_size, 32], param_attr='emb.w', is_sparse=True) embs.append(emb) embs = layers.concat(input=embs, axis=1) loss = layers.nce(input=embs, label=words[label_word], num_total_classes=dict_size, param_attr='nce.w', bias_attr='nce.b') avg_loss = layers.mean(loss) self.assertIsNotNone(avg_loss) print(str(default_main_program()))
def test_forward(self): data = layers.data(name='X', shape=[1], dtype='float32') data.stop_gradient = False cond = layers.ConditionalBlock(inputs=[data]) out = layers.create_tensor(dtype='float32') with cond.block(): hidden = layers.fc(input=data, size=10) layers.assign(hidden, out) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) x = numpy.random.random(size=(10, 1)).astype('float32') outs = exe.run(feed={'X': x}, fetch_list=[out])[0] print outs loss = layers.mean(out) append_backward(loss=loss) outs = exe.run( feed={'X': x}, fetch_list=[ default_main_program().block(0).var(data.name + "@GRAD") ])[0] print outs
def test_recognize_digits_conv(self): program = Program() with program_guard(program, startup_program=Program()): images = layers.data(name='pixel', shape=[1, 28, 28], dtype='float32') label = layers.data(name='label', shape=[1], dtype='int32') conv_pool_1 = nets.simple_img_conv_pool(input=images, filter_size=5, num_filters=2, pool_size=2, pool_stride=2, act="relu") conv_pool_2 = nets.simple_img_conv_pool(input=conv_pool_1, filter_size=5, num_filters=4, pool_size=2, pool_stride=2, act="relu") predict = layers.fc(input=conv_pool_2, size=10, act="softmax") cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(cost) print(str(program))
def _get_metrics(self, inputs, outputs): metrics = {} fc_out = self._calc_logits(enc_out=outputs["enc_out"], seq_pos=inputs["tgt_pos"]) #fc_out = self._calc_logits(outputs["enc_out"], outputs["checkpoints"], inputs["tgt_pos"]) lm_loss = layers.softmax_with_cross_entropy(logits=fc_out, label=inputs["tgt_pos"]) need_cal = layers.not_equal( inputs["tgt_label"], layers.fill_constant(shape=[1], dtype="int64", value=1)) need_cal = layers.cast(need_cal, self.dtype) mean_lm_loss = layers.reduce_sum( lm_loss * need_cal) / (layers.reduce_sum(need_cal) + 1e-10) pooled_out = self._get_pooled_output(outputs["enc_out"], inputs["label_pos"]) nsp_fc_out = layers.fc(input=pooled_out, size=2, param_attr=fluid.ParamAttr( name="next_sent_fc.w_0", initializer=self.param_initializer), bias_attr="next_sent_fc.b_0") nsp_loss, nsp_softmax = layers.softmax_with_cross_entropy( logits=nsp_fc_out, label=inputs["label"], return_softmax=True) nsp_acc = layers.accuracy(nsp_softmax, inputs["label"]) mean_nsp_loss = layers.mean(nsp_loss) metrics["loss"] = mean_lm_loss + mean_nsp_loss metrics["lm_loss"] = mean_lm_loss metrics["nsp_loss"] = mean_nsp_loss metrics["nsp_acc"] = nsp_acc return metrics
def create_loss_op(self, predict, label, epsilon=1e-7): """compute loss with tensor Args: predict: model output tensor activated by softmax label: a non-sparse tensor Returns: loss: cross-entropy loss """ if self.loss_type == "nl" and self.model_type == "train": one_hot_label = fluid.one_hot(label, depth=predict.shape[-1]) one_hot_label = FL.squeeze(one_hot_label, axes=[-2]) # log neg_prob = 1 - predict log_neg_prob = FL.log( fluid.layers.clip(neg_prob, min=epsilon, max=1.)) ce_loss = -1 * log_neg_prob * one_hot_label cost = FL.reduce_sum(ce_loss, dim=-1, keep_dim=True) else: # PL or evaluation cost = FL.cross_entropy(predict, label) loss = FL.mean(cost) return loss
def test_recognize_digits_conv(self): program = Program() with program_guard(program, startup_program=Program()): images = layers.data( name='pixel', shape=[1, 28, 28], dtype='float32') label = layers.data(name='label', shape=[1], dtype='int32') conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, num_filters=2, pool_size=2, pool_stride=2, act="relu") conv_pool_2 = nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, num_filters=4, pool_size=2, pool_stride=2, act="relu") predict = layers.fc(input=conv_pool_2, size=10, act="softmax") cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(cost) print(str(program))
def not_test_raw_api(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant(shape=[1], dtype='int64', value=5) cond = layers.less_than(x=label, y=limit) true_image, false_image = split_lod_tensor(input=image, mask=cond) true_out = layers.create_tensor(dtype='float32') true_cond = ConditionalBlock([cond]) with true_cond.block(): hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=true_out) false_out = layers.create_tensor(dtype='float32') false_cond = ConditionalBlock([cond]) with false_cond.block(): hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=false_out) prob = merge_lod_tensor( in_true=true_out, in_false=false_out, mask=cond, x=image) loss = layers.cross_entropy(input=prob, label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=10) place = core.CPUPlace() exe = Executor(place) exe.run(startup_prog) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array([x[0] for x in data]).astype("float32") y_data = np.array([x[1] for x in data]).astype("int64") y_data = np.expand_dims(y_data, axis=1) outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) print(outs[0]) if outs[0] < 1.0: return self.assertFalse(True)
def test_nce(self): window_size = 5 words = [] for i in xrange(window_size): words.append( layers.data( name='word_{0}'.format(i), shape=[1], dtype='int64')) dict_size = 10000 label_word = int(window_size / 2) + 1 embs = [] for i in xrange(window_size): if i == label_word: continue emb = layers.embedding( input=words[i], size=[dict_size, 32], param_attr='emb.w', is_sparse=True) embs.append(emb) embs = layers.concat(input=embs, axis=1) loss = layers.nce(input=embs, label=words[label_word], num_total_classes=dict_size, param_attr='nce.w', bias_attr='nce.b') avg_loss = layers.mean(loss) self.assertIsNotNone(avg_loss) print(str(default_main_program()))
def get_metrics(self, inputs, outputs): """Get metrics.""" metrics = {} pooled_out = self._get_pooled_output(outputs["enc_out"]) cls_logits = self._get_classifier_output(pooled_out, num_classes=self.num_classes, name="cls") cls_loss, cls_softmax = layers.softmax_with_cross_entropy( logits=cls_logits, label=inputs["label"], return_softmax=True) cls_acc = layers.accuracy(cls_softmax, inputs["label"]) mean_cls_loss = layers.mean(cls_loss) metrics["loss"] = mean_cls_loss metrics["cls_loss"] = mean_cls_loss metrics["cls_acc"] = cls_acc # statistics for recall & precision & f1 if self.num_classes == 2: pred = layers.argmax(cls_softmax, axis=1) label = layers.squeeze(inputs["label"], axes=[1]) metrics["stat_tp"] = layers.reduce_sum( layers.logical_and(pred == 1, label == 1).astype("float32")) metrics["stat_fp"] = layers.reduce_sum( layers.logical_and(pred == 1, label == 0).astype("float32")) metrics["stat_tn"] = layers.reduce_sum( layers.logical_and(pred == 0, label == 0).astype("float32")) metrics["stat_fn"] = layers.reduce_sum( layers.logical_and(pred == 0, label == 1).astype("float32")) return metrics
def build_network(self, only_forward, **kargs): x = layers.data('x', shape=[3], dtype='float32', lod_level=1) x.stop_gradient = False layers.Print(input=x, **kargs) loss = layers.mean(x) append_backward(loss=loss) return loss
def test_save_inference_model(self): MODEL_DIR = "./tmp/inference_model3" init_program = Program() program = Program() # fake program without feed/fetch with program_guard(program, init_program): x = layers.data(name='x', shape=[2], dtype='float32') y = layers.data(name='y', shape=[1], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = layers.mean(cost) place = core.CPUPlace() exe = executor.Executor(place) exe.run(init_program, feed={}, fetch_list=[]) # will print warning message cp_prog = CompiledProgram(program).with_data_parallel( loss_name=avg_cost.name) save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, cp_prog) self.assertRaises(TypeError, save_inference_model, [MODEL_DIR, ["x", "y"], [avg_cost], [], cp_prog])
def forward_attention(indicator, support_x, support_y_embed, support_mask, query_x, query_y, query_mask): """ support_indicator: length = support_len if attention(support, query), indicator = 0 if attention(support, support), indicator = 1 """ support_y_embed = support_y_embed * support_mask support_xy = layers.concat([support_x, support_y_embed, indicator], axis=1) pad_value = layers.assign( input=numpy.array([0.0], dtype=numpy.float32)) support_pad, support_len = layers.sequence_pad(support_xy, pad_value=pad_value) query_pad, query_len = layers.sequence_pad(query_x, pad_value=pad_value) attention = self.attention(query_pad, support_pad, support_pad, self.hidden_dim, 'meta') attention = layers.sequence_unpad(attention, length=query_len) pred_input = layers.concat([attention, query_x], axis=1) pred = self.prepare_preds_with_name(pred_input, 'out_pred') label = layers.cast(query_y, dtype='float32') label = layers.scale(label, scale=0.01) loss = layers.huber_loss(pred, label, 1.0) * query_mask loss = layers.mean(loss) return pred, label, loss
def loss(self, predictions, labels): ce_loss, probs = L.softmax_with_cross_entropy(logits=predictions, label=labels, return_softmax=True) #L.Print(ce_loss, message='per_example_loss') loss = L.mean(x=ce_loss) return loss
def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): x = layers.data( name='x', shape=[1], dtype='float32', stop_gradient=False) table = layers.lod_rank_table(x, level=0) array = layers.lod_tensor_to_array(x, table) result = layers.array_to_lod_tensor(array, table) mean = layers.mean(result) append_backward(mean) tensor = core.LoDTensor() tensor.set(numpy.arange(10).reshape(10, 1).astype('float32'), place) tensor.set_lod([[0, 3, 9, 10]]) g_vars = program.global_block().var(x.name + "@GRAD") exe = Executor(place) g_out = [ numpy.array(item).sum() for item in exe.run(program, feed={'x': tensor}, fetch_list=[g_vars], return_numpy=False) ] g_out_sum = numpy.array(g_out).sum() self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def std_enc(path='miku.png', steps=2000, lr=4e-3): model_cache.train_mode = False model_cache.initialized = False img = Image.open(path) w, h = img.size min_size = min(w, h) x0 = (w - min_size) // 2 y0 = (h - min_size) // 2 x1 = x0 + min_size y1 = y0 + min_size img = img.crop([x0, y0, x1, y1]).convert('RGB') img = _img = img.resize([256, 256], Image.BILINEAR) img = np.asarray(img) / 255.0 img = dg.to_variable(img.transpose(2, 0, 1).astype('float32')[None, ...]) m_latent = Latents() optimizer = fluid.optimizer.AdamOptimizer( learning_rate=lr, parameter_list=m_latent.parameters()) for i in range(steps): z, class_emb = m_latent() out = model_cache.G(z, class_emb, input_class_emb=True) loss = layers.mean((out - img)**2) loss.backward() optimizer.minimize(loss) optimizer.clear_gradients() stdout.write(f'loss: {loss.numpy().mean()} {i+1}/{steps}\r') stdout.flush() print('') out = np.uint8(out.numpy()[0].transpose(1, 2, 0).clip(0, 1) * 255) return Image.fromarray(out), _img
def mlp_pretrain_forward(train_program, start_program): with static.program_guard(train_program, start_program), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 input = static.data(name="input", shape=[batch_size, sequence_len, hidden_size], dtype='float32') label = static.data(name="label", shape=[batch_size, sequence_len, 1], dtype='float32') auto.shard_tensor(input, dist_attr={ "process_mesh": _global_process_mesh, "dims_mappig": [-1, -1, -1] }) mlp = MLPLayer(hidden_size=hidden_size, intermediate_size=4 * hidden_size, dropout_ratio=0.1, initializer_range=0.02) predict = mlp(input) cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(x=cost) return avg_cost, train_program, start_program
def get_metrics(self, inputs, outputs): """Get metrics.""" metrics = {} tgt_logits = self._calc_logits(outputs["enc_out"], inputs["tgt_idx"]) lm_loss = layers.softmax_with_cross_entropy(logits=tgt_logits, label=inputs["tgt_label"]) need_cal = layers.not_equal( inputs["tgt_label"], layers.fill_constant(shape=[1], dtype="int64", value=1) ) need_cal = layers.cast(need_cal, self.dtype) mean_lm_loss = layers.reduce_sum(lm_loss * need_cal) / (layers.reduce_sum(need_cal) + 1e-10) pooled_out = self._get_pooled_output(outputs["enc_out"], inputs["label_idx"]) nsp_logits = self._get_classifier_output(pooled_out, name="next_sent") nsp_loss, nsp_softmax = layers.softmax_with_cross_entropy( logits=nsp_logits, label=inputs["label"], return_softmax=True) nsp_acc = layers.accuracy(nsp_softmax, inputs["label"]) mean_nsp_loss = layers.mean(nsp_loss) loss = mean_nsp_loss if self.use_mlm: loss = loss + mean_lm_loss metrics["token_lm_loss"] = mean_lm_loss metrics["loss"] = loss metrics["nsp_loss"] = mean_nsp_loss metrics["nsp_acc"] = nsp_acc return metrics
def loss(self, predictions, labels): labels = L.softmax(labels) loss = L.softmax_with_cross_entropy(predictions, labels, soft_label=True) loss = L.mean(loss) return loss
def test_forward(self): data = layers.data(name='X', shape=[1], dtype='float32') data.stop_gradient = False cond = ConditionalBlock(inputs=[data]) out = layers.create_tensor(dtype='float32') with cond.block(): hidden = layers.fc(input=data, size=10) layers.assign(hidden, out) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) x = numpy.random.random(size=(10, 1)).astype('float32') outs = exe.run(feed={'X': x}, fetch_list=[out])[0] print(outs) loss = layers.mean(out) append_backward(loss=loss) outs = exe.run(feed={'X': x}, fetch_list=[ default_main_program().block(0).var(data.name + "@GRAD") ])[0] print(outs)
def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): x = layers.data(name='x', shape=[1], dtype='float32', stop_gradient=False) table = layers.lod_rank_table(x, level=0) array = layers.lod_tensor_to_array(x, table) result = layers.array_to_lod_tensor(array, table) mean = layers.mean(result) append_backward(mean) tensor = core.LoDTensor() tensor.set(numpy.arange(10).reshape(10, 1).astype('float32'), place) tensor.set_recursive_sequence_lengths([[3, 6, 1]]) g_vars = program.global_block().var(x.name + "@GRAD") exe = Executor(place) g_out = [ numpy.array(item).sum() for item in exe.run(program, feed={'x': tensor}, fetch_list=[g_vars], return_numpy=False) ] g_out_sum = numpy.array(g_out).sum() self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def build(self, inputs, scope_name=''): sent_emb = inputs['backbone']['sentence_embedding'] if self._is_training: label_ids = inputs['reader']['label_ids'] cls_feats = fluid.layers.dropout( x=sent_emb, dropout_prob=self._dropout_prob, dropout_implementation="upscale_in_train") logits = fluid.layers.fc( input=sent_emb, size=self.num_classes, param_attr=fluid.ParamAttr( name=scope_name+"cls_out_w", initializer=self._param_initializer), bias_attr=fluid.ParamAttr( name=scope_name+"cls_out_b", initializer=fluid.initializer.Constant(0.))) if self._is_training: loss = fluid.layers.softmax_with_cross_entropy( logits=logits, label=label_ids) loss = layers.mean(loss) return {"loss": loss} else: return {"logits":logits}
def test_forward_backward_single_tensor_output(self): program = Program() with program_guard(program): x = layers.data(name='x', shape=[2], dtype='float32') x.stop_gradient = False # For test gradient mask = layers.data(name='mask', shape=[1], dtype='int32') out = program.current_block().create_var( dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR) select_output(x, out, mask) y = select_input(out, mask) mean = layers.mean(y) append_backward(mean) place = fluid.CUDAPlace( 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() exe = Executor(place) feed_x = np.asarray([1.3, -1.4]).astype(np.float32) feed_mask = np.asarray([0]).astype(np.int32) ret = exe.run(program, feed={ 'x': feed_x, 'mask': feed_mask }, fetch_list=[y.name, x.grad_name]) x_grad = np.asarray([0.5, 0.5]).astype(np.float32) self.assertTrue(np.allclose(np.asarray(ret[0]), feed_x)) self.assertTrue(np.allclose(np.asarray(ret[1]), x_grad))
def fn_2(opt, avg_loss=None, pred=None, label=None): if avg_loss is None: loss = layers.softmax_with_cross_entropy(logits=pred, label=label) avg_loss = layers.mean(loss, name='mean_softmax_loss') opt.minimize(avg_loss) return avg_loss
def _get_metrics(self, inputs, outputs): metrics = super(Plato, self)._get_metrics(inputs, outputs) if self.use_bow: fc_out = self._calc_bow_logits(outputs["enc_out"], inputs["bow_pos"]) bow_loss = layers.softmax_with_cross_entropy( logits=fc_out, label=inputs["bow_label"]) mean_bow_loss = layers.mean(bow_loss) metrics["bow_loss"] = mean_bow_loss metrics["loss"] = metrics["loss"] + mean_bow_loss entropy_loss = layers.reduce_sum(outputs["post_probs"] * layers.log(outputs["post_probs"]), dim=1) mean_entropy_loss = layers.mean(entropy_loss) metrics["entropy_loss"] = mean_entropy_loss if self.use_entropy: metrics["loss"] = metrics["loss"] + mean_entropy_loss return metrics
def build_network(self, only_forward, **kargs): x = layers.data('x', shape=[3], dtype='float32', lod_level=1) x.stop_gradient = False printed = layers.Print(input=x, **kargs) if only_forward: return printed loss = layers.mean(printed) append_backward(loss=loss) return loss
def train_program(self, ): label = F.data(name="label", shape=[None, 1], dtype="int64") train_idx = F.data(name='train_idx', shape=[None], dtype="int64") prediction = L.gather(self.out_feat, train_idx, overwrite=False) label = L.gather(label, train_idx, overwrite=False) cost = L.softmax_with_cross_entropy(logits=prediction, label=label) avg_cost = L.mean(cost) self.avg_cost = avg_cost
def train_program(is_sparse): context = encoder(is_sparse) rnn_out = train_decoder(context, is_sparse) label = pd.data( name="target_language_next_word", shape=[1], dtype='int64', lod_level=1) cost = pd.cross_entropy(input=rnn_out, label=label) avg_cost = pd.mean(cost) return avg_cost
def test_ifelse(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant_batch_size_like(input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) ie = layers.IfElse(cond) with ie.true_block(): true_image = ie.input(image) hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) with ie.false_block(): false_image = ie.input(image) hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) prob = ie() loss = layers.cross_entropy(input=prob[0], label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=200) place = core.CPUPlace() exe = Executor(place) exe.run(kwargs['startup_program']) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) outs = exe.run(kwargs['main_program'], feed={ 'x': x_data, 'y': y_data }, fetch_list=[avg_loss]) print outs[0] if outs[0] < 1.0: return self.assertFalse(True)
def loss(self, predictions, labels): logits, input_seqlen = predictions logits = L.flatten(logits, axis=2) labels = L.flatten(labels, axis=2) ce_loss, probs = L.softmax_with_cross_entropy(logits=logits, label=labels, return_softmax=True) loss = L.mean(x=ce_loss) return loss
def simple_net(self): d0 = layers.data( "d0", shape=[10], append_batch_size=False, dtype='float32') d1 = layers.data( "d1", shape=[10], append_batch_size=False, dtype='float32') d2 = layers.data( "d2", shape=[10], append_batch_size=False, dtype='float32') # fill_constant npu op doesn't support int64 i = layers.zeros(shape=[1], dtype='int32') i = layers.cast(i, 'int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') mem_array = layers.array_write(x=init, i=i) data_array = layers.array_write(x=d0, i=i) i = layers.increment(i) layers.array_write(d1, i, array=data_array) i = layers.increment(i) layers.array_write(d2, i, array=data_array) i = layers.zeros(shape=[1], dtype='int32') i = layers.cast(i, 'int64') i.stop_gradient = True array_len = layers.fill_constant(shape=[1], dtype='int32', value=5) array_len = layers.cast(array_len, 'int64') array_len.stop_gradient = True cond = layers.ones(shape=[1], dtype='int32') cond = layers.cast(cond, 'bool') j = layers.fill_constant(shape=[1], dtype='int32', value=1) j = layers.cast(j, 'int64') j.stop_gradient = True array_len2 = layers.fill_constant(shape=[1], dtype='int32', value=3) array_len2 = layers.cast(array_len2, 'int64') array_len2.stop_gradient = True cond2 = layers.logical_or(x=j, y=array_len2) cond2 = layers.ones(shape=[1], dtype='int32') cond2 = layers.cast(cond2, 'bool') while_op = layers.While(cond=cond) while_op2 = layers.While(cond=cond2) with while_op.block(): d = layers.array_read(array=data_array, i=i) prev = layers.array_read(array=mem_array, i=i) result = layers.sums(input=[d, prev]) i = layers.increment(x=i, in_place=True) layers.array_write(result, i=i, array=mem_array) layers.less_than(x=i, y=array_len, cond=cond) with while_op2.block(): d2 = layers.array_read(array=data_array, i=j) prev2 = layers.array_read(array=mem_array, i=j) result2 = layers.sums(input=[d2, prev2]) j = layers.increment(x=j, in_place=True) layers.array_write(result2, i=j, array=mem_array) layers.less_than(x=j, y=array_len2, cond=cond2) sum_result = layers.array_read(array=mem_array, i=j) loss = layers.mean(sum_result) return loss, sum_result
def setUp(self): self.setup_program() self.data_field = {"x", "h_boot"} self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) self.output = layers.mean(self.create_rnn_op(), **self.p_info)
def train_program(): scale_infer = inference_program() label = layers.data(name='score', shape=[1], dtype='float32') square_cost = layers.square_error_cost(input=scale_infer, label=label) avg_cost = layers.mean(square_cost) return [avg_cost, scale_infer]
def test_fit_line_inference_model(self): MODEL_DIR = "./tmp/inference_model" init_program = Program() program = Program() with program_guard(program, init_program): x = layers.data(name='x', shape=[2], dtype='float32') y = layers.data(name='y', shape=[1], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = layers.mean(cost) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) sgd_optimizer.minimize(avg_cost, init_program) place = core.CPUPlace() exe = executor.Executor(place) exe.run(init_program, feed={}, fetch_list=[]) for i in xrange(100): tensor_x = np.array( [[1, 1], [1, 2], [3, 4], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") exe.run(program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost]) save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, program) expected = exe.run(program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost])[0] reload(executor) # reload to build a new scope exe = executor.Executor(place) [infer_prog, feed_var_names, fetch_vars] = load_inference_model( MODEL_DIR, exe) outs = exe.run( infer_prog, feed={feed_var_names[0]: tensor_x, feed_var_names[1]: tensor_y}, fetch_list=fetch_vars) actual = outs[0] self.assertEqual(feed_var_names, ["x", "y"]) self.assertEqual(len(fetch_vars), 1) self.assertEqual(str(fetch_vars[0]), str(avg_cost)) self.assertEqual(expected, actual)
def test_ifelse(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) ie = layers.IfElse(cond) with ie.true_block(): true_image = ie.input(image) hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) with ie.false_block(): false_image = ie.input(image) hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) prob = ie() loss = layers.cross_entropy(input=prob[0], label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=200) place = core.CPUPlace() exe = Executor(place) exe.run(kwargs['startup_program']) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) outs = exe.run(kwargs['main_program'], feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) print outs[0] if outs[0] < 1.0: return self.assertFalse(True)
def test_fit_a_line(self): program = Program() with program_guard(program, startup_program=Program()): x = layers.data(name='x', shape=[13], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) y = layers.data(name='y', shape=[1], dtype='float32') cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = layers.mean(cost) self.assertIsNotNone(avg_cost) print(str(program))
def test_calc_gradient(self): x = layers.create_parameter(dtype="float32", shape=[5, 10]) y = layers.create_parameter(dtype="float32", shape=[10, 8]) mul_out = layers.mul(x=x, y=y) mean_out = layers.mean(mul_out) a = calc_gradient(mean_out, mul_out) b = calc_gradient(mean_out, x) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) exe.run(fluid.default_main_program(), feed={}, fetch_list=[a, b])
def setUp(self): self.setup_program() self.data_field = {"x"} self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4(self.input_shape, self.output_shape) self.output = layers.mean(self.create_rnn_op(), **self.p_info) print self.main_program
def setUp(self): self.setup_program() self.data_field = {"x", "h_boot1", "h_boot2"} self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpMultipleMemoryTest.PySimpleRNN3( self.input_shape, self.output_shape) self.output = layers.mean(self.create_rnn_op(), **self.p_info)
def test_simple_forward(self): d0 = layers.data( "d0", shape=[10], append_batch_size=False, dtype='float32') d1 = layers.data( "d1", shape=[10], append_batch_size=False, dtype='float32') d2 = layers.data( "d2", shape=[10], append_batch_size=False, dtype='float32') i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') mem_array = layers.array_write(x=init, i=i) data_array = layers.array_write(x=d0, i=i) i = layers.increment(i) layers.array_write(d1, i, array=data_array) i = layers.increment(i) layers.array_write(d2, i, array=data_array) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True array_len = layers.fill_constant(shape=[1], dtype='int64', value=3) array_len.stop_gradient = True cond = layers.less_than(x=i, y=array_len) while_op = layers.While(cond=cond) with while_op.block(): d = layers.array_read(array=data_array, i=i) prev = layers.array_read(array=mem_array, i=i) result = layers.sums(input=[d, prev]) i = layers.increment(x=i, in_place=True) layers.array_write(result, i=i, array=mem_array) layers.less_than(x=i, y=array_len, cond=cond) sum_result = layers.array_read(array=mem_array, i=i) loss = layers.mean(sum_result) append_backward(loss) cpu = core.CPUPlace() exe = Executor(cpu) d = [] for i in xrange(3): d.append(numpy.random.random(size=[10]).astype('float32')) outs = exe.run(feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, fetch_list=[sum_result]) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01)
def setUp(self): program = Program() with program_guard(program, startup_program=Program()): x = layers.data(name='x', shape=[13], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) y = layers.data(name='y', shape=[1], dtype='float32') cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = layers.mean(cost) opt = optimizer.SGD(learning_rate=0.001) opt = opt.minimize(avg_cost) self.program = program
def model(): usr_combined_features = get_usr_combined_features() mov_combined_features = get_mov_combined_features() # need cos sim inference = layers.cos_sim(X=usr_combined_features, Y=mov_combined_features) scale_infer = layers.scale(x=inference, scale=5.0) label = layers.data(name='score', shape=[1], dtype='float32') square_cost = layers.square_error_cost(input=scale_infer, label=label) avg_cost = layers.mean(square_cost) return scale_infer, avg_cost
def test_recognize_digits_mlp(self): program = Program() with program_guard(program, startup_program=Program()): # Change g_program, so the rest layers use `g_program` images = layers.data(name='pixel', shape=[784], dtype='float32') label = layers.data(name='label', shape=[1], dtype='int32') hidden1 = layers.fc(input=images, size=128, act='relu') hidden2 = layers.fc(input=hidden1, size=64, act='relu') predict = layers.fc(input=[hidden2, hidden1], size=10, act='softmax', param_attr=["sftmax.w1", "sftmax.w2"]) cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(cost) self.assertIsNotNone(avg_cost) print(str(program))
def test_word_embedding(self): program = Program() with program_guard(program, startup_program=Program()): dict_size = 10000 embed_size = 32 first_word = layers.data(name='firstw', shape=[1], dtype='int64') second_word = layers.data(name='secondw', shape=[1], dtype='int64') third_word = layers.data(name='thirdw', shape=[1], dtype='int64') forth_word = layers.data(name='forthw', shape=[1], dtype='int64') next_word = layers.data(name='nextw', shape=[1], dtype='int64') embed_first = layers.embedding( input=first_word, size=[dict_size, embed_size], dtype='float32', param_attr='shared_w') embed_second = layers.embedding( input=second_word, size=[dict_size, embed_size], dtype='float32', param_attr='shared_w') embed_third = layers.embedding( input=third_word, size=[dict_size, embed_size], dtype='float32', param_attr='shared_w') embed_forth = layers.embedding( input=forth_word, size=[dict_size, embed_size], dtype='float32', param_attr='shared_w') concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1) hidden1 = layers.fc(input=concat_embed, size=256, act='sigmoid') predict_word = layers.fc(input=hidden1, size=dict_size, act='softmax') cost = layers.cross_entropy(input=predict_word, label=next_word) avg_cost = layers.mean(cost) self.assertIsNotNone(avg_cost) print(str(program))
def setUp(self): self.main_program = Program() switch_main_program(self.main_program) x = layers.data('x', shape=[100], dtype='float32') x.stop_gradient = False rank_table_tensor = layers.data( 'rank_table_tensor', shape=[1], dtype='float32', lod_level=1) table = layers.lod_rank_table(x=rank_table_tensor) i = layers.zeros(dtype='int64', shape=[1]) self.mem1 = layers.shrink_memory(x=x, i=i, table=table) i = layers.increment(x=i) i.stop_gradient = True self.mem2 = layers.shrink_memory(x=self.mem1, i=i, table=table) i = layers.increment(x=i) i.stop_gradient = True self.mem3 = layers.shrink_memory(x=self.mem2, i=i, table=table) mem3_mean = layers.mean(self.mem3) append_backward(loss=mem3_mean) self.x_grad = self.main_program.global_block().var('x@GRAD')
def test_raw_api(self): prog = Program() startup_prog = Program() with program_guard(prog, startup_prog): image = layers.data(name='x', shape=[784], dtype='float32') label = layers.data(name='y', shape=[1], dtype='int64') limit = layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) true_image, false_image = layers.split_lod_tensor( input=image, mask=cond) true_out = layers.create_tensor(dtype='float32') true_cond = layers.ConditionalBlock([true_image]) with true_cond.block(): hidden = layers.fc(input=true_image, size=100, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=true_out) false_out = layers.create_tensor(dtype='float32') false_cond = layers.ConditionalBlock([false_image]) with false_cond.block(): hidden = layers.fc(input=false_image, size=200, act='tanh') prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=false_out) prob = layers.merge_lod_tensor( in_true=true_out, in_false=false_out, mask=cond, x=image) loss = layers.cross_entropy(input=prob, label=label) avg_loss = layers.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), batch_size=200) place = core.CPUPlace() exe = Executor(place) exe.run(startup_prog) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = np.expand_dims(y_data, axis=1) outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) print outs[0] if outs[0] < 1.0: return self.assertFalse(True)
def test_read_write(self): x = [ layers.data( name='x0', shape=[100]), layers.data( name='x1', shape=[100]), layers.data( name='x2', shape=[100]) ] for each_x in x: each_x.stop_gradient = False i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = False arr = layers.array_write(x=x[0], i=i) i = layers.increment(x=i) arr = layers.array_write(x=x[1], i=i, array=arr) i = layers.increment(x=i) arr = layers.array_write(x=x[2], i=i, array=arr) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = False a0 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) a1 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) a2 = layers.array_read(array=arr, i=i) mean_a0 = layers.mean(a0) mean_a1 = layers.mean(a1) mean_a2 = layers.mean(a2) a_sum = layers.sums(input=[mean_a0, mean_a1, mean_a2]) mean_x0 = layers.mean(x[0]) mean_x1 = layers.mean(x[1]) mean_x2 = layers.mean(x[2]) x_sum = layers.sums(input=[mean_x0, mean_x1, mean_x2]) scope = core.Scope() cpu = core.CPUPlace() exe = Executor(cpu) tensor = numpy.random.random(size=(100, 100)).astype('float32') outs = exe.run(feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, fetch_list=[a_sum, x_sum], scope=scope) self.assertEqual(outs[0], outs[1]) total_sum = layers.sums(input=[a_sum, x_sum]) total_sum_scaled = layers.scale(x=total_sum, scale=1 / 6.0) append_backward(total_sum_scaled) g_vars = map(default_main_program().global_block().var, [each_x.name + "@GRAD" for each_x in x]) g_out = [ item.sum() for item in exe.run( feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, fetch_list=g_vars) ] g_out_sum = numpy.array(g_out).sum() # since our final gradient is 1 and the neural network are all linear # with mean_op. # the input gradient should also be 1 self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)