def test_api(self): x_1 = paddle.data(shape=[None, 1, 4, 5], dtype='int32', name='x_1') paddle.concat([x_1, x_1], 0) input_2 = np.random.random([2, 1, 4, 5]).astype("int32") input_3 = np.random.random([2, 2, 4, 5]).astype("int32") x_2 = fluid.data(shape=[2, 1, 4, 5], dtype='int32', name='x_2') x_3 = fluid.data(shape=[2, 2, 4, 5], dtype='int32', name='x_3') positive_1_int32 = paddle.fill_constant([1], "int32", 1) positive_1_int64 = paddle.fill_constant([1], "int64", 1) negative_int64 = paddle.fill_constant([1], "int64", -3) out_1 = paddle.concat(x=[x_2, x_3], axis=1) out_2 = paddle.concat(x=[x_2, x_3], axis=positive_1_int32) out_3 = paddle.concat(x=[x_2, x_3], axis=positive_1_int64) out_4 = paddle.concat(x=[x_2, x_3], axis=negative_int64) exe = paddle.static.Executor(place=paddle.CPUPlace()) [res_1, res_2, res_3, res_4] = exe.run(paddle.static.default_main_program(), feed={ "x_1": input_2, "x_2": input_2, "x_3": input_3 }, fetch_list=[out_1, out_2, out_3, out_4]) assert np.array_equal(res_1, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_2, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_3, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_4, np.concatenate((input_2, input_3), axis=1))
def get_target_tensor(self, prediction, target_is_real): """Create label tensors with the same size as the input. Parameters: prediction (tensor) - - tpyically the prediction from a discriminator target_is_real (bool) - - if the ground truth label is for real images or fake images Returns: A label tensor filled with ground truth label, and with the size of the input """ if target_is_real: if not hasattr(self, 'target_real_tensor'): self.target_real_tensor = paddle.fill_constant( shape=paddle.shape(prediction), value=self.target_real_label, dtype='float32') target_tensor = self.target_real_tensor else: if not hasattr(self, 'target_fake_tensor'): self.target_fake_tensor = paddle.fill_constant( shape=paddle.shape(prediction), value=self.target_fake_label, dtype='float32') target_tensor = self.target_fake_tensor # target_tensor.stop_gradient = True return target_tensor
def test_api(self): shape = [1000, 784] train_program = Program() startup_program = Program() with program_guard(train_program, startup_program): x1 = paddle.randn(shape, 'float32') x2 = paddle.randn(shape, 'float64') dim_1 = paddle.fill_constant([1], "int64", 20) dim_2 = paddle.fill_constant([1], "int32", 50) x3 = paddle.randn([dim_1, dim_2, 784]) var_shape = paddle.static.data('X', [2], 'int32') x4 = paddle.randn(var_shape) place = paddle.CUDAPlace( 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) res = exe.run(train_program, feed={'X': np.array(shape, dtype='int32')}, fetch_list=[x1, x2, x3, x4]) for out in res: self.assertAlmostEqual(np.mean(out), .0, delta=0.1) self.assertAlmostEqual(np.std(out), 1., delta=0.1)
def barrier(group=0): """ Barrier among all participators in the group. Args: group (int): The id of the process group to work on. Returns: None. Examples: .. code-block:: python import paddle from paddle.distributed import init_parallel_env paddle.disable_static() paddle.set_device('gpu:%d'%paddle.distributed.ParallelEnv().dev_id) init_parallel_env() paddle.distributed.barrier() """ op_type = 'barrier' temp = paddle.fill_constant([1], dtype="int32", value="1") if in_dygraph_mode(): return core.ops.barrier(temp, temp, 'ring_id', group) if not isinstance(group, int): raise ValueError("The type of 'group' for barrier must be int.") helper = LayerHelper(op_type, **locals()) helper.append_op(type=op_type, inputs={'X': [temp]}, outputs={'Out': [temp]}, attrs={'ring_id': group})
def init_weights(layer): if type(layer) == nn.Linear: new_weight = paddle.fill_constant(layer.weight.shape, layer.weight.dtype, value=0.9) layer.weight.set_value(new_weight) new_bias = paddle.fill_constant(layer.bias.shape, layer.bias.dtype, value=-0.1) layer.bias.set_value(new_bias) elif type(layer) == nn.Conv2d: new_weight = paddle.fill_constant(layer.weight.shape, layer.weight.dtype, value=0.7) layer.weight.set_value(new_weight) new_bias = paddle.fill_constant(layer.bias.shape, layer.bias.dtype, value=-0.2) layer.bias.set_value(new_bias)
def test_api(self): shape = [1000, 784] place = paddle.CUDAPlace( 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() paddle.disable_static(place) x1 = paddle.randn(shape, 'float32') x2 = paddle.randn(shape, 'float64') dim_1 = paddle.fill_constant([1], "int64", 20) dim_2 = paddle.fill_constant([1], "int32", 50) x3 = paddle.randn(shape=[dim_1, dim_2, 784]) var_shape = paddle.to_variable(np.array(shape)) x4 = paddle.randn(var_shape) for out in [x1, x2, x3, x4]: self.assertAlmostEqual(np.mean(out.numpy()), .0, delta=0.1) self.assertAlmostEqual(np.std(out.numpy()), 1., delta=0.1) paddle.enable_static()
def run_retain(self, need_retain): g = Generator() d = Discriminator() optim_g = paddle.optimizer.Adam(parameters=g.parameters()) optim_d = paddle.optimizer.Adam(parameters=d.parameters()) gan_criterion = paddle.nn.MSELoss() l1_criterion = paddle.nn.L1Loss() A = np.random.rand(2, 3, 32, 32).astype('float32') B = np.random.rand(2, 3, 32, 32).astype('float32') realA = paddle.to_variable(A) realB = paddle.to_variable(B) fakeB = g(realA) optim_d.clear_gradients() fake_AB = paddle.concat((realA, fakeB), 1) G_pred_fake = d(fake_AB.detach()) false_target = paddle.fill_constant(G_pred_fake.shape, 'float32', 0.0) G_gradient_penalty, _ = self.cal_gradient_penalty( d, realA, fakeB, lambda_gp=10.0) loss_d = gan_criterion(G_pred_fake, false_target) + G_gradient_penalty loss_d.backward(retain_graph=need_retain) optim_d.minimize(loss_d) optim_g.clear_gradients() fake_AB = paddle.concat((realA, fakeB), 1) G_pred_fake = d(fake_AB) true_target = paddle.fill_constant(G_pred_fake.shape, 'float32', 1.0) loss_g = l1_criterion(fakeB, realB) + gan_criterion(G_pred_fake, true_target) loss_g.backward() optim_g.minimize(loss_g)
def test_api(self): with program_guard(Program(), Program()): # results are from [0, 5). out1 = paddle.randint(5) # shape is a list and dtype is 'int32' out2 = paddle.randint(low=-100, high=100, shape=[64, 64], dtype='int32') # shape is a tuple and dtype is 'int64' out3 = paddle.randint(low=-100, high=100, shape=(32, 32, 3), dtype='int64') # shape is a tensorlist and dtype is 'float32' dim_1 = paddle.fill_constant([1], "int64", 32) dim_2 = paddle.fill_constant([1], "int32", 50) out4 = paddle.randint(low=-100, high=100, shape=[dim_1, 5, dim_2], dtype='int32') # shape is a tensor and dtype is 'float64' var_shape = paddle.static.data(name='var_shape', shape=[2], dtype="int64") out5 = paddle.randint(low=1, high=1000, shape=var_shape, dtype='int64') place = paddle.CUDAPlace( 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) outs = exe.run( feed={'var_shape': np.array([100, 100]).astype('int64')}, fetch_list=[out1, out2, out3, out4, out5])
def cal_gradient_penalty(netD, real_data, fake_data, edge_data=None, type='mixed', constant=1.0, lambda_gp=10.0): if lambda_gp > 0.0: if type == 'real': # either use real images, fake images, or a linear interpolation of two. interpolatesv = real_data elif type == 'fake': interpolatesv = fake_data elif type == 'mixed': alpha = paddle.rand((real_data.shape[0], 1)) alpha = paddle.expand( alpha, [1, np.prod(real_data.shape) // real_data.shape[0]]) alpha = paddle.reshape(alpha, real_data.shape) interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) else: raise NotImplementedError('{} not implemented'.format(type)) # interpolatesv.requires_grad_(True) interpolatesv.stop_gradient = False real_data.stop_gradient = True fake_AB = paddle.concat((real_data.detach(), interpolatesv), 1) disc_interpolates = netD(fake_AB) # FIXME: use paddle.ones outs = paddle.fill_constant(disc_interpolates.shape, disc_interpolates.dtype, 1.0) gradients = paddle.imperative.grad( outputs=disc_interpolates, inputs=fake_AB, grad_outputs=outs, # paddle.ones(list(disc_interpolates.shape)), create_graph=True, retain_graph=True, only_inputs=True, # no_grad_vars=set(netD.parameters()) ) gradients = paddle.reshape(gradients[0], [real_data.shape[0], -1]) # flat the data gradient_penalty = paddle.reduce_mean( (paddle.norm(gradients + 1e-16, 2, 1) - constant)** 2) * lambda_gp # added eps return gradient_penalty, gradients else: return 0.0, None
def cal_gradient_penalty(self, netD, real_data, fake_data, edge_data=None, type='mixed', constant=1.0, lambda_gp=10.0): if lambda_gp > 0.0: if type == 'real': interpolatesv = real_data elif type == 'fake': interpolatesv = fake_data elif type == 'mixed': alpha = paddle.rand((real_data.shape[0], 1)) alpha = paddle.expand(alpha, [ real_data.shape[0], np.prod(real_data.shape) // real_data.shape[0] ]) alpha = paddle.reshape(alpha, real_data.shape) interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) else: raise NotImplementedError('{} not implemented'.format(type)) interpolatesv.stop_gradient = False real_data.stop_gradient = True fake_AB = paddle.concat((real_data.detach(), interpolatesv), 1) disc_interpolates = netD(fake_AB) outs = paddle.fill_constant(disc_interpolates.shape, disc_interpolates.dtype, 1.0) gradients = paddle.grad( outputs=disc_interpolates, inputs=fake_AB, grad_outputs=outs, create_graph=True, retain_graph=True, only_inputs=True) gradients = paddle.reshape(gradients[0], [real_data.shape[0], -1]) gradient_penalty = paddle.reduce_mean((paddle.norm( gradients + 1e-16, 2, 1) - constant)** 2) * lambda_gp # added eps return gradient_penalty, gradients else: return 0.0, None
def constant_(x, value): temp_value = paddle.fill_constant(x.shape, x.dtype, value) x.set_value(temp_value) return x
def margin_ranking_loss(input, other, label, margin=0.0, reduction='mean', name=None): """ This op the calcluate the the margin rank loss between the input, other and label, use the math function as follows. .. math:: margin\_rank\_loss = max(0, -label * (input - other) + margin) If :attr:`reduction` set to ``'mean'``, the reduced mean loss is: .. math:: Out = MEAN(margin\_rank\_loss) If :attr:`reduction` set to ``'sum'``, the reduced sum loss is: .. math:: Out = SUM(margin\_rank\_loss) If :attr:`reduction` set to ``'none'``, just return the origin ``margin_rank_loss``. Parameters: input(Tensor): the first input tensor, it's data type should be float32, float64. other(Tensor): the second input tensor, it's data type should be float32, float64. label(Tensor): the label value corresponding to input, it's data type should be float32, float64. margin (float, optional): The margin value to add, default value is 0; reduction (str, optional): Indicate the reduction to apply to the loss, the candicates are ``'none'``, ``'mean'``, ``'sum'``.If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned. If :attr:`reduction` is ``'sum'``, the reduced sum loss is returned. Default is ``'mean'``. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, if :attr:`reduction` is ``'mean'`` or ``'sum'``, the out shape is :math:`[1]`, otherwise the shape is the same as `input` .The same dtype as input tensor. Examples: .. code-block:: python import paddle paddle.disable_static() input = paddle.to_tensor([[1, 2], [3, 4]], dtype='float32') other = paddle.to_tensor([[2, 1], [2, 4]], dtype='float32') label = paddle.to_tensor([[1, -1], [-1, -1]], dtype='float32') loss = paddle.nn.functional.margin_ranking_loss(input, other, label) print(loss.numpy()) # [0.75] """ if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in MarginRankingLoss should be 'sum', 'mean' or 'none', but " "received %s, which is not allowed." % reduction) if fluid.framework.in_dygraph_mode(): out = core.ops.elementwise_sub(other, input) out = core.ops.elementwise_mul(out, label) if margin != 0.0: margin = fluid.dygraph.base.to_variable([margin], dtype=out.dtype) out = core.ops.elementwise_add(out, margin) out = core.ops.relu(out) if reduction == 'sum': return core.ops.reduce_sum(out, 'reduce_all', True) elif reduction == 'mean': return core.ops.mean(out) return out helper = LayerHelper("margin_ranking_loss", **locals()) fluid.data_feeder.check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'margin_rank_loss') fluid.data_feeder.check_variable_and_dtype(other, 'other', ['float32', 'float64'], 'margin_rank_loss') fluid.data_feeder.check_variable_and_dtype(label, 'label', ['float32', 'float64'], 'margin_rank_loss') out = paddle.elementwise_sub(other, input) out = paddle.multiply(out, label) if margin != 0.0: margin_var = out.block.create_var(dtype=out.dtype) paddle.fill_constant([1], out.dtype, margin, out=margin_var) out = paddle.add(out, margin_var) result_out = helper.create_variable_for_type_inference(input.dtype) if reduction == 'none': helper.append_op(type="relu", inputs={"X": out}, outputs={"Out": result_out}) return result_out elif reduction == 'sum': out = paddle.nn.functional.relu(out) attrs = {"dim": [0], "keep_dim": False, "reduce_all": True} helper.append_op(type="reduce_sum", inputs={"X": out}, outputs={"Out": result_out}, attrs=attrs) return result_out elif reduction == 'mean': out = paddle.nn.functional.relu(out) helper.append_op(type="mean", inputs={"X": out}, outputs={"Out": result_out}, attrs={}) return result_out
def binary_cross_entropy_with_logits(logit, label, weight=None, reduction='mean', pos_weight=None, name=None): """ This operator combines the sigmoid layer and the :ref:`api_nn_loss_BCELoss` layer. Also, we can see it as the combine of ``sigmoid_cross_entropy_with_logits`` layer and some reduce operations. This measures the element-wise probability error in classification tasks in which each class is independent. This can be thought of as predicting labels for a data-point, where labels are not mutually exclusive. For example, a news article can be about politics, technology or sports at the same time or none of these. First this operator calculate loss function as follows: .. math:: Out = -Labels * \\log(\\sigma(Logit)) - (1 - Labels) * \\log(1 - \\sigma(Logit)) We know that :math:`\\sigma(Logit) = \\frac{1}{1 + \\e^{-Logit}}`. By substituting this we get: .. math:: Out = Logit - Logit * Labels + \\log(1 + \\e^{-Logit}) For stability and to prevent overflow of :math:`\\e^{-Logit}` when Logit < 0, we reformulate the loss as follows: .. math:: Out = \\max(Logit, 0) - Logit * Labels + \\log(1 + \\e^{-\|Logit\|}) Then, if ``weight`` or ``pos_weight`` is not None, this operator multiply the weight tensor on the loss `Out`. The ``weight`` tensor will attach different weight on every items in the batch. The ``pos_weight`` will attach different weight on the positive label of each class. Finally, this operator applies reduce operation on the loss. If :attr:`reduction` set to ``'none'``, the operator will return the original loss `Out`. If :attr:`reduction` set to ``'mean'``, the reduced mean loss is :math:`Out = MEAN(Out)`. If :attr:`reduction` set to ``'sum'``, the reduced sum loss is :math:`Out = SUM(Out)`. Note that the target labels ``label`` should be numbers between 0 and 1. Args: logit (Tensor): The input predications tensor. 2-D tensor with shape: [N, *], N is batch_size, `*` means number of additional dimensions. The ``logit`` is usually the output of Linear layer. Available dtype is float32, float64. label (Tensor): The target labels tensor. 2-D tensor with the same shape as ``logit``. The target labels which values should be numbers between 0 and 1. Available dtype is float32, float64. weight (Tensor, optional): A manual rescaling weight given to the loss of each batch element. If given, it has to be a 1D Tensor whose size is `[N, ]`, The data type is float32, float64. Default is ``'None'``. reduction (str, optional): Indicate how to average the loss by batch_size, the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. If :attr:`reduction` is ``'none'``, the unreduced loss is returned; If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; If :attr:`reduction` is ``'sum'``, the summed loss is returned. Default is ``'mean'``. pos_weight (Tensor, optional): A weight of positive examples. Must be a vector with length equal to the number of classes. The data type is float32, float64. Default is ``'None'``. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: output (Tensor): If ``reduction`` is ``'none'``, the shape of output is same as ``logit`` , else the shape of output is scalar. Examples: .. code-block:: python import paddle paddle.disable_static() logit = paddle.to_tensor([5.0, 1.0, 3.0]) label = paddle.to_tensor([1.0, 0.0, 1.0]) output = paddle.nn.functional.binary_cross_entropy_with_logits(logit, label) print(output.numpy()) # [0.45618808] """ if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in binary_cross_entropy_with_logits " "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." % reduction) if in_dygraph_mode(): one = _varbase_creator(dtype=logit.dtype) core.ops.fill_constant(one, 'value', float(1.0), 'force_cpu', False, 'dtype', one.dtype, 'str_value', '1.0', 'shape', [1]) out = core.ops.sigmoid_cross_entropy_with_logits(logit, label) if pos_weight is not None: log_weight = core.ops.elementwise_add( core.ops.elementwise_mul( label, core.ops.elementwise_sub(pos_weight, one)), one) out = core.ops.elementwise_mul(out, log_weight) if weight is not None: out = core.ops.elementwise_mul(out, weight) if reduction == "sum": return core.ops.reduce_sum(out, 'reduce_all', True) elif reduction == "mean": return core.ops.mean(out) else: return out fluid.data_feeder.check_variable_and_dtype( logit, 'logit', ['float32', 'float64'], 'binary_cross_entropy_with_logits') fluid.data_feeder.check_variable_and_dtype( label, 'label', ['float32', 'float64'], 'binary_cross_entropy_with_logits') sigmoid_name = None if reduction == 'none' and pos_weight is None and weight is None: sigmoid_name = name out = paddle.nn.functional.sigmoid_cross_entropy_with_logits( logit, label, name=sigmoid_name) one = paddle.fill_constant(shape=[1], value=1.0, dtype=logit.dtype) if pos_weight is not None: fluid.data_feeder.check_variable_and_dtype( pos_weight, 'pos_weight', ['float32', 'float64'], 'binary_cross_entropy_with_logits') log_weight = paddle.add( paddle.multiply(label, paddle.elementwise_sub(pos_weight, one)), one) pos_weight_name = name if reduction == 'none' and weight is None else None out = paddle.multiply(out, log_weight, name=pos_weight_name) if weight is not None: fluid.data_feeder.check_variable_and_dtype( weight, 'weight', ['float32', 'float64'], 'binary_cross_entropy_with_logits') weight_name = name if reduction == 'none' else None out = paddle.multiply(out, weight, name=weight_name) if reduction == "sum": return paddle.sum(out, name=name) elif reduction == "mean": return paddle.mean(out, name=name) return out
def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): """ This op normalizes ``x`` along dimension ``axis`` using :math:`L_p` norm. This layer computes .. math:: y = \frac{x}{ \max\left( \lvert \lvert x \rvert \rvert_p, epsilon\right) } .. math:: \lvert \lvert x \rvert \rvert_p = \left(\sum_i {\lvert x_i\rvert^p} \right)^{1/p} where, :math:`\sum_i{\lvert x_i\rvert^p}` is calculated along the ``axis`` dimension. Args: x (Tensor): The input tensor could be N-D tensor, and the input data type could be float32 or float64. p (float|int, optional): The exponent value in the norm formulation. Default: 2 axis (int, optional): The axis on which to apply normalization. If `axis < 0`, the dimension to normalization is `x.ndim + axis`. -1 is the last dimension. epsilon (float, optional): Small float added to denominator to avoid dividing by zero. Default is 1e-12. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, the output has the same shape and data type with ``x``. Examples: .. code-block:: python import numpy as np import paddle import paddle.nn.functional as F paddle.disable_static() x = np.arange(6, dtype=np.float32).reshape(2,3) x = paddle.to_tensor(x) y = F.normalize(x) print(y.numpy()) # [[0. 0.4472136 0.8944272 ] # [0.42426404 0.5656854 0.7071067 ]] y = F.normalize(x, p=1.5) print(y.numpy()) # [[0. 0.40862012 0.81724024] # [0.35684016 0.4757869 0.5947336 ]] y = F.normalize(x, axis=0) print(y.numpy()) # [[0. 0.24253564 0.37139067] # [1. 0.97014254 0.9284767 ]] """ if in_dygraph_mode(): eps = fluid.dygraph.base.to_variable([epsilon], dtype=x.dtype) out = core.ops.p_norm(x, 'axis', axis, 'porder', float(p), 'keepdim', True, 'epsilon', epsilon) return x / core.ops.elementwise_max(out, eps) check_type(p, 'p', (float, int), 'normalize') check_type(axis, 'axis', (int), 'normalize') check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'normalize') if len(x.shape) == 1 and axis != 0 and axis != -1: raise ValueError( "Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {}" .format(axis)) attrs = { 'axis': axis, 'porder': float(p), 'keepdim': True, 'epsilon': epsilon, } helper = LayerHelper('p_norm', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) helper.append_op(type='p_norm', inputs={'X': x}, outputs={'Out': out}, attrs=attrs) eps = out.block.create_var(dtype=out.dtype) paddle.fill_constant([1], out.dtype, epsilon, out=eps) return paddle.elementwise_div(x, paddle.maximum(out, eps), name=name)