def test_truncated_normal_initializer_default_value(self):
"""Test the truncated normal initializer with default value
"""
paddle.enable_static()
program = framework.Program()
block = program.global_block()
for _ in range(2):
block.create_parameter(dtype="float32",
shape=[5, 10],
lod_level=0,
name="param",
initializer=initializer.TruncatedNormal())
self.assertEqual(len(block.ops), 1)
init_op = block.ops[0]
self.assertEqual(init_op.type, 'truncated_gaussian_random')
self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA)
self.assertAlmostEqual(init_op.attr('std'), 1.0, delta=DELTA)
self.assertEqual(init_op.attr('seed'), 0)
paddle.disable_static()
def test_truncated_normal_initializer(self, dtype="float32"):
"""Test truncated normal initializer with supplied attributes
"""
paddle.enable_static()
program = framework.Program()
block = program.global_block()
for _ in range(2):
block.create_parameter(dtype=dtype,
shape=[5, 10],
lod_level=0,
name="param",
initializer=initializer.TruncatedNormal(
2.3, 1.9))
num_ops = 2 if dtype in ["float16", "uint16"] else 1
self.assertEqual(len(block.ops), num_ops)
init_op = block.ops[0]
self.assertEqual(init_op.type, 'truncated_gaussian_random')
self.assertAlmostEqual(init_op.attr('mean'), 2.3, delta=DELTA)
self.assertAlmostEqual(init_op.attr('std'), 1.9, delta=DELTA)
paddle.disable_static()
return block
shape = (paddle.shape(x)[0], ) + (1, ) * (x.ndim - 1)
random_tensor = keep_prob + paddle.rand(shape, dtype=x.dtype)
random_tensor = paddle.floor(random_tensor) # binarize
output = x.divide(keep_prob) * random_tensor
return output
class DropPath(nn.Layer):
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
"""
def __init__(self, drop_prob=None):
super(DropPath, self).__init__()
self.drop_prob = drop_prob
def forward(self, x):
return drop_path(x, self.drop_prob, self.training)
class Identity(nn.Layer):
def __init__(self):
super(Identity, self).__init__()
def forward(self, input):
return input
trunc_normal_ = paddle_init.TruncatedNormal(std=.02)
zeros_ = paddle_init.Constant(value=0.)
ones_ = paddle_init.Constant(value=1.)