def test_xavier_initializer_supplied_arguments(self, dtype="float32", uniform=True): """Test the Xavier initializer with supplied arguments """ program = framework.Program() block = program.global_block() for _ in range(2): block.create_parameter( dtype=dtype, shape=[5, 10], lod_level=0, name="param", initializer=initializer.XavierInitializer( uniform=uniform, fan_in=12, fan_out=23, seed=134)) num_ops = 2 if (dtype == "float16" or (dtype == "uint16" and not uniform)) else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] if uniform: self.assertEqual(init_op.type, 'uniform_random') limit = np.sqrt(6.0 / (12 + 23)) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) else: self.assertEqual(init_op.type, 'gaussian_random') self.assertEqual(init_op.attr('seed'), 134) return block
def test_xavier_initializer_supplied_arguments(self): """Test the Xavier initializer with supplied arguments """ program = framework.Program() block = program.global_block() block.create_parameter(dtype="float32", shape=[5, 10], lod_level=0, name="param", initializer=initializer.XavierInitializer( fan_in=12, fan_out=23, seed=134)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') limit = np.sqrt(6.0 / (12 + 23)) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 134)
def test_normal_xavier_initializer(self): """Test Xavier initializer with normal distribution on for matrix multiply. """ program = framework.Program() block = program.global_block() param = block.create_parameter( dtype="float32", shape=[5, 10], lod_level=0, name="param", initializer=initializer.XavierInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') std = np.sqrt(2.0 / (param.shape[0] + param.shape[1])) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0)
def test_uniform_xavier_initializer(self): """Test Xavier initializer with uniform distribution on for matrix multiply. """ program = framework.Program() block = program.global_block() param = block.create_parameter( dtype="float32", shape=[5, 10], lod_level=0, name="param", initializer=initializer.XavierInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') limit = np.sqrt(6.0 / (param.shape[0] + param.shape[1])) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0)
def test_normal_xavier_initializer_conv(self): """Test Xavier initializer with normal distribution on for convolutions. """ program = framework.Program() block = program.global_block() for _ in range(2): param = block.create_parameter( dtype="float32", shape=[5, 10, 15, 20], lod_level=0, name="param", initializer=initializer.XavierInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') receptive_field_size = float(15 * 20) std = np.sqrt( 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0)