def __call__(self, shape, dtype=dtypes.float32, **kwargs):
"""Returns a tensor object initialized as specified by the initializer.
Args:
shape: Shape of the tensor.
dtype: Optional dtype of the tensor. Only floating point types are
supported.
**kwargs: Additional keyword arguments.
Raises:
ValueError: If the dtype is not floating point
"""
self._validate_kwargs(kwargs)
dtype = _assert_float_dtype(dtype)
scale = self.scale
fan_in, fan_out = _compute_fans(shape)
if _PARTITION_SHAPE in kwargs:
shape = kwargs[_PARTITION_SHAPE]
if self.mode == "fan_in":
scale /= max(1., fan_in)
elif self.mode == "fan_out":
scale /= max(1., fan_out)
else:
scale /= max(1., (fan_in + fan_out) / 2.)
if self.distribution == "truncated_normal":
# constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
stddev = math.sqrt(scale) / .87962566103423978
return self._random_generator.truncated_normal(shape, 0.0, stddev, dtype)
elif self.distribution == "untruncated_normal":
stddev = math.sqrt(scale)
return self._random_generator.random_normal(shape, 0.0, stddev, dtype)
else:
limit = math.sqrt(3.0 * scale)
return self._random_generator.random_uniform(shape, -limit, limit, dtype)
def test_he_uniform(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
self._runner(keras.initializers.he_uniform(seed=123), tensor_shape,
target_mean=0., target_std=std)
def test_lecun_normal(self):
tensor_shape = (5, 6, 4, 2)
with self.test_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
self._runner(keras.initializers.lecun_normal(seed=123), tensor_shape,
target_mean=0., target_std=std)
def test_glorot_normal(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, fan_out = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / (fan_in + fan_out))
self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape,
target_mean=0., target_std=std)
def test_he_uniform(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
self._runner(keras.initializers.he_uniform(seed=123), tensor_shape,
target_mean=0., target_std=std)
def test_glorot_normal(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, fan_out = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / (fan_in + fan_out))
self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape,
target_mean=0., target_std=std)
def test_he_normal(self):
tensor_shape = (5, 6, 4, 2)
with self.test_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
scale = np.sqrt(2. / fan_in)
self._runner(keras.initializers.he_normal(seed=123), tensor_shape,
target_mean=0., target_std=None, target_max=2 * scale)
def __call__(self, shape, dtype=None, partition_info=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(
self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = _compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters))
if self.criterion == 'glorot': # differnt weight initializers
s = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_real = modulus * np.cos(phase)
weight_imag = modulus * np.sin(phase)
weight = np.concatenate([weight_real, weight_imag], axis=-1)
return weight
def test_he_uniform(self):
tensor_shape = (5, 6, 4, 2)
with self.test_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
scale = np.sqrt(6. / fan_in)
self._runner(keras.initializers.he_uniform(seed=123), tensor_shape,
target_mean=0., target_max=scale, target_min=-scale)
def test_lecun_uniform(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
self._runner(init_ops.lecun_uniform(seed=123),
tensor_shape,
target_mean=0.,
target_std=std)
def __call__(self, shape, dtype=None, partition_info=None):
if dtype is None:
dtype = self.dtype
scale_shape = shape
if partition_info is not None:
scale_shape = partition_info.full_shape
fan_in, fan_out = _compute_fans(scale_shape)
return -tf.ones(shape, dtype=dtype) * fan_in * self.w / 2.0 + self.b
def test_lecun_uniform(self):
tensor_shape = (5, 6, 4, 2)
with self.cached_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
self._runner(
init_ops.lecun_uniform(seed=123),
tensor_shape,
target_mean=0.,
target_std=std)
def test_he_normal(self):
tensor_shape = (5, 6, 4, 2)
with self.test_session():
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
self._runner(
init_ops.he_normal(seed=123),
tensor_shape,
target_mean=0.,
target_std=std)
def test_he_normal(self):
shape = (5, 6, 4, 2)
with self.cached_session():
for tensor_shape in [shape, tensor_shape_lib.TensorShape(shape)]:
fan_in, _ = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
self._runner(init_ops.he_normal(seed=123),
tensor_shape,
target_mean=0.,
target_std=std)
def test_glorot_normal_initializer(self):
shape = (5, 6, 4, 2)
with self.cached_session():
for tensor_shape in [shape, tensor_shape_lib.TensorShape(shape)]:
fan_in, fan_out = init_ops._compute_fans(tensor_shape)
std = np.sqrt(2. / (fan_in + fan_out))
self._runner(init_ops.glorot_normal_initializer(seed=123),
tensor_shape,
target_mean=0.,
target_std=std)
def _random_modulus_and_phase(self, shape):
shape = to_int_shape(shape)
fan_in, fan_out = _compute_fans(shape)
if self.criterion == 'glorot':
scale = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
scale = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
self.modulus = random_rayleigh(shape, scale=scale)
self.phase = tensorflow.random_uniform(shape,
minval=-math.pi,
maxval=math.pi)
def __call__(self, shape, dtype=None, partition_info=None):
if dtype is None:
dtype = self.dtype
scale = self.scale
scale_shape = shape
if partition_info is not None:
scale_shape = partition_info.full_shape
fan_in, fan_out = _compute_fans(scale_shape)
if self.mode == "fan_in":
scale /= max(1., fan_in)
elif self.mode == "fan_out":
scale /= max(1., fan_out)
else:
scale /= max(1., (fan_in + fan_out) / 2.)
#if self.distribution == "normal" or self.distribution == "truncated_normal":
# constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
stddev = tf.sqrt(scale) / .87962566103423978
#if self.scale == 1:
if scale == 1:
return tf.random.truncated_normal(shape,
0.0,
stddev,
dtype,
seed=self.seed)
#logging.info(scale)
new_shape = shape[:3] + (shape[3] // (self.scale**2), )
x = tf.random.truncated_normal(new_shape,
0.0,
stddev,
dtype,
seed=self.seed)
x = tf.transpose(x, perm=[2, 0, 1, 3])
x = tf.image.resize_nearest_neighbor(x,
size=(shape[0] * self.scale,
shape[1] * self.scale))
#x = tf.space_to_depth(x, block_size=scale)
x = tf.space_to_depth(x, block_size=self.scale)
x = tf.transpose(x, perm=[1, 2, 0, 3])
return x
def __call__(self, shape, dtype=None, partition_info=None):
if partition_info is not None:
raise ValueError("partition_info not supported.")
if dtype is None:
dtype = self.dtype
if len(shape) != 2:
raise ValueError("Weights must be 2-dimensional.")
fan_in, fan_out = init_ops._compute_fans(shape) # pylint: disable=protected-access
# Calculate the number of non-zero weights in the weight matrix
nnz = 1.0
for d in shape:
nnz *= d
nnz *= 1 - self.sparsity
limit = math.sqrt(6.0 / (nnz / fan_out + nnz / fan_in))
return tf.random_uniform(shape, -limit, limit, dtype, seed=self.seed)
def __call__(self, shape, dtype=dtypes.float32):
"""Returns a tensor object initialized as specified by the initializer.
Args:
shape: Shape of the tensor.
dtype: Optional dtype of the tensor. Only floating point types are
supported.
Raises:
ValueError: If the dtype is not floating point
"""
partition_info = None # Keeps logic so can be readded later if necessary
dtype = _assert_float_dtype(dtype)
scale = self.scale
scale_shape = shape
if partition_info is not None:
scale_shape = partition_info.full_shape
fan_in, fan_out = _compute_fans(scale_shape)
if self.mode == "fan_in":
scale /= max(1., fan_in)
elif self.mode == "fan_out":
scale /= max(1., fan_out)
else:
scale /= max(1., (fan_in + fan_out) / 2.)
if self.distribution == "truncated_normal":
# constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
stddev = math.sqrt(scale) / .87962566103423978
return self._random_generator.truncated_normal(
shape, 0.0, stddev, dtype)
elif self.distribution == "untruncated_normal":
stddev = math.sqrt(scale)
return self._random_generator.random_normal(
shape, 0.0, stddev, dtype)
else:
limit = math.sqrt(3.0 * scale)
return self._random_generator.random_uniform(
shape, -limit, limit, dtype)
def __call__(self, shape, dtype=None, partition_info=None):
# Note here, shape refers to dims not distribution shape parameter!
# We will switch in one line:
dims = shape if partition_info is None else partition_info.full_shape
if dtype is None:
dtype = self.dtype
if not dtype.is_floating:
raise TypeError("Only floating data types supported.")
# Calculate denominator
ninp, nout = _compute_fans(dims)
n = None
if self.fan == 'in':
n = max(1., ninp)
elif self.fan == 'out':
n = max(1., nout)
elif self.fan == 'avg':
n = max(1., 0.5 * (ninp + nout))
# Random number generator
rand = None
stdev = math.sqrt(2. / n) # a la he2015delving
if self.distr == 'unif':
limit = stdev * math.sqrt(3.)
rand = random_ops.uniform(dims,
-limit,
limit,
self.dtype,
seed=self.seed)
elif self.distr == 'norm':
rand = random_ops.random_normal(dims,
0.,
stdev,
self.dtype,
seed=self.seed)
elif self.distr == 'trun': # -2SD to +2SD truncated normal has a SD of 1./1.137
rand = random_ops.truncated_normal(dims,
0.,
1.137 * stdev,
self.dtype,
seed=self.seed)
elif self.distr == 'gamma':
gamma = self.shape
invlb = math.sqrt(gamma) / stdev
rand = random_ops.random_gamma(
dims, gamma, invlb, self.dtype,
seed=self.seed) - (gamma / invlb)
elif self.distr == 'beta':
gamma = self.shape
rand_0 = random_ops.random_gamma(dims,
gamma,
1.,
self.dtype,
seed=self.seed)
rand_1 = random_ops.random_gamma(dims,
gamma,
1.,
self.dtype,
seed=self.seed)
rand = stdev * (
(rand_0 /
(rand_0 + rand_1)) - 0.5) * math.sqrt(8. * self.shape + 4.)
# Scale and offset as required
randscaled = rand if self.scale == 1. else rand * self.scale
randoffset = randscaled if self.locan == 0. else randscaled + (
self.locan / n)
return randoffset
