def __call__(self, shape, dtype=None):
from keras.initializers import _compute_fans
fan_in, fan_out = _compute_fans(shape)
scale = self.scale
if self.mode == 'fan_in':
scale /= max(1., fan_in)
elif self.mode == 'fan_out':
scale /= max(1., fan_out)
else:
scale /= max(1., float(fan_in + fan_out) / 2)
if self.distribution == 'normal':
# 0.879... = scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
stddev = np.sqrt(scale) / .87962566103423978
init = K.truncated_normal(shape,
0.,
stddev,
dtype=dtype,
seed=self.seed)
else:
limit = np.sqrt(3. * scale)
init = K.random_uniform(shape,
-limit,
limit,
dtype=dtype,
seed=self.seed)
return init * self.tree_weight
def build(self, input_shape):
assert len(input_shape) == 2
assert input_shape[-1] % 2 == 0
input_dim = input_shape[-1] // 4
data_format = K.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
if self.init_criterion == 'he':
s = np.sqrt(1. / fan_in)
elif self.init_criterion == 'glorot':
s = np.sqrt(1. / (fan_in + fan_out))
rng = RandomStreams(seed=self.seed)
# Initialization using euclidean representation:
def init_w_real(shape, dtype=None):
return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype)
def init_w_imag(shape, dtype=None):
return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype)
if self.kernel_initializer in {'quaternion'}:
real_init = init_w_real
imag_init = init_w_imag
else:
real_init = self.kernel_initializer
imag_init = self.kernel_initializer
self.r = self.add_weight(shape=kernel_shape,
initializer=real_init,
name='r',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.i = self.add_weight(shape=kernel_shape,
initializer=imag_init,
name='i',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.j = self.add_weight(shape=kernel_shape,
initializer=imag_init,
name='j',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.k = self.add_weight(shape=kernel_shape,
initializer=imag_init,
name='k',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(4 * self.units, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.input_spec = InputSpec(ndim=2, axes={-1: 4 * input_dim})
self.built = True
def build(self, input_shape):
assert len(input_shape) == 2
assert input_shape[-1] % 2 == 0
input_dim = input_shape[-1] // 2
data_format = K.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
if self.init_criterion == 'he':
s = K.sqrt(1. / fan_in)
elif self.init_criterion == 'glorot':
s = K.sqrt(1. / (fan_in + fan_out))
rng = RandomStreams(seed=self.seed)
def init_theta(shape, dtype=None):
return rng.uniform(size=kernel_shape, low=0, high=6)
if self.kernel_initializer in {'complex'}:
theta_init = init_theta
else:
raise ValueError('Not recognized choice of initialization!')
# Defining layer parameters (Codebook):
self.theta = self.add_weight(shape=kernel_shape,
initializer=theta_init,
name='theta_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.real_kernel = (1 / self.scale) * K.cos(self.theta) #
self.imag_kernel = (1 / self.scale) * K.sin(self.theta) #
self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim})
self.built = True
def test_lecun_normal(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
_runner(initializers.lecun_normal(),
tensor_shape,
target_mean=0.,
target_std=std)
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
)
if self.criterion == 'glorot':
s = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(1337)
modulus = rng.uniform(low=-np.sqrt(s)*np.sqrt(3), high=np.sqrt(s)*np.sqrt(3), size=kernel_shape)
phase = rng.uniform(low=-np.pi/2, high=np.pi/2, size=kernel_shape)
wm = modulus
wp = phase
weight = np.concatenate([wp, wm], axis=-1)
return weight
def test_glorot_uniform(tensor_shape):
fan_in, fan_out = initializers._compute_fans(tensor_shape)
std = np.sqrt(2. / (fan_in + fan_out))
_runner(initializers.glorot_uniform(),
tensor_shape,
target_mean=0.,
target_std=std)
def test_he_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
_runner(initializers.he_uniform(),
tensor_shape,
target_mean=0.,
target_std=std)
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
)
if self.criterion == 'glorot':
s = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_real = modulus * np.cos(phase)
weight_imag = modulus * np.sin(phase)
weight = np.concatenate([weight_real, weight_imag], axis=-1)
return weight
def glorot_init(shape, seed=0):
assert len(shape) == 2
from keras.initializers import _compute_fans
fan_in, fan_out = _compute_fans(shape)
scale = 1 / max(1., float(fan_in + fan_out) / 2)
limit = np.sqrt(3. * scale)
return np.random.uniform(-limit, limit, shape)
def build(self, input_shape):
assert len(input_shape) == 2
assert input_shape[-1] % 2 == 0
input_dim = input_shape[-1] // 2
data_format = K.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
if self.init_criterion == 'he':
s = K.sqrt(1. / fan_in)
elif self.init_criterion == 'glorot':
s = K.sqrt(1. / (fan_in + fan_out))
rng = RandomStreams(seed=self.seed)
# Equivalent initialization using amplitude phase representation:
"""modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
def init_w_real(shape, dtype=None):
return modulus * K.cos(phase)
def init_w_imag(shape, dtype=None):
return modulus * K.sin(phase)"""
# Initialization using euclidean representation:
def init_w_real(shape, dtype=None):
return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype)
def init_w_imag(shape, dtype=None):
return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype)
if self.kernel_initializer in {'complex'}:
real_init = init_w_real
imag_init = init_w_imag
else:
real_init = self.kernel_initializer
imag_init = self.kernel_initializer
self.real_kernel = self.add_weight(shape=kernel_shape,
initializer=real_init,
name='real_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.imag_kernel = self.add_weight(shape=kernel_shape,
initializer=imag_init,
name='imag_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(2 * self.units, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim})
self.built = True
def __call__(self, shape):
rank = len(shape)
if self.seed is not None:
np.random.seed(self.seed)
fan_in, fan_out = _compute_fans(shape, K.image_data_format())
variance = 2 / fan_in
if rank == 3:
row, stack_size, filters_size = shape
transpose_dimensions = (2, 1, 0)
kernel_shape = (row, )
correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0])
correct_fft = np.fft.rfft
elif rank == 4:
row, column, stack_size, filters_size = shape
transpose_dimensions = (2, 3, 0, 1)
kernel_shape = (row, column)
correct_ifft = np.fft.irfft2
correct_fft = np.fft.rfft2
elif rank == 5:
x, y, z, stack_size, filters_size = shape
transpose_dimensions = (3, 4, 0, 1, 2)
kernel_shape = (x, y, z)
correct_fft = np.fft.rfftn
correct_ifft = np.fft.irfftn
else:
return K.variable(self.orthogonal(shape), dtype=K.floatx())
kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape
init = []
for i in range(filters_size):
basis = self._create_basis(stack_size,
np.prod(kernel_fourier_shape))
basis = basis.reshape((stack_size, ) + kernel_fourier_shape)
filters = [
correct_ifft(x, kernel_shape) +
np.random.normal(0, self.eps_std, kernel_shape) for x in basis
]
init.append(filters)
# Format of array is now: filters, stack, row, column
init = np.array(init)
init = self._scale_filters(init, variance)
return init.transpose(transpose_dimensions)
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
num_rows = self.nb_filters * self.input_dim
num_cols = np.prod(self.kernel_size)
else:
# in case it is the kernel is a matrix and not a filter
# which is the case of 2D input (No feature maps).
num_rows = self.input_dim
num_cols = self.kernel_size[-1]
flat_shape = (num_rows, num_cols)
rng = RandomState(self.seed)
x = rng.uniform(size=flat_shape)
u, _, v = np.linalg.svd(x)
orthogonal_x = np.dot(u, np.dot(np.eye(num_rows, num_cols), v.T))
if self.nb_filters is not None:
independent_filters = np.reshape(orthogonal_x, (num_rows, ) +
tuple(self.kernel_size))
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters))
else:
independent_filters = orthogonal_x
fan_in, fan_out = (self.input_dim, self.kernel_size[-1])
if self.criterion == 'glorot':
desired_var = 2. / (fan_in + fan_out)
elif self.criterion == 'he':
desired_var = 2. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
multip_constant = np.sqrt(desired_var / np.var(independent_filters))
scaled_indep = multip_constant * independent_filters
if self.weight_dim == 2 and self.nb_filters is None:
weight_real = scaled_real
weight_imag = scaled_imag
else:
kernel_shape = tuple(
self.kernel_size) + (self.input_dim, self.nb_filters)
if self.weight_dim == 1:
transpose_shape = (1, 0)
elif self.weight_dim == 2 and self.nb_filters is not None:
transpose_shape = (1, 2, 0)
elif self.weight_dim == 3 and self.nb_filters is not None:
transpose_shape = (1, 2, 3, 0)
weight = np.transpose(scaled_indep, transpose_shape)
weight = np.reshape(weight, kernel_shape)
return weight
def test_convolution_aware(tensor_shape):
""" Convolution Aware Initialization Test
Parameters
----------
tensor_shape: tuple
The shape of the tensor to feed to the initializer
"""
fan_in, _ = k_initializers._compute_fans(tensor_shape) # pylint:disable=protected-access
std = np.sqrt(2. / fan_in)
_runner(initializers.ConvolutionAware(seed=123, init=True),
tensor_shape,
target_mean=0,
target_std=std)
def glorot_uniform_sigm(shape):
"""
Glorot style weight initializer for sigmoid activations.
Like keras.initializations.glorot_uniform(), but with uniform random interval like in
Deeplearning.net tutorials.
They claim that the initialization random interval should be
+/- sqrt(6 / (fan_in + fan_out)) (like Keras' glorot_uniform()) when tanh activations are used,
+/- 4 sqrt(6 / (fan_in + fan_out)) when sigmoid activations are used.
See: http://deeplearning.net/tutorial/mlp.html#going-from-logistic-regression-to-mlp
"""
fan_in, fan_out = _compute_fans(shape)
s = 4. * np.sqrt(6. / (fan_in + fan_out))
return uniform(shape, s)
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(
self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters))
if self.criterion == 'glorot':
s = 1. / np.sqrt(2 * (fan_in + fan_out))
elif self.criterion == 'he':
s = 1. / np.sqrt(2 * fan_in)
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = Chi4Random(s)
flat_size = np.product(kernel_shape)
modulus = rng.random(N=flat_size)
modulus = modulus.reshape(kernel_shape)
rng = RandomState(self.seed)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
# must make random unit vector for quaternion vector
def make_rand_vector(dims):
vec = [gauss(0, 1) for i in range(dims)]
mag = sum(x**2 for x in vec)**0.5
return [x / mag for x in vec]
u_i = np.zeros(flat_size)
u_j = np.zeros(flat_size)
u_k = np.zeros(flat_size)
for u in range(flat_size):
unit = make_rand_vector(3)
u_i[u] = unit[0]
u_j[u] = unit[1]
u_k[u] = unit[2]
u_i = u_i.reshape(kernel_shape)
u_j = u_j.reshape(kernel_shape)
u_k = u_k.reshape(kernel_shape)
weight_r = modulus * np.cos(phase)
weight_i = modulus * u_i * np.sin(phase)
weight_j = modulus * u_j * np.sin(phase)
weight_k = modulus * u_k * np.sin(phase)
weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
axis=-1)
return weight
def test_icnr(tensor_shape):
""" ICNR Initialization Test
Parameters
----------
tensor_shape: tuple
The shape of the tensor to feed to the initializer
"""
fan_in, _ = k_initializers._compute_fans(tensor_shape) # pylint:disable=protected-access
std = np.sqrt(2. / fan_in)
_runner(initializers.ICNR(initializer=k_initializers.he_uniform(),
scale=2),
tensor_shape,
target_mean=0,
target_std=std)
def build(self, input_shape):
assert len(input_shape) == 2
assert input_shape[-1] % 2 == 0
input_dim = input_shape[-1] // 3
data_format = K.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
s = np.sqrt(1. / fan_in)
def init_phase(shape, dtype=None):
return np.random.normal(
size=kernel_shape,
loc=0,
scale=np.pi / 2,
)
def init_modulus(shape, dtype=None):
return np.random.normal(size=kernel_shape, loc=0, scale=s)
phase_init = init_phase
modulus_init = init_modulus
self.phase_kernel = self.add_weight(
shape=kernel_shape,
initializer=phase_init,
name='phase_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.modulus_kernel = self.add_weight(
shape=kernel_shape,
initializer=modulus_init,
name='modulus_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(3 * self.units, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.input_spec = InputSpec(ndim=2, axes={-1: 3 * input_dim})
self.built = True
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(
self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters))
# Quaternion operations start here
if self.criterion == 'glorot':
s = 1. / np.sqrt(2 * (fan_in + fan_out))
elif self.criterion == 'he':
s = 1. / np.sqrt(2 * fan_in)
else:
raise ValueError('Invalid criterion: ' + self.criterion)
#Generating randoms and purely imaginary quaternions :
number_of_weights = np.prod(kernel_shape)
v_i = np.random.uniform(0.0, 1.0, number_of_weights)
v_j = np.random.uniform(0.0, 1.0, number_of_weights)
v_k = np.random.uniform(0.0, 1.0, number_of_weights)
#Make these purely imaginary quaternions unitary
for i in range(0, number_of_weights):
norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2)
v_i[i] /= norm
v_j[i] /= norm
v_k[i] /= norm
v_i = v_i.reshape(kernel_shape)
v_j = v_j.reshape(kernel_shape)
v_k = v_k.reshape(kernel_shape)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_r = modulus * np.cos(phase)
weight_i = modulus * v_i * np.sin(phase)
weight_j = modulus * v_j * np.sin(phase)
weight_k = modulus * v_k * np.sin(phase)
weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
axis=-1)
return weight
def __call__(self, shape, dtype=None):
# The only difference between this and the built-in VarianceScaling is that
# we multiply element-wise by our adjacency matrix as the final step.
fan_in, fan_out = _compute_fans(shape)
scale = self.scale
if self.mode == 'fan_in':
scale /= max(1., fan_in)
elif self.mode == 'fan_out':
scale /= max(1., fan_out)
else:
scale /= max(1., float(fan_in + fan_out) / 2)
final = None
if self.distribution == 'normal':
mean = 0.0
stddev = np.sqrt(scale)
normal_mat = np.random.normal(loc=mean, scale=stddev, size=shape)
clipped = np.clip(normal_mat, mean - 2 * stddev, mean + 2 * stddev)
return clipped * self.adjacency_mat
else:
limit = np.sqrt(3. * scale)
dense_initial = np.random.uniform(low=-limit,
high=limit,
size=shape)
return dense_initial * self.adjacency_mat
def test_lecun_normal(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
scale = np.sqrt(1. / fan_in)
_runner(initializers.lecun_normal(), tensor_shape,
target_mean=0., target_std=scale)
def build(self, input_shape):
assert len(input_shape) == 2
# We have both real and imaginary components so the last dimension must be even
assert input_shape[-1] % 2 == 0
# Remember: the input is [real, imaginary], so what looks like a 6 unit input is actually 3
input_dim = input_shape[-1] // 2
data_format = kb.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
if self.init_criterion == 'he':
s = np.sqrt(1. / fan_in)
elif self.init_criterion == 'glorot':
s = np.sqrt(1. / (fan_in + fan_out))
else:
s = 1.0
rng = np.random.RandomState(seed=self.seed)
def init_w_real(shape, dtype=None):
return rng.normal(loc=0.0, scale=s, size=kernel_shape)
def init_w_imag(shape, dtype=None):
return rng.normal(loc=0.0, scale=s, size=kernel_shape)
if self.kernel_initializer in {'complex'}:
real_init = init_w_real
imag_init = init_w_imag
else:
real_init = self.kernel_initializer
imag_init = self.kernel_initializer
self.real_kernel = self.add_weight(name='real_kernel',
shape=kernel_shape,
initializer=real_init,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.imag_kernel = self.add_weight(name='imag_kernel',
shape=kernel_shape,
initializer=imag_init,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.kernel = kb.concatenate([
kb.concatenate([self.real_kernel, -self.imag_kernel], axis=-1),
kb.concatenate([self.imag_kernel, self.real_kernel], axis=-1)
],
axis=0)
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(2 * self.units, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim})
self.built = True
def build(self, input_shape):
# assert len(input_shape) == 2
# assert input_shape[-1] % 2 == 0
# input_dim = input_shape[-1] // 2
if not isinstance(input_shape, list):
raise ValueError('This layer should be called '
'on a list of 2 inputs.')
if len(input_shape) != 2:
raise ValueError('This layer should be called '
'on a list of 2 inputs.'
'Got ' + str(len(input_shape)) + ' inputs.')
input_dim = input_shape[0][-1]
self.input_dim = input_dim
data_format = K.image_data_format()
kernel_shape = (input_dim, 2 * self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
fan_in = tf.to_float(fan_in)
fan_out = tf.to_float(fan_out)
if self.init_criterion == 'he':
s = K.sqrt(1. / fan_in)
elif self.init_criterion == 'glorot':
s = K.sqrt(1. / (fan_in + fan_out))
# rng = RandomStreams(seed=self.seed)
# Equivalent initialization using amplitude phase representation:
"""modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
def init_w_real(shape, dtype=None):
return modulus * K.cos(phase)
def init_w_imag(shape, dtype=None):
return modulus * K.sin(phase)"""
# Initialization using euclidean representation:
def init_w(shape, dtype=None):
return tf.random_normal(shape=kernel_shape,
mean=0.0,
stddev=s,
dtype=tf.float32,
seed=self.seed,
name=None)
# return rng.normal(
# size=kernel_shape,
# avg=0,
# std=s,
# dtype=dtype
# )
if self.kernel_initializer in {'complex'}:
weight_init = init_w
else:
weight_init = self.kernel_initializer
self.kernel = self.add_weight(shape=kernel_shape,
initializer=weight_init,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(2 * self.units, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
# self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim})
self.built = True
def test_he_normal(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(2. / fan_in)
_runner(initializers.he_normal(), tensor_shape,
target_mean=0., target_std=std)
def test_glorot_normal(tensor_shape):
fan_in, fan_out = initializers._compute_fans(tensor_shape)
std = np.sqrt(2. / (fan_in + fan_out))
_runner(initializers.glorot_normal(), tensor_shape,
target_mean=0., target_std=std)
def test_lecun_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
_runner(initializers.lecun_uniform(), tensor_shape,
target_mean=0., target_std=std)
def __call__(self, shape, dtype=None):
dtype = K.floatx() if dtype is None else dtype
if self._init:
logger.info("Calculating Convolution Aware Initializer for shape: %s", shape)
else:
logger.debug("Bypassing Convolutional Aware Initializer for saved model")
# Dummy in he_uniform just in case there aren't any weighs being loaded
# and it needs some kind of initialization
return self.he_uniform(shape, dtype=dtype)
rank = len(shape)
if self.seed is not None:
np.random.seed(self.seed)
#计算输入范围的数量
fan_in, _ = initializers._compute_fans(shape) # pylint:disable=protected-access
variance = 2 / fan_in
if rank == 3:
row, stack_size, filters_size = shape
transpose_dimensions = (2, 1, 0)
kernel_shape = (row,)
correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0]) # noqa
correct_fft = np.fft.rfft
elif rank == 4:
row, column, stack_size, filters_size = shape
transpose_dimensions = (2, 3, 1, 0)
kernel_shape = (row, column)
correct_ifft = np.fft.irfft2
correct_fft = np.fft.rfft2
elif rank == 5:
var_x, var_y, var_z, stack_size, filters_size = shape
transpose_dimensions = (3, 4, 0, 1, 2)
kernel_shape = (var_x, var_y, var_z)
correct_fft = np.fft.rfftn
correct_ifft = np.fft.irfftn
else:
return K.variable(self.orthogonal(shape), dtype=dtype)
kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape
init = []
for _ in range(filters_size):
basis = self._create_basis(
stack_size, np.prod(kernel_fourier_shape), dtype)
basis = basis.reshape((stack_size,) + kernel_fourier_shape)
filters = [correct_ifft(x, kernel_shape) +
np.random.normal(0, self.eps_std, kernel_shape) for
x in basis]
init.append(filters)
# Format of array is now: filters, stack, row, column
init = np.array(init)
init = self._scale_filters(init, variance)
return K.variable(init.transpose(transpose_dimensions), dtype=dtype, name="conv_aware")
def test_he_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
scale = np.sqrt(6. / fan_in)
_runner(initializers.he_uniform(), tensor_shape,
target_mean=0., target_max=scale, target_min=-scale)
def _dense_kernel_initializer(shape, dtype=None):
fan_in, fan_out = _compute_fans(shape)
stddev = 1. / np.sqrt(fan_in)
return K.random_uniform(shape, -stddev, stddev, dtype)
def test_glorot_uniform(tensor_shape):
fan_in, fan_out = initializers._compute_fans(tensor_shape)
scale = np.sqrt(6. / (fan_in + fan_out))
_runner(initializers.glorot_uniform(), tensor_shape,
target_mean=0., target_max=scale, target_min=-scale)
def __call__(self, shape, dtype=None):
""" Call function for the ICNR initializer.
Parameters
----------
shape: tuple or list
The required shape for the output tensor
dtype: str
The data type for the tensor
Returns
-------
tensor
The modified kernel weights
"""
dtype = K.floatx() if dtype is None else dtype
if self._init:
logger.info(
"Calculating Convolution Aware Initializer for shape: %s",
shape)
else:
logger.debug(
"Bypassing Convolutional Aware Initializer for saved model")
# Dummy in he_uniform just in case there aren't any weighs being loaded
# and it needs some kind of initialization
return self.he_uniform(shape, dtype=dtype)
rank = len(shape)
if self.seed is not None:
np.random.seed(self.seed)
fan_in, _ = initializers._compute_fans(shape) # pylint:disable=protected-access
variance = 2 / fan_in
if rank == 3:
row, stack_size, filters_size = shape
transpose_dimensions = (2, 1, 0)
kernel_shape = (row, )
correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0]
) # noqa
correct_fft = np.fft.rfft
elif rank == 4:
row, column, stack_size, filters_size = shape
transpose_dimensions = (2, 3, 1, 0)
kernel_shape = (row, column)
correct_ifft = np.fft.irfft2
correct_fft = np.fft.rfft2
elif rank == 5:
var_x, var_y, var_z, stack_size, filters_size = shape
transpose_dimensions = (3, 4, 0, 1, 2)
kernel_shape = (var_x, var_y, var_z)
correct_fft = np.fft.rfftn
correct_ifft = np.fft.irfftn
else:
return K.variable(self.orthogonal(shape), dtype=dtype)
kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape
basis = self._create_basis(filters_size, stack_size,
np.prod(kernel_fourier_shape), dtype)
basis = basis.reshape((
filters_size,
stack_size,
) + kernel_fourier_shape)
randoms = np.random.normal(0, self.eps_std,
basis.shape[:-2] + kernel_shape)
init = correct_ifft(basis, kernel_shape) + randoms
init = self._scale_filters(init, variance)
return K.variable(init.transpose(transpose_dimensions),
dtype=dtype,
name="conv_aware")
def test_he_normal(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
scale = np.sqrt(2. / fan_in)
_runner(initializers.he_normal(), tensor_shape,
target_mean=0., target_std=None, target_max=2 * scale)
def build(self, input_shape):
assert len(input_shape) == 2
assert input_shape[-1] % 4 == 0
input_dim = input_shape[-1] // 4
data_format = K.image_data_format()
kernel_shape = (input_dim, self.units)
fan_in, fan_out = initializers._compute_fans(kernel_shape,
data_format=data_format)
if self.init_criterion == 'he':
s = tf.sqrt(tf.cast(1. / fan_in, tf.float32))
elif self.init_criterion == 'glorot':
s = tf.sqrt(tf.cast(1. / (fan_in + fan_out), tf.float32))
# Equivalent initialization using amplitude phase representation:
"""modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
def init_w_real(shape, dtype=None):
return modulus * K.cos(phase)
def init_w_imag(shape, dtype=None):
return modulus * K.sin(phase)"""
# Initialization using euclidean representation:
def init_w_r(shape, dtype=None):
return K.random_normal_variable(kernel_shape,
mean=0,
scale=s,
dtype=tf.float32,
seed=self.seed)
def init_w_i(shape, dtype=None):
return K.random_normal_variable(kernel_shape,
mean=0,
scale=s,
dtype=tf.float32,
seed=self.seed)
def init_w_j(shape, dtype=None):
return K.random_normal_variable(kernel_shape,
mean=0,
scale=s,
dtype=tf.float32,
seed=self.seed)
def init_w_k(shape, dtype=None):
return K.random_normal_variable(kernel_shape,
mean=0,
scale=s,
dtype=tf.float32,
seed=self.seed)
if self.kernel_initializer in {'quaternion'}:
r_init = init_w_r
i_init = init_w_i
j_init = init_w_j
k_init = init_w_k
else:
r_init = self.kernel_initializer
i_init = self.kernel_initializer
j_init = self.kernel_initializer
k_init = self.kernel_initializer
self.r_kernel = self.add_weight(shape=kernel_shape,
initializer=r_init,
name='r_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.i_kernel = self.add_weight(shape=kernel_shape,
initializer=i_init,
name='i_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.j_kernel = self.add_weight(shape=kernel_shape,
initializer=j_init,
name='j_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.k_kernel = self.add_weight(shape=kernel_shape,
initializer=k_init,
name='k_kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(4 * self.units, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
self.input_spec = InputSpec(ndim=2, axes={-1: 4 * input_dim})
self.built = True
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
num_rows = self.nb_filters * self.input_dim
num_cols = np.prod(self.kernel_size)
else:
# in case it is the kernel is a matrix and not a filter
# which is the case of 2D input (No feature maps).
num_rows = self.input_dim
num_cols = self.kernel_size[-1]
flat_shape = (int(num_rows), int(num_cols))
rng = RandomState(self.seed)
r = rng.uniform(size=flat_shape)
i = rng.uniform(size=flat_shape)
z = r + 1j * i
u, _, v = np.linalg.svd(z)
unitary_z = np.dot(u, np.dot(np.eye(int(num_rows), int(num_cols)), np.conjugate(v).T))
real_unitary = unitary_z.real
imag_unitary = unitary_z.imag
if self.nb_filters is not None:
indep_real = np.reshape(real_unitary, (num_rows,) + tuple(self.kernel_size))
indep_imag = np.reshape(imag_unitary, (num_rows,) + tuple(self.kernel_size))
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
)
else:
indep_real = real_unitary
indep_imag = imag_unitary
fan_in, fan_out = (int(self.input_dim), self.kernel_size[-1])
if self.criterion == 'glorot':
desired_var = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
desired_var = 1. / (fan_in)
else:
raise ValueError('Invalid criterion: ' + self.criterion)
multip_real = np.sqrt(desired_var / np.var(indep_real))
multip_imag = np.sqrt(desired_var / np.var(indep_imag))
scaled_real = multip_real * indep_real
scaled_imag = multip_imag * indep_imag
if self.weight_dim == 2 and self.nb_filters is None:
weight_real = scaled_real
weight_imag = scaled_imag
else:
kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
if self.weight_dim == 1:
transpose_shape = (1, 0)
elif self.weight_dim == 2 and self.nb_filters is not None:
transpose_shape = (1, 2, 0)
elif self.weight_dim == 3 and self.nb_filters is not None:
transpose_shape = (1, 2, 3, 0)
weight_real = np.transpose(scaled_real, transpose_shape)
weight_imag = np.transpose(scaled_imag, transpose_shape)
weight_real = np.reshape(weight_real, kernel_shape)
weight_imag = np.reshape(weight_imag, kernel_shape)
weight = np.concatenate([weight_real, weight_imag], axis=-1)
return weight
def _conv_kernel_initializer(shape, dtype=None):
fan_in, fan_out = _compute_fans(shape)
stddev = np.sqrt(2. / fan_in)
return K.random_normal(shape, 0., stddev, dtype)
def my_init_others(shape, dtype=None):
rnd = K.random_uniform(shape, 0., 1., dtype)
from keras.initializers import _compute_fans
fan_in, fan_out = _compute_fans(shape)
return 2. * (rnd - 0.5) / math.sqrt(fan_in)
def test_glorot_normal(tensor_shape):
fan_in, fan_out = initializers._compute_fans(tensor_shape)
scale = np.sqrt(2. / (fan_in + fan_out))
_runner(initializers.glorot_normal(), tensor_shape,
target_mean=0., target_std=None, target_max=2 * scale)
