def __call__(self, shape, dtype=None): from keras.initializers import _compute_fans fan_in, fan_out = _compute_fans(shape) scale = self.scale if self.mode == 'fan_in': scale /= max(1., fan_in) elif self.mode == 'fan_out': scale /= max(1., fan_out) else: scale /= max(1., float(fan_in + fan_out) / 2) if self.distribution == 'normal': # 0.879... = scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = np.sqrt(scale) / .87962566103423978 init = K.truncated_normal(shape, 0., stddev, dtype=dtype, seed=self.seed) else: limit = np.sqrt(3. * scale) init = K.random_uniform(shape, -limit, limit, dtype=dtype, seed=self.seed) return init * self.tree_weight
def build(self, input_shape): assert len(input_shape) == 2 assert input_shape[-1] % 2 == 0 input_dim = input_shape[-1] // 4 data_format = K.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) if self.init_criterion == 'he': s = np.sqrt(1. / fan_in) elif self.init_criterion == 'glorot': s = np.sqrt(1. / (fan_in + fan_out)) rng = RandomStreams(seed=self.seed) # Initialization using euclidean representation: def init_w_real(shape, dtype=None): return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype) def init_w_imag(shape, dtype=None): return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype) if self.kernel_initializer in {'quaternion'}: real_init = init_w_real imag_init = init_w_imag else: real_init = self.kernel_initializer imag_init = self.kernel_initializer self.r = self.add_weight(shape=kernel_shape, initializer=real_init, name='r', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.i = self.add_weight(shape=kernel_shape, initializer=imag_init, name='i', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.j = self.add_weight(shape=kernel_shape, initializer=imag_init, name='j', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.k = self.add_weight(shape=kernel_shape, initializer=imag_init, name='k', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(4 * self.units, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = InputSpec(ndim=2, axes={-1: 4 * input_dim}) self.built = True
def build(self, input_shape): assert len(input_shape) == 2 assert input_shape[-1] % 2 == 0 input_dim = input_shape[-1] // 2 data_format = K.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) if self.init_criterion == 'he': s = K.sqrt(1. / fan_in) elif self.init_criterion == 'glorot': s = K.sqrt(1. / (fan_in + fan_out)) rng = RandomStreams(seed=self.seed) def init_theta(shape, dtype=None): return rng.uniform(size=kernel_shape, low=0, high=6) if self.kernel_initializer in {'complex'}: theta_init = init_theta else: raise ValueError('Not recognized choice of initialization!') # Defining layer parameters (Codebook): self.theta = self.add_weight(shape=kernel_shape, initializer=theta_init, name='theta_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.real_kernel = (1 / self.scale) * K.cos(self.theta) # self.imag_kernel = (1 / self.scale) * K.sin(self.theta) # self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim}) self.built = True
def test_lecun_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) _runner(initializers.lecun_normal(), tensor_shape, target_mean=0., target_std=std)
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters) ) if self.criterion == 'glorot': s = 1. / (fan_in + fan_out) elif self.criterion == 'he': s = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(1337) modulus = rng.uniform(low=-np.sqrt(s)*np.sqrt(3), high=np.sqrt(s)*np.sqrt(3), size=kernel_shape) phase = rng.uniform(low=-np.pi/2, high=np.pi/2, size=kernel_shape) wm = modulus wp = phase weight = np.concatenate([wp, wm], axis=-1) return weight
def test_glorot_uniform(tensor_shape): fan_in, fan_out = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) _runner(initializers.glorot_uniform(), tensor_shape, target_mean=0., target_std=std)
def test_he_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) _runner(initializers.he_uniform(), tensor_shape, target_mean=0., target_std=std)
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters) ) if self.criterion == 'glorot': s = 1. / (fan_in + fan_out) elif self.criterion == 'he': s = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_real = modulus * np.cos(phase) weight_imag = modulus * np.sin(phase) weight = np.concatenate([weight_real, weight_imag], axis=-1) return weight
def glorot_init(shape, seed=0): assert len(shape) == 2 from keras.initializers import _compute_fans fan_in, fan_out = _compute_fans(shape) scale = 1 / max(1., float(fan_in + fan_out) / 2) limit = np.sqrt(3. * scale) return np.random.uniform(-limit, limit, shape)
def build(self, input_shape): assert len(input_shape) == 2 assert input_shape[-1] % 2 == 0 input_dim = input_shape[-1] // 2 data_format = K.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) if self.init_criterion == 'he': s = K.sqrt(1. / fan_in) elif self.init_criterion == 'glorot': s = K.sqrt(1. / (fan_in + fan_out)) rng = RandomStreams(seed=self.seed) # Equivalent initialization using amplitude phase representation: """modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) def init_w_real(shape, dtype=None): return modulus * K.cos(phase) def init_w_imag(shape, dtype=None): return modulus * K.sin(phase)""" # Initialization using euclidean representation: def init_w_real(shape, dtype=None): return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype) def init_w_imag(shape, dtype=None): return rng.normal(size=kernel_shape, avg=0, std=s, dtype=dtype) if self.kernel_initializer in {'complex'}: real_init = init_w_real imag_init = init_w_imag else: real_init = self.kernel_initializer imag_init = self.kernel_initializer self.real_kernel = self.add_weight(shape=kernel_shape, initializer=real_init, name='real_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.imag_kernel = self.add_weight(shape=kernel_shape, initializer=imag_init, name='imag_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(2 * self.units, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim}) self.built = True
def __call__(self, shape): rank = len(shape) if self.seed is not None: np.random.seed(self.seed) fan_in, fan_out = _compute_fans(shape, K.image_data_format()) variance = 2 / fan_in if rank == 3: row, stack_size, filters_size = shape transpose_dimensions = (2, 1, 0) kernel_shape = (row, ) correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0]) correct_fft = np.fft.rfft elif rank == 4: row, column, stack_size, filters_size = shape transpose_dimensions = (2, 3, 0, 1) kernel_shape = (row, column) correct_ifft = np.fft.irfft2 correct_fft = np.fft.rfft2 elif rank == 5: x, y, z, stack_size, filters_size = shape transpose_dimensions = (3, 4, 0, 1, 2) kernel_shape = (x, y, z) correct_fft = np.fft.rfftn correct_ifft = np.fft.irfftn else: return K.variable(self.orthogonal(shape), dtype=K.floatx()) kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape init = [] for i in range(filters_size): basis = self._create_basis(stack_size, np.prod(kernel_fourier_shape)) basis = basis.reshape((stack_size, ) + kernel_fourier_shape) filters = [ correct_ifft(x, kernel_shape) + np.random.normal(0, self.eps_std, kernel_shape) for x in basis ] init.append(filters) # Format of array is now: filters, stack, row, column init = np.array(init) init = self._scale_filters(init, variance) return init.transpose(transpose_dimensions)
def __call__(self, shape, dtype=None): if self.nb_filters is not None: num_rows = self.nb_filters * self.input_dim num_cols = np.prod(self.kernel_size) else: # in case it is the kernel is a matrix and not a filter # which is the case of 2D input (No feature maps). num_rows = self.input_dim num_cols = self.kernel_size[-1] flat_shape = (num_rows, num_cols) rng = RandomState(self.seed) x = rng.uniform(size=flat_shape) u, _, v = np.linalg.svd(x) orthogonal_x = np.dot(u, np.dot(np.eye(num_rows, num_cols), v.T)) if self.nb_filters is not None: independent_filters = np.reshape(orthogonal_x, (num_rows, ) + tuple(self.kernel_size)) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters)) else: independent_filters = orthogonal_x fan_in, fan_out = (self.input_dim, self.kernel_size[-1]) if self.criterion == 'glorot': desired_var = 2. / (fan_in + fan_out) elif self.criterion == 'he': desired_var = 2. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) multip_constant = np.sqrt(desired_var / np.var(independent_filters)) scaled_indep = multip_constant * independent_filters if self.weight_dim == 2 and self.nb_filters is None: weight_real = scaled_real weight_imag = scaled_imag else: kernel_shape = tuple( self.kernel_size) + (self.input_dim, self.nb_filters) if self.weight_dim == 1: transpose_shape = (1, 0) elif self.weight_dim == 2 and self.nb_filters is not None: transpose_shape = (1, 2, 0) elif self.weight_dim == 3 and self.nb_filters is not None: transpose_shape = (1, 2, 3, 0) weight = np.transpose(scaled_indep, transpose_shape) weight = np.reshape(weight, kernel_shape) return weight
def test_convolution_aware(tensor_shape): """ Convolution Aware Initialization Test Parameters ---------- tensor_shape: tuple The shape of the tensor to feed to the initializer """ fan_in, _ = k_initializers._compute_fans(tensor_shape) # pylint:disable=protected-access std = np.sqrt(2. / fan_in) _runner(initializers.ConvolutionAware(seed=123, init=True), tensor_shape, target_mean=0, target_std=std)
def glorot_uniform_sigm(shape): """ Glorot style weight initializer for sigmoid activations. Like keras.initializations.glorot_uniform(), but with uniform random interval like in Deeplearning.net tutorials. They claim that the initialization random interval should be +/- sqrt(6 / (fan_in + fan_out)) (like Keras' glorot_uniform()) when tanh activations are used, +/- 4 sqrt(6 / (fan_in + fan_out)) when sigmoid activations are used. See: http://deeplearning.net/tutorial/mlp.html#going-from-logistic-regression-to-mlp """ fan_in, fan_out = _compute_fans(shape) s = 4. * np.sqrt(6. / (fan_in + fan_out)) return uniform(shape, s)
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int( self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters)) if self.criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif self.criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + self.criterion) rng = Chi4Random(s) flat_size = np.product(kernel_shape) modulus = rng.random(N=flat_size) modulus = modulus.reshape(kernel_shape) rng = RandomState(self.seed) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) # must make random unit vector for quaternion vector def make_rand_vector(dims): vec = [gauss(0, 1) for i in range(dims)] mag = sum(x**2 for x in vec)**0.5 return [x / mag for x in vec] u_i = np.zeros(flat_size) u_j = np.zeros(flat_size) u_k = np.zeros(flat_size) for u in range(flat_size): unit = make_rand_vector(3) u_i[u] = unit[0] u_j[u] = unit[1] u_k[u] = unit[2] u_i = u_i.reshape(kernel_shape) u_j = u_j.reshape(kernel_shape) u_k = u_k.reshape(kernel_shape) weight_r = modulus * np.cos(phase) weight_i = modulus * u_i * np.sin(phase) weight_j = modulus * u_j * np.sin(phase) weight_k = modulus * u_k * np.sin(phase) weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) return weight
def test_icnr(tensor_shape): """ ICNR Initialization Test Parameters ---------- tensor_shape: tuple The shape of the tensor to feed to the initializer """ fan_in, _ = k_initializers._compute_fans(tensor_shape) # pylint:disable=protected-access std = np.sqrt(2. / fan_in) _runner(initializers.ICNR(initializer=k_initializers.he_uniform(), scale=2), tensor_shape, target_mean=0, target_std=std)
def build(self, input_shape): assert len(input_shape) == 2 assert input_shape[-1] % 2 == 0 input_dim = input_shape[-1] // 3 data_format = K.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) s = np.sqrt(1. / fan_in) def init_phase(shape, dtype=None): return np.random.normal( size=kernel_shape, loc=0, scale=np.pi / 2, ) def init_modulus(shape, dtype=None): return np.random.normal(size=kernel_shape, loc=0, scale=s) phase_init = init_phase modulus_init = init_modulus self.phase_kernel = self.add_weight( shape=kernel_shape, initializer=phase_init, name='phase_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.modulus_kernel = self.add_weight( shape=kernel_shape, initializer=modulus_init, name='modulus_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(3 * self.units, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = InputSpec(ndim=2, axes={-1: 3 * input_dim}) self.built = True
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int( self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters)) # Quaternion operations start here if self.criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif self.criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + self.criterion) #Generating randoms and purely imaginary quaternions : number_of_weights = np.prod(kernel_shape) v_i = np.random.uniform(0.0, 1.0, number_of_weights) v_j = np.random.uniform(0.0, 1.0, number_of_weights) v_k = np.random.uniform(0.0, 1.0, number_of_weights) #Make these purely imaginary quaternions unitary for i in range(0, number_of_weights): norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(kernel_shape) v_j = v_j.reshape(kernel_shape) v_k = v_k.reshape(kernel_shape) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_r = modulus * np.cos(phase) weight_i = modulus * v_i * np.sin(phase) weight_j = modulus * v_j * np.sin(phase) weight_k = modulus * v_k * np.sin(phase) weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) return weight
def __call__(self, shape, dtype=None): # The only difference between this and the built-in VarianceScaling is that # we multiply element-wise by our adjacency matrix as the final step. fan_in, fan_out = _compute_fans(shape) scale = self.scale if self.mode == 'fan_in': scale /= max(1., fan_in) elif self.mode == 'fan_out': scale /= max(1., fan_out) else: scale /= max(1., float(fan_in + fan_out) / 2) final = None if self.distribution == 'normal': mean = 0.0 stddev = np.sqrt(scale) normal_mat = np.random.normal(loc=mean, scale=stddev, size=shape) clipped = np.clip(normal_mat, mean - 2 * stddev, mean + 2 * stddev) return clipped * self.adjacency_mat else: limit = np.sqrt(3. * scale) dense_initial = np.random.uniform(low=-limit, high=limit, size=shape) return dense_initial * self.adjacency_mat
def test_lecun_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) scale = np.sqrt(1. / fan_in) _runner(initializers.lecun_normal(), tensor_shape, target_mean=0., target_std=scale)
def build(self, input_shape): assert len(input_shape) == 2 # We have both real and imaginary components so the last dimension must be even assert input_shape[-1] % 2 == 0 # Remember: the input is [real, imaginary], so what looks like a 6 unit input is actually 3 input_dim = input_shape[-1] // 2 data_format = kb.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) if self.init_criterion == 'he': s = np.sqrt(1. / fan_in) elif self.init_criterion == 'glorot': s = np.sqrt(1. / (fan_in + fan_out)) else: s = 1.0 rng = np.random.RandomState(seed=self.seed) def init_w_real(shape, dtype=None): return rng.normal(loc=0.0, scale=s, size=kernel_shape) def init_w_imag(shape, dtype=None): return rng.normal(loc=0.0, scale=s, size=kernel_shape) if self.kernel_initializer in {'complex'}: real_init = init_w_real imag_init = init_w_imag else: real_init = self.kernel_initializer imag_init = self.kernel_initializer self.real_kernel = self.add_weight(name='real_kernel', shape=kernel_shape, initializer=real_init, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.imag_kernel = self.add_weight(name='imag_kernel', shape=kernel_shape, initializer=imag_init, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.kernel = kb.concatenate([ kb.concatenate([self.real_kernel, -self.imag_kernel], axis=-1), kb.concatenate([self.imag_kernel, self.real_kernel], axis=-1) ], axis=0) if self.use_bias: self.bias = self.add_weight(name='bias', shape=(2 * self.units, ), initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim}) self.built = True
def build(self, input_shape): # assert len(input_shape) == 2 # assert input_shape[-1] % 2 == 0 # input_dim = input_shape[-1] // 2 if not isinstance(input_shape, list): raise ValueError('This layer should be called ' 'on a list of 2 inputs.') if len(input_shape) != 2: raise ValueError('This layer should be called ' 'on a list of 2 inputs.' 'Got ' + str(len(input_shape)) + ' inputs.') input_dim = input_shape[0][-1] self.input_dim = input_dim data_format = K.image_data_format() kernel_shape = (input_dim, 2 * self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) fan_in = tf.to_float(fan_in) fan_out = tf.to_float(fan_out) if self.init_criterion == 'he': s = K.sqrt(1. / fan_in) elif self.init_criterion == 'glorot': s = K.sqrt(1. / (fan_in + fan_out)) # rng = RandomStreams(seed=self.seed) # Equivalent initialization using amplitude phase representation: """modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) def init_w_real(shape, dtype=None): return modulus * K.cos(phase) def init_w_imag(shape, dtype=None): return modulus * K.sin(phase)""" # Initialization using euclidean representation: def init_w(shape, dtype=None): return tf.random_normal(shape=kernel_shape, mean=0.0, stddev=s, dtype=tf.float32, seed=self.seed, name=None) # return rng.normal( # size=kernel_shape, # avg=0, # std=s, # dtype=dtype # ) if self.kernel_initializer in {'complex'}: weight_init = init_w else: weight_init = self.kernel_initializer self.kernel = self.add_weight(shape=kernel_shape, initializer=weight_init, name='kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(2 * self.units, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None # self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim}) self.built = True
def test_he_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) _runner(initializers.he_normal(), tensor_shape, target_mean=0., target_std=std)
def test_glorot_normal(tensor_shape): fan_in, fan_out = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) _runner(initializers.glorot_normal(), tensor_shape, target_mean=0., target_std=std)
def test_lecun_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) _runner(initializers.lecun_uniform(), tensor_shape, target_mean=0., target_std=std)
def __call__(self, shape, dtype=None): dtype = K.floatx() if dtype is None else dtype if self._init: logger.info("Calculating Convolution Aware Initializer for shape: %s", shape) else: logger.debug("Bypassing Convolutional Aware Initializer for saved model") # Dummy in he_uniform just in case there aren't any weighs being loaded # and it needs some kind of initialization return self.he_uniform(shape, dtype=dtype) rank = len(shape) if self.seed is not None: np.random.seed(self.seed) #计算输入范围的数量 fan_in, _ = initializers._compute_fans(shape) # pylint:disable=protected-access variance = 2 / fan_in if rank == 3: row, stack_size, filters_size = shape transpose_dimensions = (2, 1, 0) kernel_shape = (row,) correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0]) # noqa correct_fft = np.fft.rfft elif rank == 4: row, column, stack_size, filters_size = shape transpose_dimensions = (2, 3, 1, 0) kernel_shape = (row, column) correct_ifft = np.fft.irfft2 correct_fft = np.fft.rfft2 elif rank == 5: var_x, var_y, var_z, stack_size, filters_size = shape transpose_dimensions = (3, 4, 0, 1, 2) kernel_shape = (var_x, var_y, var_z) correct_fft = np.fft.rfftn correct_ifft = np.fft.irfftn else: return K.variable(self.orthogonal(shape), dtype=dtype) kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape init = [] for _ in range(filters_size): basis = self._create_basis( stack_size, np.prod(kernel_fourier_shape), dtype) basis = basis.reshape((stack_size,) + kernel_fourier_shape) filters = [correct_ifft(x, kernel_shape) + np.random.normal(0, self.eps_std, kernel_shape) for x in basis] init.append(filters) # Format of array is now: filters, stack, row, column init = np.array(init) init = self._scale_filters(init, variance) return K.variable(init.transpose(transpose_dimensions), dtype=dtype, name="conv_aware")
def test_he_uniform(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) scale = np.sqrt(6. / fan_in) _runner(initializers.he_uniform(), tensor_shape, target_mean=0., target_max=scale, target_min=-scale)
def _dense_kernel_initializer(shape, dtype=None): fan_in, fan_out = _compute_fans(shape) stddev = 1. / np.sqrt(fan_in) return K.random_uniform(shape, -stddev, stddev, dtype)
def test_glorot_uniform(tensor_shape): fan_in, fan_out = initializers._compute_fans(tensor_shape) scale = np.sqrt(6. / (fan_in + fan_out)) _runner(initializers.glorot_uniform(), tensor_shape, target_mean=0., target_max=scale, target_min=-scale)
def __call__(self, shape, dtype=None): """ Call function for the ICNR initializer. Parameters ---------- shape: tuple or list The required shape for the output tensor dtype: str The data type for the tensor Returns ------- tensor The modified kernel weights """ dtype = K.floatx() if dtype is None else dtype if self._init: logger.info( "Calculating Convolution Aware Initializer for shape: %s", shape) else: logger.debug( "Bypassing Convolutional Aware Initializer for saved model") # Dummy in he_uniform just in case there aren't any weighs being loaded # and it needs some kind of initialization return self.he_uniform(shape, dtype=dtype) rank = len(shape) if self.seed is not None: np.random.seed(self.seed) fan_in, _ = initializers._compute_fans(shape) # pylint:disable=protected-access variance = 2 / fan_in if rank == 3: row, stack_size, filters_size = shape transpose_dimensions = (2, 1, 0) kernel_shape = (row, ) correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0] ) # noqa correct_fft = np.fft.rfft elif rank == 4: row, column, stack_size, filters_size = shape transpose_dimensions = (2, 3, 1, 0) kernel_shape = (row, column) correct_ifft = np.fft.irfft2 correct_fft = np.fft.rfft2 elif rank == 5: var_x, var_y, var_z, stack_size, filters_size = shape transpose_dimensions = (3, 4, 0, 1, 2) kernel_shape = (var_x, var_y, var_z) correct_fft = np.fft.rfftn correct_ifft = np.fft.irfftn else: return K.variable(self.orthogonal(shape), dtype=dtype) kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape basis = self._create_basis(filters_size, stack_size, np.prod(kernel_fourier_shape), dtype) basis = basis.reshape(( filters_size, stack_size, ) + kernel_fourier_shape) randoms = np.random.normal(0, self.eps_std, basis.shape[:-2] + kernel_shape) init = correct_ifft(basis, kernel_shape) + randoms init = self._scale_filters(init, variance) return K.variable(init.transpose(transpose_dimensions), dtype=dtype, name="conv_aware")
def test_he_normal(tensor_shape): fan_in, _ = initializers._compute_fans(tensor_shape) scale = np.sqrt(2. / fan_in) _runner(initializers.he_normal(), tensor_shape, target_mean=0., target_std=None, target_max=2 * scale)
def build(self, input_shape): assert len(input_shape) == 2 assert input_shape[-1] % 4 == 0 input_dim = input_shape[-1] // 4 data_format = K.image_data_format() kernel_shape = (input_dim, self.units) fan_in, fan_out = initializers._compute_fans(kernel_shape, data_format=data_format) if self.init_criterion == 'he': s = tf.sqrt(tf.cast(1. / fan_in, tf.float32)) elif self.init_criterion == 'glorot': s = tf.sqrt(tf.cast(1. / (fan_in + fan_out), tf.float32)) # Equivalent initialization using amplitude phase representation: """modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) def init_w_real(shape, dtype=None): return modulus * K.cos(phase) def init_w_imag(shape, dtype=None): return modulus * K.sin(phase)""" # Initialization using euclidean representation: def init_w_r(shape, dtype=None): return K.random_normal_variable(kernel_shape, mean=0, scale=s, dtype=tf.float32, seed=self.seed) def init_w_i(shape, dtype=None): return K.random_normal_variable(kernel_shape, mean=0, scale=s, dtype=tf.float32, seed=self.seed) def init_w_j(shape, dtype=None): return K.random_normal_variable(kernel_shape, mean=0, scale=s, dtype=tf.float32, seed=self.seed) def init_w_k(shape, dtype=None): return K.random_normal_variable(kernel_shape, mean=0, scale=s, dtype=tf.float32, seed=self.seed) if self.kernel_initializer in {'quaternion'}: r_init = init_w_r i_init = init_w_i j_init = init_w_j k_init = init_w_k else: r_init = self.kernel_initializer i_init = self.kernel_initializer j_init = self.kernel_initializer k_init = self.kernel_initializer self.r_kernel = self.add_weight(shape=kernel_shape, initializer=r_init, name='r_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.i_kernel = self.add_weight(shape=kernel_shape, initializer=i_init, name='i_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.j_kernel = self.add_weight(shape=kernel_shape, initializer=j_init, name='j_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.k_kernel = self.add_weight(shape=kernel_shape, initializer=k_init, name='k_kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(4 * self.units, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = InputSpec(ndim=2, axes={-1: 4 * input_dim}) self.built = True
def __call__(self, shape, dtype=None): if self.nb_filters is not None: num_rows = self.nb_filters * self.input_dim num_cols = np.prod(self.kernel_size) else: # in case it is the kernel is a matrix and not a filter # which is the case of 2D input (No feature maps). num_rows = self.input_dim num_cols = self.kernel_size[-1] flat_shape = (int(num_rows), int(num_cols)) rng = RandomState(self.seed) r = rng.uniform(size=flat_shape) i = rng.uniform(size=flat_shape) z = r + 1j * i u, _, v = np.linalg.svd(z) unitary_z = np.dot(u, np.dot(np.eye(int(num_rows), int(num_cols)), np.conjugate(v).T)) real_unitary = unitary_z.real imag_unitary = unitary_z.imag if self.nb_filters is not None: indep_real = np.reshape(real_unitary, (num_rows,) + tuple(self.kernel_size)) indep_imag = np.reshape(imag_unitary, (num_rows,) + tuple(self.kernel_size)) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) ) else: indep_real = real_unitary indep_imag = imag_unitary fan_in, fan_out = (int(self.input_dim), self.kernel_size[-1]) if self.criterion == 'glorot': desired_var = 1. / (fan_in + fan_out) elif self.criterion == 'he': desired_var = 1. / (fan_in) else: raise ValueError('Invalid criterion: ' + self.criterion) multip_real = np.sqrt(desired_var / np.var(indep_real)) multip_imag = np.sqrt(desired_var / np.var(indep_imag)) scaled_real = multip_real * indep_real scaled_imag = multip_imag * indep_imag if self.weight_dim == 2 and self.nb_filters is None: weight_real = scaled_real weight_imag = scaled_imag else: kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) if self.weight_dim == 1: transpose_shape = (1, 0) elif self.weight_dim == 2 and self.nb_filters is not None: transpose_shape = (1, 2, 0) elif self.weight_dim == 3 and self.nb_filters is not None: transpose_shape = (1, 2, 3, 0) weight_real = np.transpose(scaled_real, transpose_shape) weight_imag = np.transpose(scaled_imag, transpose_shape) weight_real = np.reshape(weight_real, kernel_shape) weight_imag = np.reshape(weight_imag, kernel_shape) weight = np.concatenate([weight_real, weight_imag], axis=-1) return weight
def _conv_kernel_initializer(shape, dtype=None): fan_in, fan_out = _compute_fans(shape) stddev = np.sqrt(2. / fan_in) return K.random_normal(shape, 0., stddev, dtype)
def my_init_others(shape, dtype=None): rnd = K.random_uniform(shape, 0., 1., dtype) from keras.initializers import _compute_fans fan_in, fan_out = _compute_fans(shape) return 2. * (rnd - 0.5) / math.sqrt(fan_in)
def test_glorot_normal(tensor_shape): fan_in, fan_out = initializers._compute_fans(tensor_shape) scale = np.sqrt(2. / (fan_in + fan_out)) _runner(initializers.glorot_normal(), tensor_shape, target_mean=0., target_std=None, target_max=2 * scale)