def __call__(self, shape, dtype=dtypes.float32, **kwargs): """Returns a tensor object initialized as specified by the initializer. Args: shape: Shape of the tensor. dtype: Optional dtype of the tensor. Only floating point types are supported. **kwargs: Additional keyword arguments. Raises: ValueError: If the dtype is not floating point """ self._validate_kwargs(kwargs) dtype = _assert_float_dtype(dtype) scale = self.scale fan_in, fan_out = _compute_fans(shape) if _PARTITION_SHAPE in kwargs: shape = kwargs[_PARTITION_SHAPE] if self.mode == "fan_in": scale /= max(1., fan_in) elif self.mode == "fan_out": scale /= max(1., fan_out) else: scale /= max(1., (fan_in + fan_out) / 2.) if self.distribution == "truncated_normal": # constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = math.sqrt(scale) / .87962566103423978 return self._random_generator.truncated_normal(shape, 0.0, stddev, dtype) elif self.distribution == "untruncated_normal": stddev = math.sqrt(scale) return self._random_generator.random_normal(shape, 0.0, stddev, dtype) else: limit = math.sqrt(3.0 * scale) return self._random_generator.random_uniform(shape, -limit, limit, dtype)
def test_he_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) self._runner(keras.initializers.he_uniform(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_lecun_normal(self): tensor_shape = (5, 6, 4, 2) with self.test_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) self._runner(keras.initializers.lecun_normal(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_glorot_normal(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, fan_out = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_he_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) self._runner(keras.initializers.he_uniform(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_glorot_normal(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, fan_out = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_he_normal(self): tensor_shape = (5, 6, 4, 2) with self.test_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) scale = np.sqrt(2. / fan_in) self._runner(keras.initializers.he_normal(seed=123), tensor_shape, target_mean=0., target_std=None, target_max=2 * scale)
def __call__(self, shape, dtype=None, partition_info=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int( self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = _compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters)) if self.criterion == 'glorot': # differnt weight initializers s = 1. / (fan_in + fan_out) elif self.criterion == 'he': s = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_real = modulus * np.cos(phase) weight_imag = modulus * np.sin(phase) weight = np.concatenate([weight_real, weight_imag], axis=-1) return weight
def test_he_uniform(self): tensor_shape = (5, 6, 4, 2) with self.test_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) scale = np.sqrt(6. / fan_in) self._runner(keras.initializers.he_uniform(seed=123), tensor_shape, target_mean=0., target_max=scale, target_min=-scale)
def test_lecun_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) self._runner(init_ops.lecun_uniform(seed=123), tensor_shape, target_mean=0., target_std=std)
def __call__(self, shape, dtype=None, partition_info=None): if dtype is None: dtype = self.dtype scale_shape = shape if partition_info is not None: scale_shape = partition_info.full_shape fan_in, fan_out = _compute_fans(scale_shape) return -tf.ones(shape, dtype=dtype) * fan_in * self.w / 2.0 + self.b
def test_lecun_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(1. / fan_in) self._runner( init_ops.lecun_uniform(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_he_normal(self): tensor_shape = (5, 6, 4, 2) with self.test_session(): fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) self._runner( init_ops.he_normal(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_he_normal(self): shape = (5, 6, 4, 2) with self.cached_session(): for tensor_shape in [shape, tensor_shape_lib.TensorShape(shape)]: fan_in, _ = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / fan_in) self._runner(init_ops.he_normal(seed=123), tensor_shape, target_mean=0., target_std=std)
def test_glorot_normal_initializer(self): shape = (5, 6, 4, 2) with self.cached_session(): for tensor_shape in [shape, tensor_shape_lib.TensorShape(shape)]: fan_in, fan_out = init_ops._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) self._runner(init_ops.glorot_normal_initializer(seed=123), tensor_shape, target_mean=0., target_std=std)
def _random_modulus_and_phase(self, shape): shape = to_int_shape(shape) fan_in, fan_out = _compute_fans(shape) if self.criterion == 'glorot': scale = 1. / (fan_in + fan_out) elif self.criterion == 'he': scale = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) self.modulus = random_rayleigh(shape, scale=scale) self.phase = tensorflow.random_uniform(shape, minval=-math.pi, maxval=math.pi)
def __call__(self, shape, dtype=None, partition_info=None): if dtype is None: dtype = self.dtype scale = self.scale scale_shape = shape if partition_info is not None: scale_shape = partition_info.full_shape fan_in, fan_out = _compute_fans(scale_shape) if self.mode == "fan_in": scale /= max(1., fan_in) elif self.mode == "fan_out": scale /= max(1., fan_out) else: scale /= max(1., (fan_in + fan_out) / 2.) #if self.distribution == "normal" or self.distribution == "truncated_normal": # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = tf.sqrt(scale) / .87962566103423978 #if self.scale == 1: if scale == 1: return tf.random.truncated_normal(shape, 0.0, stddev, dtype, seed=self.seed) #logging.info(scale) new_shape = shape[:3] + (shape[3] // (self.scale**2), ) x = tf.random.truncated_normal(new_shape, 0.0, stddev, dtype, seed=self.seed) x = tf.transpose(x, perm=[2, 0, 1, 3]) x = tf.image.resize_nearest_neighbor(x, size=(shape[0] * self.scale, shape[1] * self.scale)) #x = tf.space_to_depth(x, block_size=scale) x = tf.space_to_depth(x, block_size=self.scale) x = tf.transpose(x, perm=[1, 2, 0, 3]) return x
def __call__(self, shape, dtype=None, partition_info=None): if partition_info is not None: raise ValueError("partition_info not supported.") if dtype is None: dtype = self.dtype if len(shape) != 2: raise ValueError("Weights must be 2-dimensional.") fan_in, fan_out = init_ops._compute_fans(shape) # pylint: disable=protected-access # Calculate the number of non-zero weights in the weight matrix nnz = 1.0 for d in shape: nnz *= d nnz *= 1 - self.sparsity limit = math.sqrt(6.0 / (nnz / fan_out + nnz / fan_in)) return tf.random_uniform(shape, -limit, limit, dtype, seed=self.seed)
def __call__(self, shape, dtype=dtypes.float32): """Returns a tensor object initialized as specified by the initializer. Args: shape: Shape of the tensor. dtype: Optional dtype of the tensor. Only floating point types are supported. Raises: ValueError: If the dtype is not floating point """ partition_info = None # Keeps logic so can be readded later if necessary dtype = _assert_float_dtype(dtype) scale = self.scale scale_shape = shape if partition_info is not None: scale_shape = partition_info.full_shape fan_in, fan_out = _compute_fans(scale_shape) if self.mode == "fan_in": scale /= max(1., fan_in) elif self.mode == "fan_out": scale /= max(1., fan_out) else: scale /= max(1., (fan_in + fan_out) / 2.) if self.distribution == "truncated_normal": # constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = math.sqrt(scale) / .87962566103423978 return self._random_generator.truncated_normal( shape, 0.0, stddev, dtype) elif self.distribution == "untruncated_normal": stddev = math.sqrt(scale) return self._random_generator.random_normal( shape, 0.0, stddev, dtype) else: limit = math.sqrt(3.0 * scale) return self._random_generator.random_uniform( shape, -limit, limit, dtype)
def __call__(self, shape, dtype=None, partition_info=None): # Note here, shape refers to dims not distribution shape parameter! # We will switch in one line: dims = shape if partition_info is None else partition_info.full_shape if dtype is None: dtype = self.dtype if not dtype.is_floating: raise TypeError("Only floating data types supported.") # Calculate denominator ninp, nout = _compute_fans(dims) n = None if self.fan == 'in': n = max(1., ninp) elif self.fan == 'out': n = max(1., nout) elif self.fan == 'avg': n = max(1., 0.5 * (ninp + nout)) # Random number generator rand = None stdev = math.sqrt(2. / n) # a la he2015delving if self.distr == 'unif': limit = stdev * math.sqrt(3.) rand = random_ops.uniform(dims, -limit, limit, self.dtype, seed=self.seed) elif self.distr == 'norm': rand = random_ops.random_normal(dims, 0., stdev, self.dtype, seed=self.seed) elif self.distr == 'trun': # -2SD to +2SD truncated normal has a SD of 1./1.137 rand = random_ops.truncated_normal(dims, 0., 1.137 * stdev, self.dtype, seed=self.seed) elif self.distr == 'gamma': gamma = self.shape invlb = math.sqrt(gamma) / stdev rand = random_ops.random_gamma( dims, gamma, invlb, self.dtype, seed=self.seed) - (gamma / invlb) elif self.distr == 'beta': gamma = self.shape rand_0 = random_ops.random_gamma(dims, gamma, 1., self.dtype, seed=self.seed) rand_1 = random_ops.random_gamma(dims, gamma, 1., self.dtype, seed=self.seed) rand = stdev * ( (rand_0 / (rand_0 + rand_1)) - 0.5) * math.sqrt(8. * self.shape + 4.) # Scale and offset as required randscaled = rand if self.scale == 1. else rand * self.scale randoffset = randscaled if self.locan == 0. else randscaled + ( self.locan / n) return randoffset