def __c_1_lambda_om(_lambda, dim, ecdf): _lambda_half = int(ceil(_lambda / 2.0)) proj_max = zeros(n_sample_per_cycle) for i in range(n_sample): # mirrored orthogonal sampling q = qr(randn(dim, dim))[0] l = chi.rvs(dim, size=dim) s = l * q samples = s[:, 0:_lambda_half] samples = append(samples, -samples, axis=1) # projection onto e1 proj = samples[0, :] # the largest order statistic proj_sorted = sort(proj) proj_max[i % n_sample_per_cycle] = proj_sorted[-1] if (i + 1) % n_sample_per_cycle == 0: for k, x in enumerate(x_point): ecdf[k] += np.sum(proj_max <= x)
def fit(self, X, y=None): """Generate random weights according to n_features. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ random_state = check_random_state(self.random_state) X = check_array(X, accept_sparse=True) n_samples, n_features = X.shape n_features_padded = next_pow_of_two(n_features) n_stacks = int(np.ceil(self.n_components / n_features_padded)) if self.random_fourier and not self.use_offset: n_stacks = int(np.ceil(n_stacks / 2)) n_components = 2 * n_stacks * n_features_padded else: n_components = n_stacks * n_features_padded if n_components != self.n_components: warnings.warn("n_components is changed from {0} to {1}. " "You should set n_components n-tuple of the next " "power of two of n_features.".format( self.n_components, n_components)) self.n_components = n_components # n_stacks * n_features_padded = self.n_components size = (n_stacks, n_features_padded) if isinstance(self.distribution, str): distribution = _get_random_matrix(self.distribution) else: distribution = self.distribution self.random_weights_ = distribution(random_state, size) self.random_sign_ = rademacher(random_state, (n_stacks, n_features)) self.random_perm_ = np.zeros(size, dtype=np.int32) self._fy_vector_ = np.zeros(size, dtype=np.int32) for t in range(n_stacks): perm, fyvec = fisher_yates_shuffle_with_indices( n_features_padded, random_state) self.random_perm_[t] = perm self._fy_vector_[t] = fyvec Frobs = np.sqrt(np.sum(self.random_weights_**2, axis=1, keepdims=True)) self.random_scaling_ = chi.rvs( n_features_padded, size=size, random_state=random_state) / Frobs if self.random_fourier and self.use_offset: self.random_offset_ = random_state.uniform(0, 2 * np.pi, self.n_components) else: self.random_offset_ = None return self
def fit(self, X, y=None): """Fit the model with X. Samples a couple of random based vectors to approximate a Gaussian random projection matrix to generate n_components features. Parameters ---------- X : {array-like}, shape (n_samples, n_features), Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object, Returns the transformer. """ X = check_array(X) d_orig = X.shape[1] self.d, self.n, self.times_to_stack_v = Fastfood.enforce_dimensionality_constraints( d_orig, self.n_components) self.number_of_features_to_pad_with_zeros = self.d - d_orig self.G = self.rng.normal(size=(self.times_to_stack_v, self.d)) self.B = choice([-1, 1], size=(self.times_to_stack_v, self.d), replace=True) self.P = np.hstack([(i * self.d) + self.rng.permutation(self.d) for i in range(self.times_to_stack_v)]) self.S = np.multiply( 1 / self.l2norm_along_axis1(self.G).reshape((-1, 1)), chi.rvs(self.d, size=(self.times_to_stack_v, self.d))) self.U = self.uniform_vector() return self
def sampler(self, size): """ Defines the sampler object Args: size: Return: """ if self.kernel == "squared_exponential": distribution = lambda size: np.random.normal(size=size) * (1. / self.gamma) inv_cum_dist = lambda x: norm.ppf(x) * (1. / self.gamma) elif self.kernel == "laplace": distribution = None inv_cum_dist = lambda x: (np.tan(np.pi * x - np.pi) / self.gamma) elif self.kernel == "modified_matern": if self.nu == 2: distribution = None inv_cum_dist = None pdf = lambda x: np.prod(2*(self.gamma)/(np.power((1. + self.gamma**2*x**2),2) * np.pi),axis =1) elif self.nu == 3: distribution = None inv_cum_dist = None pdf = lambda x: np.prod((8.*self.gamma)/(np.power((1. + self.gamma**2*x**2),3) *3* np.pi),axis =1) elif self.nu == 4: distribution = None inv_cum_dist = None pdf = lambda x: np.prod((16.*self.gamma)/(np.power((1. + self.gamma**2*x**2),4) *5 * np.pi),axis =1) # Random Fourier Features if self.approx == "rff": if distribution == None: if inv_cum_dist == None: self.W = helper.rejection_sampling(pdf,size = size) else: self.W = helper.sample_custom(inv_cum_dist, size=size) else: self.W = distribution(size) # Quasi Fourier Features elif self.approx == "halton": if inv_cum_dist != None: self.W = helper.sample_qmc_halton(inv_cum_dist, size=size) else: raise AssertionError("Inverse Cumulative Distribution could not be deduced") elif self.approx == "orf": distribution = lambda size: np.random.normal(size=size) * (1.) self.W = distribution(size) # QR decomposition self.Q,_ = np.linalg.qr(self.W) # df and size self.S = np.diag(chi.rvs(size[1], size=size[0])) self.W = np.dot(self.S,self.Q)/self.gamma**2 return self.W
def hypercomplex_init(in_features, out_features, rng, kernel_size=None, criterion='glorot', num_components=8): if kernel_size is not None: receptive_field = np.prod(kernel_size) fan_in = in_features * receptive_field fan_out = out_features * receptive_field else: fan_in = in_features fan_out = out_features if criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + criterion) # rng = RandomState(np.random.randint(1, 1234)) # Generating randoms and purely imaginary hyper(complex) : if kernel_size is None: kernel_shape = (in_features, out_features) else: if type(kernel_size) is int: kernel_shape = (out_features, in_features) + tuple((kernel_size, )) else: kernel_shape = (out_features, in_features) + (*kernel_size, ) modulus = chi.rvs(num_components, loc=0, scale=s, size=kernel_shape) number_of_weights = np.prod(kernel_shape) v = [ np.random.uniform(-1.0, 1.0, number_of_weights) for component in range(num_components - 1) ] # Purely imaginary hyper(complex) unitary for i in range(0, number_of_weights): norm = np.sqrt( sum(v[j][i]**2 for j in range(num_components - 1)) + 0.0001) for j in range(num_components - 1): v[j][i] /= norm v = [v_c.reshape(kernel_shape) for v_c in v] phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight = [ torch.from_numpy(modulus * np.cos(phase)).type(torch.FloatTensor) ] weight.extend([ torch.from_numpy(modulus * v_c * np.sin(phase)).type(torch.FloatTensor) for v_c in v ]) return weight
def quaternion_init(in_features, out_features, rng, kernel_size=None, criterion='glorot'): if kernel_size is not None: receptive_field = np.prod(kernel_size) fan_in = in_features * receptive_field fan_out = out_features * receptive_field else: fan_in = in_features fan_out = out_features if criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + criterion) rng = RandomState(np.random.randint(1, 1234)) # Generating randoms and purely imaginary quaternions : if kernel_size is None: kernel_shape = (in_features, out_features) else: if type(kernel_size) is int: kernel_shape = (out_features, in_features) + tuple((kernel_size, )) else: kernel_shape = (out_features, in_features) + (*kernel_size, ) modulus = chi.rvs(4, loc=0, scale=s, size=kernel_shape) # modulus= rng.uniform(size=kernel_shape) number_of_weights = np.prod(kernel_shape) v_i = np.random.normal(0, 1.0, number_of_weights) v_j = np.random.normal(0, 1.0, number_of_weights) v_k = np.random.normal(0, 1.0, number_of_weights) # Purely imaginary quaternions unitary for i in range(0, number_of_weights): norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2 + 0.0001) v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(kernel_shape) v_j = v_j.reshape(kernel_shape) v_k = v_k.reshape(kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_r = modulus * np.cos(phase) weight_i = modulus * v_i * np.sin(phase) weight_j = modulus * v_j * np.sin(phase) weight_k = modulus * v_k * np.sin(phase) return (weight_r, weight_i, weight_j, weight_k)
def fit(self, X, y=None): """Fit the model with X. Samples a couple of random based vectors to approximate a Gaussian random projection matrix to generate n_components features. Parameters ---------- X : {array-like}, shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ X = check_array(X) d_orig = X.shape[1] # Initial number of features # n_components (self.n) is the final number of features # times_to_stack_v is the integer division of n by d # we use times_to_stack_v according to the paper FastFood: # "When n > d, we replicate (7) for n/d independent random matrices # Vi, and stack them via Vt = [V_1, V_2, ..., V_(n/d)]t until we have # enoug dimensions." self.d, self.n, self.times_to_stack_v = \ Fastfood.enforce_dimensionality_constraints(d_orig, self.n_components) self.number_of_features_to_pad_with_zeros = self.d - d_orig if self.d != d_orig: warn( "Dimensionality of the input space as been changed (zero padding) from {} to {}." .format(d_orig, self.d)) self.G = self.rng.normal(size=(self.times_to_stack_v, self.d)) # G is a random matrix following normal distribution self.B = choice([-1, 1], size=(self.times_to_stack_v, self.d), replace=True, random_state=self.random_state) # B is a random matrix of -1 and 1 self.P = np.hstack([(i * self.d) + self.rng.permutation(self.d) for i in range(self.times_to_stack_v)]) # P is a matrix of size d*n/d = n -> the dimension of the embedding space # P is for the permutation and respects the V stacks (see FastFood paper) self.S = np.multiply( 1 / self.l2norm_along_axis1(self.G).reshape((-1, 1)), chi.rvs(self.d, size=(self.times_to_stack_v, self.d))) self.H = scipy.linalg.hadamard(self.d) self.U = self.uniform_vector() return self
def quaternion_init(in_features, out_features, rng, kernel_size=None, criterion='glorot'): """Initialize quaternion layer weights """ if kernel_size is not None: receptive_field = np.prod(kernel_size) fan_in = in_features * receptive_field fan_out = out_features * receptive_field else: fan_in = in_features fan_out = out_features if criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + criterion) rng = np.random.RandomState(np.random.randint(1, 1234)) # Generate randoms and purely imaginary quaternions if kernel_size is None: kernel_shape = (in_features, out_features) else: # TODO(Emanuele): Expand different kernel sizes... kernel_shape = ... # Produce random variable vector modulus = chi.rvs(4, loc=0, scale=s, size=kernel_size) no_weights = np.prod(kernel_shape) v_i = np.random.uniform(-1.0, 1.0, no_weights) v_j = np.random.uniform(-1.0, 1.0, no_weights) v_k = np.random.uniform(-1.0, 1.0, no_weights) # Generate purely imaginary quaternions for i in range(0, no_weights): norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2 + 0.0001) v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(kernel_shape) v_j = v_j.reshape(kernel_shape) v_k = v_k.reshape(kernel_shape) phase = rng.uniform(low=np.pi, high=np.pi, size=kernel_shape) weight_r = modulus * np.cos(phase) weight_i = modulus * v_i * np.sin(phase) weight_j = modulus * v_j * np.sin(phase) weight_k = modulus * v_k * np.sin(phase) return weight_r, weight_i, weight_j, weight_k
def fit(self, X, y=None): """Fit the model with X. Samples a couple of random based vectors to approximate a Gaussian random projection matrix to generate n_components features. Parameters ---------- X : {array-like}, shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ X = check_array(X, order="C", dtype=np.float64) d_orig = X.shape[1] rng = check_random_state(self.random_state) ( self._d, self._n, self._times_to_stack_v, ) = Fastfood._enforce_dimensionality_constraints( d_orig, self.n_components ) self._number_of_features_to_pad_with_zeros = self._d - d_orig self._G = rng.normal(size=(self._times_to_stack_v, self._d)) self._B = rng.choice( [-1, 1], size=(self._times_to_stack_v, self._d), replace=True ) self._P = np.hstack( [ (i * self._d) + rng.permutation(self._d) for i in range(self._times_to_stack_v) ] ) self._S = np.multiply( 1 / self._l2norm_along_axis1(self._G).reshape((-1, 1)), chi.rvs( self._d, size=(self._times_to_stack_v, self._d), random_state=rng, ), ) self._U = self._uniform_vector(rng) self.is_fitted_ = True return self
def vectormap_init(vectormap_dim, in_features, out_features, rng, kernel_size=None, criterion='glorot'): if kernel_size is not None: receptive_field = np.prod(kernel_size) fan_in = in_features * receptive_field fan_out = out_features * receptive_field else: fan_in = in_features fan_out = out_features if criterion == 'glorot': s = np.sqrt(2. / (vectormap_dim * (fan_in + fan_out))) elif criterion == 'he': s = np.sqrt(2. / (vectormap_dim * fan_in)) else: raise ValueError('Invalid criterion: ' + criterion) rng = RandomState(np.random.randint(1, 1234)) if kernel_size is None: kernel_shape = (in_features, out_features) else: if type(kernel_size) is int: kernel_shape = (out_features, in_features) + tuple((kernel_size, )) else: kernel_shape = (out_features, in_features) + (*kernel_size, ) modulus = chi.rvs(vectormap_dim, loc=0, scale=s, size=kernel_shape) number_of_weights = np.prod(kernel_shape) v_s = np.array([ np.random.uniform(-1.0, 1.0, number_of_weights) for _ in range(vectormap_dim - 1) ]) for i in range(0, number_of_weights): v_s[:, i] = v_s[:, i] / np.linalg.norm(v_s[:, i]) v_s = [v.reshape(kernel_shape) for v in v_s] phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight = [modulus * np.cos(phase)] for v in v_s: weight.append(modulus * v * np.sin(phase)) return tuple(weight)
def parameters(n_number, p, d, FLAGS): G = np.random.randn(p * d) B = np.random.uniform(-1, 1, p * d) B[B > 0] = 1 B[B < 0] = -1 # PI_value = np.hstack PI_value = np.hstack([(i * d) + np.random.permutation(d) for i in range(p)]) G_fro = G.reshape(p, d) s_i = chi.rvs(d, size=(p, d)) S = np.multiply(s_i, np.array(np.linalg.norm(G_fro, axis=1)).reshape(p, -1)) S = S.reshape(1, -1) FLAGS.b = np.random.uniform(0, 2 * math.pi, d * p) FLAGS.t = np.random.uniform(-1, 1, d * p) return PI_value, G, B, S
def travelDistance(speed): """ Determine how far an agent moves in a given step Draw from a chi distribution, then multiply by Agent's speed Minimum travel distance is 0.1 * speed Parameters ---------- speed : float, Agent's current speed aka step size (Agent.speed) Returns ------- distanceToTravel : float """ r = np.max([chi.rvs(dfConstant),0.1]) distanceToTravel = r * speed return distanceToTravel
def initialize_conv(in_channels, out_channels, kernel_size=[2, 2], init_mode="he"): """ Initializes quaternion weight parameter for convolution. @type in_channels: int @type out_channels: int @type kernel_size: int/list/tuple @type init_mode: str """ prod = np.prod(kernel_size) if init_mode == "he": scale = 1 / np.sqrt(in_channels * prod * 8) elif init_mode in ["xavier", "glorot"]: scale = 1 / np.sqrt((in_channels + out_channels) * prod * 8) if type(kernel_size) == int: window = [kernel_size, kernel_size] elif type(kernel_size) == tuple: window = list(kernel_size) elif type(kernel_size) == list: window = kernel_size size_real = [in_channels, out_channels] + window size_img = [size_real[0]] + [size_real[1] * 3] + size_real[2:] img_mat = torch.Tensor(*size_img).uniform_(-1, 1) mat = Q(torch.cat([torch.zeros(size_real), img_mat], 1)) mat /= mat.norm() phase = torch.Tensor(*size_real).uniform_(-np.pi, np.pi) magnitude = torch.from_numpy(chi.rvs(4, loc=0, scale=scale, size=size_real)).float() r = magnitude * torch.cos(phase) factor = magnitude * torch.sin(phase) mat *= factor mat += r return mat
def fastfood_value(n_number, T, d, FLAGS, method=1): FLAGS.T = T G = np.random.randn(T * d) B = np.random.uniform(-1, 1, T * d) B[B > 0] = 1 B[B < 0] = -1 # PI_value = np.hstack PI_value = np.hstack([(i * d) + np.random.permutation(d) for i in range(T)]) G_fro = G.reshape(T, d) s_i = chi.rvs(d, size=(T, d)) S = np.multiply(s_i, np.array(np.linalg.norm(G_fro, axis=1)).reshape(T, -1)) S = S.reshape(1, -1) FLAGS.BATCHSIZE = n_number FLAGS.b = None FLAGS.t = None if method == 1: FLAGS.b = np.random.uniform(0, 2 * math.pi, d * T) FLAGS.t = np.random.uniform(-1, 1, d * T) return PI_value, G, B, S
def phm_init(phm_dim: int, in_features: int, out_features: int, low: int = 0, high: int = 1, criterion: str = 'glorot', transpose: bool = True): fan_in = in_features fan_out = out_features if criterion == 'glorot': s = np.sqrt(2. / (phm_dim * (fan_in + fan_out))) elif criterion == 'he': s = np.sqrt(2. / (phm_dim * fan_in)) else: raise ValueError('Invalid criterion: ' + criterion) kernel_shape = (in_features, out_features) magnitude = torch.from_numpy( chi.rvs(phm_dim, loc=0, scale=s, size=kernel_shape)).to(torch.float32) # purely imaginary vectormap v = unitary_init(phm_dim=phm_dim, in_features=in_features, out_features=out_features, low=low, high=high) theta = torch.from_numpy( np.random.uniform(low=-np.pi, high=np.pi, size=kernel_shape)).to(torch.float32) weight = [magnitude * torch.cos(theta)] for vs in v[1:]: weight.append(magnitude * vs * torch.sin(theta)) weight = torch.stack(weight, dim=0) if transpose: weight = weight.permute(0, 2, 1) return weight
def __c_1_lambda_om(_lambda, dim, proj_max): _lambda_half = int(ceil(_lambda / 2.0)) # for memory concern n_sample = len(proj_max) for i in range(n_sample): # mirrored orthogonal sampling q = qr(randn(dim, dim))[0] l = chi.rvs(dim, size=dim) s = l * q samples = s[:, 0:_lambda_half] samples = append(samples, -samples, axis=1) # projection onto e1 proj = samples[0, :] # the largest order statistic proj_sorted = sort(proj) proj_max[i] = proj_sorted[-1]
def test_chi(self): from scipy.stats import chi import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 1) df = 78 mean, var, skew, kurt = chi.stats(df, moments='mvsk') x = np.linspace(chi.ppf(0.01, df), chi.ppf(0.99, df), 100) ax.plot(x, chi.pdf(x, df), 'r-', lw=5, alpha=0.6, label='chi pdf') rv = chi(df) ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf') vals = chi.ppf([0.001, 0.5, 0.999], df) np.allclose([0.001, 0.5, 0.999], chi.cdf(vals, df)) r = chi.rvs(df, size=1000) ax.hist(r, density=True, histtype='stepfilled', alpha=0.2) ax.legend(loc='best', frameon=False) self.assertEqual(str(ax), "AxesSubplot(0.125,0.11;0.775x0.77)")
def quaternion_init(in_features: int, out_features: int, criterion: str = 'glorot', low: int = 0, high: int = 1, transpose: bool = True) -> (Tensor, Tensor, Tensor, Tensor): fan_in = in_features fan_out = out_features if criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + criterion) kernel_shape = (in_features, out_features) magnitude = torch.from_numpy(chi.rvs(df=4, loc=0, scale=s, size=kernel_shape)).to(torch.float64) # Purely imaginary quaternions unitary _, v_i, v_j, v_k = unitary_init(in_features, out_features, low, high) theta = torch.from_numpy(np.random.uniform(low=-np.pi, high=np.pi, size=kernel_shape)).to(torch.float64) phi_i = torch.cos(torch.from_numpy(np.random.uniform(low=-s, high=s, size=kernel_shape)).to(torch.float64))**2 phi_j = torch.cos(torch.from_numpy(np.random.uniform(low=-s, high=s, size=kernel_shape)).to(torch.float64))**2 phi_k = torch.cos(torch.from_numpy(np.random.uniform(low=-s, high=s, size=kernel_shape)).to(torch.float64))**2 phi = phi_i / (phi_i + phi_j + phi_k) phj = phi_j / (phi_i + phi_j + phi_k) phk = phi_k / (phi_i + phi_j + phi_k) weight_r = magnitude * torch.cos(theta) weight_i = magnitude * v_i * torch.sin(theta) * phi weight_j = magnitude * v_j * torch.sin(theta) * phj weight_k = magnitude * v_k * torch.sin(theta) * phk if transpose: weight_r = weight_r.t() weight_i = weight_i.t() weight_j = weight_j.t() weight_k = weight_k.t() return weight_r.to(torch.float32), weight_i.to(torch.float32), weight_j.to(torch.float32), weight_k.to(torch.float32)
def initialize_linear(in_channels, out_channels, init_mode="he"): """ Initializes quaternion weight parameter for linear. It can be shown that the variance for the magnitude is given as 4sigma from a chi-distribution with 4 dof's. The phase is uniformly initialized in [-pi, pi]. The basis vectors are randomly initialized and normalized based on their norm. The whole initialization is performed considering the polar form of the quaternion. @type in_channels: int @type out_channels: int @type init_mode: str """ if init_mode == "he": scale = 1 / np.sqrt(in_channels * 8) elif init_mode in ["xavier", "glorot"]: scale = 1 / np.sqrt((in_channels + out_channels) * 8) size_real = [in_channels, out_channels] size_img = [in_channels, out_channels * 3] img_mat = torch.Tensor(*size_img).uniform_(-1, 1) mat = Q(torch.cat([torch.zeros(size_real), img_mat], 1)) mat /= mat.norm() phase = torch.Tensor(*size_real).uniform_(-np.pi, np.pi) magnitude = torch.from_numpy(chi.rvs(4, loc=0, scale=scale, size=size_real)).float() r = magnitude * torch.cos(phase) factor = magnitude * torch.sin(phase) mat *= factor mat += r return mat
print(opt) ngpu = int(opt.ngpu) nz = int(opt.nz) ngf = int(opt.ngf) nc = 3 netG = NetG(ngf, nz, nc, ngpu) netG.load_state_dict( torch.load(opt.netG, map_location=lambda storage, loc: storage)) netG.eval() print(netG) for j in range(opt.niter): # step 1 r = chi.rvs(df=100) # step 2 u = numpy.random.normal(0, 1, nz) w = numpy.random.normal(0, 1, nz) u /= numpy.linalg.norm(u) w /= numpy.linalg.norm(w) v = w - numpy.dot(u, w) * u v /= numpy.linalg.norm(v) ndimgs = [] for i in range(opt.n_steps): t = float(i) / float(opt.n_steps) # step 3 z = numpy.cos(t * 2 * numpy.pi) * u + numpy.sin(t * 2 * numpy.pi) * v
def _weightsamples(self): s = chi.rvs(self.d2, size=self.G.shape) return self.d2 * s / norm(self.G, axis=1)[:, np.newaxis]
def main(name): #read the dataset and initialize the parameter for hadamard transform x_train, y_train, x_test, y_test = get_data(name) x, y = x_train, y_train n_number, f_num = np.shape(x) FLAGS.BATCHSIZE = n_number d = 2**math.ceil(np.log2(f_num)) T = FLAGS.T G = np.random.randn(T * d) B = np.random.uniform(-1, 1, T * d) B[B > 0] = 1 B[B < 0] = -1 PI_value = np.random.permutation(d) G_fro = G.reshape(T, d) s_i = chi.rvs(d, size=(T, d)) S = np.multiply( s_i, np.array(np.linalg.norm(G_fro, axis=1)**(-0.5)).reshape(T, -1)) S = S.reshape(1, -1) print(np.unique(y)) class_number = len(np.unique(y)) if FLAGS.d_openml != None: ''' lable convert from str to int in openml dataset ''' y_0 = np.zeros((n_number, 1)) for i in range(class_number): y_temp = np.where(y[:] != '%d' % i, -1, 1) y_0 = np.hstack((y_0, np.mat(y_temp).T)) y = y_0[:, 1:] y = np.argmax(y, axis=1) else: y = np.array(np.where(y[:] != 1, -1, 1).reshape(n_number, 1)) print('Training Linear SVM') d = 2**math.ceil(np.log2(f_num)) x_value = np.asmatrix(hadamard(d, f_num, x, G, B, PI_value, S)) clf = LinearSVC() clf.fit(x_value, y) print('linear train', clf.score(x_value, y)) W_fcP = np.asmatrix(-np.ones((T * d, class_number))) print(W_fcP.shape) n_number, f_num = np.shape(x) FLAGS.BATCHSIZE = n_number d = 2**math.ceil(np.log2(f_num)) x_value = np.asmatrix(hadamard(d, f_num, x, G, B, PI_value, S)) print(x_value.shape) x, y = x_test, y_test n_number, f_num = np.shape(x) if FLAGS.d_openml != None: ''' lable convert from str to int in openml dataset ''' y_0 = np.zeros((n_number, 1)) for i in range(class_number): y_temp = np.where(y[:] != '%d' % i, -1, 1) y_0 = np.hstack((y_0, np.mat(y_temp).T)) y = y_0[:, 1:] y = np.argmax(y, axis=1) else: y = np.array(np.where(y[:] != 1, -1, 1).reshape(n_number, 1)) FLAGS.BATCHSIZE = n_number start = time.time() x_value = np.asmatrix(hadamard(d, f_num, x, G, B, PI_value, S)) print('linear predict', clf.score(x_value, y)) print(time.time() - start, 'linear svm')
def radius(d, n): rv = chi.rvs(d, 0, 1, n) #for x in range(0,n)] #for y in range(0,m) return rv
def toFixed(x, fb): v = (1 << fb) if (x >= 0): return int(x * v + 0.5) else: return int(x * v - 0.5) # generate parameters B = rng.randint(2, size=(k, d)) * 2 - 1 G = rng.normal(size=(k, d)) S = (1 / (sigma * np.sqrt(d))) * np.multiply( 1 / l2norm_along_axis1(G).reshape((-1, 1)), chi.rvs(d, size=(k, d))) alpha = rng.normal(size=(1, n)) x = np.linspace(0, npts, npts) / float(1 << (int(np.log2(npts)) - 1)) b = rng.uniform(0, 1, size=npts) * 2 * np.pi A = np.sqrt(2. / n) cos = A * np.cos(x * np.pi + 0) # save parameters to csv and cosTab.txt """ np.savetxt("params/B.csv", B, delimiter=",") np.savetxt("params/G.csv", G, delimiter=",") np.savetxt("params/S.csv", S, delimiter=",") np.savetxt("params/alpha.csv", alpha, delimiter=",") """
def quaternion_init(in_features, out_features, kernel_size=None, criterion="glorot"): """Returns a matrix of quaternion numbers initialised with the method described in "Quaternion Recurrent Neural Network " - Parcollt T. Arguments --------- in_features : int Number of real values of the input layer (quaternion // 4) out_features : int Number of real values of the output layer (quaternion // 4) kernel_size : int Kernel_size for convolutional layers (ex: (3,3)) criterion: str, (glorot, he) """ # We set the numpy seed equal to the torch seed for reproducibility # Indeed we use numpy and scipy here. We need % (2**31-1) or, if the # seed hasn't been set by the used in the YAML file, torch will generate # a double that would be to big for numpy. np.random.seed(seed=torch.initial_seed() % (2**31 - 1)) if kernel_size is not None: receptive_field = np.prod(kernel_size) fan_in = in_features * receptive_field fan_out = out_features * receptive_field else: fan_in = in_features fan_out = out_features if criterion == "glorot": s = 1.0 / np.sqrt(2 * (fan_in + fan_out)) else: s = 1.0 / np.sqrt(2 * fan_in) # Generating randoms and purely imaginary quaternions : if kernel_size is None: kernel_shape = (in_features, out_features) else: if type(kernel_size) is int: kernel_shape = (out_features, in_features) + tuple((kernel_size, )) else: kernel_shape = (out_features, in_features) + (*kernel_size, ) modulus = torch.from_numpy(chi.rvs(4, loc=0, scale=s, size=kernel_shape)) number_of_weights = np.prod(kernel_shape) v_i = torch.FloatTensor(number_of_weights).uniform_(-1, 1) v_j = torch.FloatTensor(number_of_weights).uniform_(-1, 1) v_k = torch.FloatTensor(number_of_weights).uniform_(-1, 1) # Purely imaginary quaternions unitary for i in range(0, number_of_weights): norm = torch.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001 v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(kernel_shape) v_j = v_j.reshape(kernel_shape) v_k = v_k.reshape(kernel_shape) phase = torch.rand(kernel_shape).uniform_(-math.pi, math.pi) weight_r = modulus * torch.cos(phase) weight_i = modulus * v_i * torch.sin(phase) weight_j = modulus * v_j * torch.sin(phase) weight_k = modulus * v_k * torch.sin(phase) return (weight_r, weight_i, weight_j, weight_k)
def fit(self, gType=1): self.T = gType if self.verbose: print "Gaussian Matrix: " print "----------------" if gType == 0: s = " N(0,1)\n" elif gType == 1: s = " {-1,1}\n" elif gType == 2: s = " {-1,0,1}\n" else: print "Error: select G marix" raise Exception print s # --- Software Diagonal Matrices --- self.B = self.rng.randint(2, size=(self.k, self.d)) * 2 - 1 if self.T == 0: self.G = self.rng.normal(size=(self.k, self.d)) coeff = 1 elif self.T == 1: coeff, self.G = sparse_random_matrix(self.k, self.d, density=1., random_state=self.rng) elif self.T == 2: coeff, self.G = sparse_random_matrix(self.k, self.d, density=self.density, random_state=self.rng) else: raise Exception self.P = np.hstack([(i * self.d) + self.rng.permutation(self.d) for i in range(self.k)]) np.random.seed(seed=23) self.S = np.multiply( 1 / l2norm_along_axis1(coeff * self.G).reshape((-1, 1)), chi.rvs(self.d, size=(self.k, self.d))) self.S = self.S * coeff self.S_hw = (1 / (self.sigma * np.sqrt(self.d)) ) * self.S #taken from scale_transform self.U = self.rng.uniform(0, 2 * np.pi, size=self.n) self.U_hw = 2 * np.pi * self.U self.A_hw = np.sqrt(2. / self.n) #amplitude of cosine LUT # --- Hardware RAM-based Shift Registers ---- ''' self.Vp - after the permutation self.Vg - after the gaussian self.Vf - after the 2nd Hadamard self.H - hadamard matrix ''' B = self.B if self.pad > 0: B[:, -self.pad:] *= 0 H = HadamardMatrix(n=self.d) # for each k in B, multiply the array elementwise with each row of H and vstack rows V = np.vstack([np.multiply(B[i], H) for i in range(self.k)]) # apply permutation to inidces (n indices for n_dicts) p = np.arange(self.n).reshape(1, -1) np.take(p, self.P, axis=1, mode='wrap', out=p) p = np.ravel(p) # then apply permutation of the indices to V. need to transpose because np.take works on cols self.Vp = np.take(V.T, p, axis=1, mode='wrap').T self.Vg = np.multiply(np.ravel(self.G), self.Vp.T).T # reshape H(d, d, 1) H = H.reshape(H.shape[0], H.shape[1], 1) # multiply combination of inputs after G with hadamard transform Vh = np.vstack([ np.multiply(self.Vg.reshape(self.k, self.d, self.d)[i], H) for i in range(self.k) ]) # simplify the expression, so it's represented as a multiple of the input (i.e. 0 + x2 -4x3 + x4) Vf = np.zeros(shape=(self.d * self.k, self.d)) for i in range(self.d * self.k): for j in range(self.d): Vf[i] += Vh[i][j] self.Vf = Vf self.H = HadamardMatrix(n=self.d) # remove zero columns if self.pad > 0: self.Vp = self.Vp[:, :-self.pad] self.Vg = self.Vg[:, :-self.pad] self.Vf = self.Vf[:, :-self.pad]
def mutation(self): #---------------------------- Mutation -------------------------------- # Mirroring mode = self.sampling_method dim, _lambda, sigma, evalcount, scale, aux = self.dim, self._lambda, self.sigma, self.evalcount, \ self.scale, self.aux if mode == 1 or mode == 11: if mod(evalcount + _lambda, 2) != 0: half = int(ceil(_lambda / 2.)) z = randn(dim, half) aux = -z[:, -1].reshape(-1, 1) z = append(z, -z[:, :-1], axis=1) else: half = int(floor(_lambda / 2.)) z = randn(dim, half) z = append(z, -z, axis=1) if len(aux) != 0: z = append(aux, z, axis=1) self.half = half # Derandomized step size elif mode == 3: z = randn(dim, _lambda) z = scale * z / sqrt(sum(power(z, 2), 0)) # Orthogonal mirrored sampling elif mode == 4 or mode == 7 or mode == 44: if mod(evalcount + _lambda, 2) != 0: half = int(ceil(_lambda / 2.)) z = zeros((dim, half)) n = int(min([dim, half])) if dim < half: z[:, dim:] = randn(dim, half - dim) q = qr(randn(dim, n))[0] l = chi.rvs(dim, size=n) z[:, 0:n] = l * q aux = -z[:, -1].reshape(-1, 1) z = append(z, -z[:, :-1], axis=1) else: half = int(floor(_lambda / 2.)) z = zeros((dim, half)) n = int(min([dim, half])) if dim < half: z[:, dim:] = randn(dim, half - dim) q = qr(randn(dim, n))[0] l = chi.rvs(dim, size=n) z[:, 0:n] = l * q z = append(z, -z, axis=1) if len(aux) != 0: z = append(aux, z, axis=1) self.half = half # Orthogonal mirrored sampling...well elif mode == 8: if mod(evalcount + _lambda, 2) != 0: half = ceil(_lambda / 2.) z = zeros((dim, half)) n = min([dim, half]) if dim < half: z[:, dim:] = randn(dim, half - dim) tmp = randn(dim, n) l = sqrt(np.sum(power(tmp, 2), 0)) q = qr(tmp)[0] z[:, 0:n] = l * q aux = -z[:, -1].reshape(-1, 1) z = append(z, -z[:, :-1], axis=1) else: half = floor(_lambda / 2.) z = zeros((dim, half)) n = min([dim, half]) if dim < half: z[:, dim:] = randn(dim, half - dim) tmp = randn(dim, n) l = sqrt(np.sum(power(tmp, 2), 0)) q = qr(tmp)[0] z[:, 0:n] = l * q z = append(z, -z, axis=1) if len(aux) != 0: z = append(aux, z, axis=1) self.half = half # Orthogonal sampling (random rotation) elif mode == 5: z = dot(rand_orth_mat(dim), eye(dim)) n = dim if dim > _lambda: p = arange(0, dim) shuffle(p) z = z[:, p[0:_lambda]] n = _lambda l = chi.rvs(dim, size=n) sign = rand(n) sign[sign > .5] = 1 sign[sign <= .5] = -1 z = sign * l * z if dim < _lambda: ss = randn(dim, _lambda - dim) z = append(z, ss, axis=1) # Orthogonal sampling (Gram-Schmidt) elif mode == 6: z = zeros((dim, _lambda)) n = min([dim, _lambda]) if dim < _lambda: z[:, dim:] = randn(dim, _lambda - dim) q = qr(randn(dim, n))[0] l = chi.rvs(dim, size=n) elif mode == 9: if mod(evalcount + _lambda, 2) != 0: half = int(ceil(_lambda / 2.)) z = randn(dim, half) q = randn(dim, half) p = array([q[:, i] - inner(q[:, i], z[:, i])*z[:, i] / norm(z[:, i])**2 \ for i in range(half)]).T p = (p / sqrt(sum(power(p, 2), 0))) * sqrt(sum(power(q, 2), 0)) aux = p[:, -1].reshape(-1, 1) z = append(z, p[:, :-1], axis=1) else: half = int(floor(_lambda / 2.)) z = randn(dim, half) q = randn(dim, half) p = array([q[:, i] - inner(q[:, i], z[:, i])*z[:, i] / norm(z[:, i])**2 \ for i in range(half)]).T p = (p / sqrt(sum(power(p, 2), 0))) * sqrt(sum(power(q, 2), 0)) z = append(z, p, axis=1) if len(aux) != 0: z = append(aux, z, axis=1) self.half = half # Standard mutation else: z = randn(dim, _lambda) self.z = z self.offspring = add(self.wcm, sigma * dot(self.e_vector, self.e_value * self.z))
def chi(self): p = chi.rvs(*self.dist_parms[:-2], loc=self.dist_parms[-2], scale=self.dist_parms[-1]) return max(0, p)
def estimate_sigma(observed, truncated_df, lower_bound, upper_bound, untruncated_df=0, factor=3, npts=50, nsample=2000): """ Produce an estimate of $\sigma$ from a constrained error sum of squares. The relevant distribution is a scaled $\chi^2$ restricted to $[0,U]$ where $U$ is `upper_bound`. Parameters ---------- observed : float The observed sum of squares. truncated_df : float Degrees of freedom of the truncated $\chi^2$ in the sum of squares. The observed sum is assumed to be the sum of an independent untruncated $\chi^2$ and the truncated one. lower_bound : float Lower limit of truncation interval. upper_bound : float Upper limit of truncation interval. untruncated_df : float Degrees of freedom of the untruncated $\chi^2$ in the sum of squares. factor : float Range of candidate values is [observed/factor, observed*factor] npts : int How many candidate values for interpolator. nsample : int How many samples for each expected value of the truncated sum of squares. Returns ------- sigma_hat : np.float Estimate of $\sigma$. """ if untruncated_df < 50: linear_term = truncated_df**(0.5) else: linear_term = 0 total_df = untruncated_df + truncated_df values = np.linspace(1./factor, factor, npts) * observed expected = 0 * values for i, value in enumerate(values): P_upper = chidist.cdf(upper_bound * np.sqrt(truncated_df) / value, truncated_df) P_lower = chidist.cdf(lower_bound * np.sqrt(truncated_df) / value, truncated_df) U = np.random.sample(nsample) if untruncated_df > 0: sample = (chidist.ppf((P_upper - P_lower) * U + P_lower, truncated_df)**2 + chidist.rvs(untruncated_df, size=nsample)**2) * value**2 else: sample = (chidist.ppf((P_upper - P_lower) * U + P_lower, truncated_df) * value)**2 expected[i] = np.mean(sample) if expected[i] >= 1.5 * (observed**2 * total_df + observed**2 * linear_term): break interpolant = interp1d(values, expected + values**2 * linear_term) V = np.linspace(1./factor,factor,10*npts) * observed # this solves for the solution to # expected(sigma) + sqrt(df) * sigma^2 = observed SS * (1 + sqrt(df)) # the usual "MAP" estimator would have RHS just observed SS # but this factor seems to correct it. # it is such that if there were no selection it would be # the usual unbiased estimate try: sigma_hat = np.min(V[interpolant(V) >= observed**2 * total_df + observed**2 * linear_term]) except ValueError: # no solution, just return observed sigma_hat = observed return sigma_hat
# Display the probability density function (``pdf``): x = np.linspace(chi.ppf(0.01, df), chi.ppf(0.99, df), 100) ax.plot(x, chi.pdf(x, df), 'r-', lw=5, alpha=0.6, label='chi pdf') # Alternatively, the distribution object can be called (as a function) # to fix the shape, location and scale parameters. This returns a "frozen" # RV object holding the given parameters fixed. # Freeze the distribution and display the frozen ``pdf``: rv = chi(df) ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf') # Check accuracy of ``cdf`` and ``ppf``: vals = chi.ppf([0.001, 0.5, 0.999], df) np.allclose([0.001, 0.5, 0.999], chi.cdf(vals, df)) # True # Generate random numbers: r = chi.rvs(df, size=1000) # And compare the histogram: ax.hist(r, normed=True, histtype='stepfilled', alpha=0.2) ax.legend(loc='best', frameon=False) plt.show()
def main(name): fig = plt.figure() ax1 = fig.add_subplot(111) ax2 = ax1.twinx() ''' read data and parameter initialize for the hadamard transform ''' x_train, y_train, x_test, y_test = get_data(name) acc_linear = np.zeros(2) acc_binary = np.zeros(2) acc_random = np.zeros(2) time_linear = np.zeros(2) time_binary = np.zeros(2) time_random = np.zeros(2) marker = ['.', 'v', '^', 's', '*'] for iter in range(5): iter_loss = [] iter_acc = [] T = 2**iter print(T) x, y = x_train, y_train n_number, f_num = np.shape(x) d = 2**math.ceil(np.log2(f_num)) G = np.random.randn(T * d) B = np.random.uniform(-1, 1, T * d) B[B > 0] = 1 B[B < 0] = -1 # PI_value = np.hstack PI_value = np.hstack([(i * d) + np.random.permutation(d) for i in range(T)]) G_fro = G.reshape(T, d) s_i = chi.rvs(d, size=(T, d)) S = np.multiply( s_i, np.array(np.linalg.norm(G_fro, axis=1)**(-0.1)).reshape(T, -1)) S = S.reshape(1, -1) FLAGS.BATCHSIZE = n_number class_number = len(np.unique(y)) if class_number == 2: class_number -= 1 x_value = np.asmatrix(hadamard(d, f_num, x, G, B, PI_value, S, T)) y_temp = label_processing(y, n_number) clf = LinearSVC() clf.fit(x_value, y) # print(clf.score(x_value,y)) print('Training coordinate descent initiate from result of linear svm') W_fcP = clf.coef_ W_fcP = np.asmatrix(np.sign(W_fcP.reshape(-1, class_number))) W = W_fcP n_number, project_d = x_value.shape test_number, f_num = np.shape(x_test) FLAGS.BATCHSIZE = test_number test_x = np.asmatrix(hadamard(d, f_num, x_test, G, B, PI_value, S, T)) test_y = label_processing(y_test, test_number) # original optimization method for c in range(class_number): W_temp = W[:, c] y_temp_c = y_temp[:, c] init = np.dot(x_value, W_temp) hinge_loss = sklearn.metrics.hinge_loss(y_temp_c, init) * n_number loss_new = np.sum(hinge_loss) if class_number != 1: predict = np.argmax(np.array(np.dot(test_x, W_temp)), axis=1) y_lable = np.argmax(test_y, axis=1) acc = accuracy_score(np.array(y_lable), np.array(predict)) else: predict = np.array(np.dot(test_x, W_temp)) acc = accuracy_score(np.sign(test_y), np.sign(predict)) iter_loss.append(loss_new) iter_acc.append(acc) loss_old = 2 * loss_new # while (loss_old - loss_new) / loss_old >= 1e-6: # loss_old = loss_new # for i in range(project_d): # derta = init - np.multiply(W_temp[i], x_value[:, i]) * 2 # loss = sklearn.metrics.hinge_loss(y_temp_c, derta) * n_number # if loss < loss_new: # loss_new = loss # init = derta # W_temp[i] = -W_temp[i] # if class_number != 1: # predict = np.argmax(np.array(np.dot(test_x, W_temp)), axis=1) # y_lable = np.argmax(test_y, axis=1) # acc = accuracy_score(np.array(y_lable), np.array(predict)) # else: # predict = np.array(np.dot(test_x, W_temp)) # acc = accuracy_score(np.sign(test_y), np.sign(predict)) plt.scatter(test_x, test_y) plt.show() exit()
os.system('mkdir -p {}'.format(output_dir)) print(opt) ngpu = int(opt.ngpu) nz = int(opt.nz) ngf = int(opt.ngf) nc = 3 netG = NetG(ngf, nz, nc, ngpu) netG.load_state_dict(torch.load(opt.netG, map_location=lambda storage, loc: storage)) netG.eval() print(netG) for j in range(opt.niter): # step 1 r = chi.rvs(df=100) # step 2 u = numpy.random.normal(0, 1, nz) w = numpy.random.normal(0, 1, nz) u /= numpy.linalg.norm(u) w /= numpy.linalg.norm(w) v = w - numpy.dot(u, w) * u v /= numpy.linalg.norm(v) ndimgs = [] for i in range(opt.n_steps): t = float(i) / float(opt.n_steps) # step 3 z = numpy.cos(t * 2 * numpy.pi) * u + numpy.sin(t * 2 * numpy.pi) * v