def hadamard2(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1): p = FLAGS.p x_ = batch n = x_.shape[0] x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])), 'constant', constant_values=(0, 0)) x_ = np.tile(x_, (1, p)) x_i = np.multiply(x_, B) x_ = x_i.reshape(n * p, d) for i in range(n * p): ffht.fht(x_[i]) x_ = x_.reshape(n, p * d) np.take(x_, PI_value, axis=1, mode='wrap', out=x_) x_transformed = np.multiply(x_, G) x_ = np.reshape(x_transformed, (n * p, d)) for i in range(n * p): ffht.fht(x_[i]) x_ = x_.reshape(n, p * d) x_value = np.multiply(x_, S) x_value = x_value / (sigma * (np.sqrt(d)**3)) x_value = np.cos(x_value + FLAGS.b) + FLAGS.t x_value = np.sign(x_value) return x_value
def fastfood(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1): start = time.time() T = FLAGS.T x_ = batch n = x_.shape[0] # x.shape [batch,d] x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])), 'constant', constant_values=(0, 0)) x_ = np.tile(x_, (1, T)) x_i = np.multiply(x_, S) x_ = x_i.reshape(FLAGS.BATCHSIZE * T, d) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) x_transformed = np.multiply(x_, G) np.take(x_, PI_value, axis=1, mode='wrap', out=x_) x_ = np.reshape(x_transformed, (FLAGS.BATCHSIZE * T, d)) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) # print(np.linalg.norm(x_[:2],axis=1)) # print(np.mean(x_[:,:2], axis=0)) x_value = np.multiply(x_, B) # print(np.linalg.norm(x_value[:2], axis=1)) # print(np.any(FLAGS.b)) x_value = (sigma**2) * x_value / np.sqrt(d) x_value = np.cos(x_value) # print(time.time()-start) return x_value
def transform(name): np.set_printoptions(threshold=np.inf, suppress=True) train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/train" % (name) # data = load_svmlight_file(train_url) train, tra_label = load_svmlight_file(train_url) train = train.todense() tra_label = np.mat(tra_label).T b_num,n_feature = np.shape(train) if n_feature < 4000: train, n_feature = ped(train) print(train.shape) start = time.process_time() else: start = time.process_time() sub = int(math.pow(2,12)) transformer = cwt_matrix(sub,n_feature) train = scipy.sparse.csr_matrix.dot(train, transformer.T) n_feature = sub rng = check_random_state(None) array = np.mat(rng.choice([1, -1], n_feature)) n_sample, d_feature = np.shape(train) a = np.array(np.multiply(train, array)) for i in range(n_sample): ffht.fht(a[i]) transform_time = time.process_time() - start return transform_time,a,array
def Fastfood_updata(self): p = FLAGS.p ''' using the sigmoid ''' self.gradient = np.dot((self.prediction-self.label),self.wb.T/self.scaling)/self.original.shape[0] ''' using the softmax ''' self.gradient = np.multiply(self.gradient,-np.sin(self.x_transfer + FLAGS.b)) self.gradient = np.array(self.gradient) S = copy.deepcopy(self.S) self.S -= self.update_S() self.gradient= np.multiply(S,self.gradient) # gradient update after S self.gradient = self.gradient.reshape(-1,self.dimension) for i in range(self.gradient.shape[0]): ffht.fht(self.gradient[i]) # self.gradient = self.gradient/(2**(self.dimension/2))# gradient update after H self.gradient = self.gradient.reshape(-1, p*self.dimension)/(np.sqrt(self.dimension) ) G = copy.deepcopy(self.G) self.G -= self.update_G()
def HD_x(D_by_sqrt_d, pad, x): # (d^{-1/2}) * [ Dx 0 ... 0 ] HDx = np.concatenate([D_by_sqrt_d * x, pad]) # (d^{-1/2}) * H [ Dx 0 ... 0 ] fht(HDx) return HDx
def HGPHD_x(D_by_sqrt_d, pad, P_seed, G, x): # (1/d) * HDx HDx = np.concatenate([x * D_by_sqrt_d, pad]) fht(HDx) # (1/d) * GPHDx HGPHDx = G * rperm(P_seed, HDx) # (1/d) * HGPHDx fht(HGPHDx) return HGPHDx
def Forwardpropogation(self): p = FLAGS.p x_ = self.batch n = x_.shape[0] x_ = np.pad(x_, ((0, 0), (0, self.dimension - x_.shape[1])), 'constant', constant_values=(0, 0)) x_ = np.tile(x_, (1, p)) self.original = x_ self.x_B = np.multiply(x_, self.B) x_ = self.x_B.reshape(n * p, self.dimension) for i in range(n * p): ffht.fht(x_[i]) x_ = x_.reshape(n, p * self.dimension) self.x_G = np.take(x_, self.PI_value, axis=1, mode='wrap') x_transformed = np.multiply(self.x_G, self.G) x_ = np.reshape(x_transformed, (n * p, self.dimension)) for i in range(n * p): ffht.fht(x_[i]) self.x_S = x_.reshape(n, p * self.dimension) x_ = np.multiply(self.x_S, self.S) # self.x_transfer = np.sign(x_) self.x_transfer = copy.deepcopy(x_) # print(np.max(self.x_transfer),np.mean(self.x_transfer),np.min(self.x_transfer)) # exit() x_transfer = np.cos(x_ + FLAGS.b) + FLAGS.t x_ = np.sign(x_transfer) self.wb = np.sign(self.coefficients) ''' using the sigmoid function we will use the sigmoid activation for the multiclass problem ''' prediction = x_.dot(np.sign(self.coefficients)) self.ywx = np.multiply(self.label, prediction) self.prediction = np.where(prediction > 0, 1, -1) self.loss = np.sum(np.clip(1 - self.ywx, 0, None)) / x_.shape[0] # self.loss = sklearn.metrics.log_loss(self.label, prediction) self.ywx = np.where(self.ywx < 1, 1, 0) self.gradient = -np.multiply(np.mean(self.ywx, axis=0), np.dot(x_.T, self.label)) # print(np.count_nonzero(self.gradient)) # print(self.gradient[0,:10]) # print(self.coefficients[0,:10]) '''
def hadamard(d, f_num, batch, G, B, PI_value, S): T = FLAGS.T x_ = batch x_ = np.pad(x_, ((0, 0), (0, d - f_num)), 'constant', constant_values=(0, 0)) # x.shape [batch,d] x_ = np.tile(x_, (1, T)) x_i = np.multiply(x_, S) x_ = x_i.reshape(FLAGS.BATCHSIZE, T, d) h = 1 # while h < d: # for i in range(0, d, h * 2): # for j in range(i, i + h): # a = x_[:, :, j] # b = x_[:, :, j + h] # temp = a - b # x_[:, :, j] = a + b # x_[:, :, j + h] = temp # h *= 2 for i in range(x_.shape[0]): for j in range(T): ffht.fht(x_[i, j]) x_transformed = np.multiply(x_.reshape(FLAGS.BATCHSIZE, d * T), G) x_transformed = np.reshape(x_transformed, (FLAGS.BATCHSIZE, T, d)) x_permutation = x_transformed[:, :, PI_value] x_ = x_permutation h = 1 while h < d: for i in range(0, d, h * 2): for j in range(i, i + h): a = x_[:, :, j] b = x_[:, :, j + h] temp = a - b x_[:, :, j] = a + b x_[:, :, j + h] = temp h *= 2 # for i in range(x_.shape[0]): # for j in range(T): # ffht.fht(x_[i,j]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) x_value = np.multiply(x_, B) x_value = np.sign(x_value) return x_value
def Hx(dim, reps) : t = 0 for i in range(reps) : x = np.random.normal(size=dim) start = timeit.default_timer() d = fht(x) stop = timeit.default_timer() t += (stop - start) return t
def main(): d = 668426 K = 1000 Q = K * 12 d_aug = 2**np.ceil(np.log2(d)).astype('int32') rng = np.random.default_rng(2) Phi_row_idx = rng.choice(d_aug, Q, replace=False) time_total = 0 for ii in range(100): rng = np.random.default_rng() supp = rng.choice(d, K, replace=False) g0 = np.zeros(d) g0[supp] = np.random.randn(K) g0 += np.random.randn(d) * 0.01 supp = np.argpartition(np.abs(g0), -K)[-K:] g0_sp = np.zeros(d) g0_sp[supp] = g0[supp] g0_wht = np.concatenate((g0, np.zeros(d_aug - d))) / np.sqrt(Q) ffht.fht(g0_wht) y = g0_wht[Phi_row_idx] t1 = timeit.default_timer() g_rec = FIHT.FastIHT_WHT(y, K, Q, d_aug, Phi_row_idx, top_k_func=1)[0:d] t2 = timeit.default_timer() time_total += t2 - t1 rel_diff_rec = np.linalg.norm(g_rec - g0) / np.linalg.norm(g0) rel_diff_bestK = np.linalg.norm(g0_sp - g0) / np.linalg.norm(g0) print("||g_rec - g|| / ||g|| = ", rel_diff_rec) if rel_diff_bestK != 0: print("||g_bestK - g|| / ||g|| = ", rel_diff_bestK) print("amplification factor = ", rel_diff_rec / rel_diff_bestK) print(time_total / 100)
def Forwardpropogation(self): p = FLAGS.p x_ = self.batch n = x_.shape[0] x_ = np.pad(x_, ((0, 0), (0, self.dimension - x_.shape[1])), 'constant', constant_values=(0, 0)) x_ = np.tile(x_, (1, p)) self.original = x_ self.x_B = np.multiply(x_, self.B) x_ = self.x_B.reshape(n * p, self.dimension) for i in range(n * p): ffht.fht(x_[i]) x_ = x_.reshape(n, p * self.dimension)/(np.sqrt(self.dimension) ) self.x_G = np.take(x_, self.PI_value, axis=1, mode='wrap') x_transformed = np.multiply(self.x_G, self.G) x_ = np.reshape(x_transformed, (n * p, self.dimension)) for i in range(n * p): ffht.fht(x_[i]) self.x_S = x_.reshape(n, p * self.dimension)/(np.sqrt(self.dimension) ) x_= np.multiply(self.x_S, self.S) # self.x_transfer = np.sign(x_) self.x_transfer = copy.deepcopy(x_) x_transfer= np.cos(x_+ FLAGS.b) + FLAGS.t ''' control whether the feature and coefficients are binary ''' x_= np.sign(x_transfer) # x_ = x_transfer self.wb = np.sign(self.coefficients) # self.wb = copy.deepcopy(self.coefficients) ''' using the sigmoid function we will use the sigmoid activation for the multiclass problem ''' prediction = 1/(1+np.exp(-x_.dot(self.wb)/self.scaling) ) self.prediction = prediction self.loss = sklearn.metrics.log_loss(self.label, prediction) self.gradient = np.dot(x_.T/self.scaling, prediction - self.label) / x_.shape[0] '''
def hadamard(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1): start = time.time() T = FLAGS.T kf = KFold(n_splits=2) x_return = np.zeros((1, T * d)) for train_index, test_index in kf.split(batch): x_ = batch[test_index] # print(x_.shape) n = x_.shape[0] # x.shape [batch,d] x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])), 'constant', constant_values=(0, 0)) x_ = np.tile(x_, (1, T)) x_i = np.multiply(x_, S) x_ = x_i.reshape(n * T, d) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(n, T * d) x_transformed = np.multiply(x_, G) np.take(x_, PI_value, axis=1, mode='wrap', out=x_) x_ = np.reshape(x_transformed, (n * T, d)) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(n, T * d) # print(np.linalg.norm(x_[:2],axis=1)) # print(np.mean(x_[:,:2], axis=0)) x_value = np.multiply(x_, B) # print(np.linalg.norm(x_value[:2], axis=1)) # print(np.any(FLAGS.b)) if np.any(FLAGS.b): # print(1) x_value = x_value / (sigma * np.sqrt(d)**3) x_value = np.cos(x_value + FLAGS.b) + FLAGS.t # print(time.time()-start) x_value = np.sign(x_value) x_return = np.vstack((x_return, x_value)) return x_return[1:, :]
def hadamard(d, f_num, batch, G, B, PI_value, S): T = FLAGS.T x_ = batch x_ = np.pad(x_, ((0, 0), (0, d - f_num)), 'constant', constant_values=(0, 0)) # x.shape [batch,d], pad to d = 2^p x_ = np.tile(x_, (1, T)) # x_i = np.multiply(x_, S) x_ = x_i.reshape(FLAGS.BATCHSIZE, T, d) for i in range(x_.shape[0]): for j in range(T): ffht.fht(x_[i, j]) x_transformed = np.multiply(x_.reshape(FLAGS.BATCHSIZE, d * T), G) x_transformed = np.reshape(x_transformed, (FLAGS.BATCHSIZE, T, d)) x_ = x_transformed for i in range(x_.shape[0]): for j in range(T): ffht.fht(x_[i, j]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) x_value = np.multiply(x_, B) x_value = np.sign(x_value) return x_value
def hadamard(d, f_num, batch, G, B, PI_value, S, T): x_ = batch n = x_.shape[0] x_ = np.pad(x_, ((0, 0), (0, d - f_num)), 'constant', constant_values=(0, 0)) # x.shape [batch,d] x_ = np.tile(x_, (1, T)) x_i = np.multiply(x_, S) x_ = x_i.reshape(FLAGS.BATCHSIZE * T, d) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) np.take(x_, PI_value, axis=1, mode='wrap', out=x_) x_transformed = np.multiply(x_, G) x_ = np.reshape(x_transformed, (FLAGS.BATCHSIZE * T, d)) for i in range(n * T): ffht.fht(x_[i]) x_ = x_.reshape(FLAGS.BATCHSIZE, T * d) x_value = np.multiply(x_, B) x_value = np.sign(x_value) # exit() return x_value
def test_transform(name,array): np.set_printoptions(threshold=np.inf, suppress=True) train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/test" % (name) # data = load_svmlight_file(train_url) train, tra_label = load_svmlight_file(train_url) train = train.todense() tra_label = np.mat(tra_label).T b_num,n_feature = np.shape(train) if n_feature < 10000: train, n_feature = ped(train) start = time.process_time() else: start = time.process_time() sub = int(math.pow(2,12)) transformer = cwt_matrix(sub,n_feature) train = scipy.dot(train[:,:n_feature], transformer.T) n_feature = sub a = np.array(np.multiply(train, array)) for i in range(b_num): ffht.fht(a[i]) transform_time = time.process_time() - start return transform_time,a
def Fastfood_updata(self): p = FLAGS.p # print(self.label.shape) self.gradient = -np.multiply( np.mean(self.ywx, axis=1).reshape(-1, 1), np.dot((self.label) / self.original.shape[0], self.wb.T)) # print(self.gradient.shape) self.gradient = np.multiply(self.gradient, -np.sin(self.x_transfer + FLAGS.b)) self.gradient = np.array(self.gradient) S = copy.deepcopy(self.S) self.S -= self.update_S() self.gradient = np.multiply(S, self.gradient) # gradient update after S self.gradient = self.gradient.reshape(-1, self.dimension) for i in range(self.gradient.shape[0]): ffht.fht(self.gradient[i]) # self.gradient = self.gradient/(2**(self.dimension/2))# gradient update after H self.gradient = self.gradient.reshape(-1, p * self.dimension) G = copy.deepcopy(self.G) self.G -= self.update_G()
def wht(msgin): for i in range(msgin.shape[0]): ffht.fht(msgin[i, :]) return msgin
psi_FFHT = np.zeros((2 * basis.Ns, ), dtype=np.float64) psi_FFHT[:basis.Ns] = psi_0.real.copy() psi_FFHT[basis.Ns:] = psi_0.imag.copy() psi_FFHT_cpx = np.zeros((basis.Ns, ), dtype=np.complex128) norm = basis.Ns #np.sqrt(2)**L a = np.zeros_like(psi_0) t1 = timeit.default_timer() for j in range(nT): # evolve state using FFHT # apply FHT ffht.fht(psi_FFHT[:basis.Ns]) ffht.fht(psi_FFHT[basis.Ns:]) #psi_FFHT/=norm # apply Ux ''' (a+ib)(c+id)=ac-bd + i(ad+bc) z=cosHx_diagonal*psi_FFHT[:basis.Ns] - sinHx_diagonal*psi_FFHT[basis.Ns:] psi_FFHT[basis.Ns:] = sinHx_diagonal*psi_FFHT[:basis.Ns] + cosHx_diagonal*psi_FFHT[basis.Ns:] psi_FFHT[:basis.Ns]=z[:] ''' a = sinHx_diagonal * psi_FFHT[basis.Ns:] psi_FFHT[basis.Ns:] *= cosHx_diagonal psi_FFHT[basis.Ns:] += sinHx_diagonal * psi_FFHT[:basis.Ns] psi_FFHT[:basis.Ns] *= cosHx_diagonal psi_FFHT[:basis.Ns] -= a
def generater(name, what = 'train_0', what2 = 'test_0'): np.set_printoptions(threshold=np.inf, suppress=True) train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/train" % (name) # data = load_svmlight_file(train_url) train, tra_label = load_svmlight_file(train_url) tra_label = np.mat(tra_label).T d_data,n_feature = np.shape(train) train_shape = n_feature # sub = int(math.pow(2,f_sub)) if n_feature>10000: sub = int(math.pow(2,13)) transformer = cwt_matrix(sub,n_feature) train = scipy.sparse.csr_matrix.dot(train, transformer.T) n_feature = sub else: train, n_feature = ped(train) array = np.mat(np.random.uniform(-1, 1, n_feature)) # -1 +1均匀采样 array[array < 0] = -1 array[array > 0] = 1 label = np.where(tra_label[:, 0] != 1, -1, 1) n_sample, d_feature = np.shape(train) train = train.todense() # scaler = preprocessing.StandardScaler(with_std=False).fit(train) scaler = preprocessing.StandardScaler(with_std=False).fit(train) train = scaler.transform(train) # print(0) a = np.asarray(np.multiply(np.asarray(train), array)) for i in range(a.shape[0]): # print(a[i].shape) ffht.fht(a[i]) train = np.around(a,decimals=4) train_lable = (np.asarray(label)).reshape(-1) # dump_svmlight_file(np.around(a, decimals=4),(np.asarray(label)).reshape(-1),'svm/BudgetedSVM/original/%s/%s'%(name,what),zero_based=False) del a gc.collect() test_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/test" % (name) test, te_label = load_svmlight_file(test_url) # test, n_feature = ped(test) te_label = np.mat(te_label).T label = np.where(te_label[:, 0] != 1, -1, 1) n_number, n_feature = np.shape(test) if n_feature > 10000: if n_feature>train_shape: # print(train_shape,n_feature) test = scipy.sparse.csr_matrix.dot(test[:,:train_shape], transformer.T) elif n_feature == train_shape: test = scipy.sparse.csr_matrix.dot(test, transformer.T) else: pad = np.zeros((n_number,train_shape-n_feature)) test = scipy.sparse.hstack(test,pad) test = scipy.sparse.csr_matrix.dot(test, transformer.T) else: test, n_feature = ped(test) # n_feature = sub test = test.todense() test = scaler.transform(test) a = np.asarray(np.multiply(np.asarray(test), array)) for i in range(a.shape[0]): ffht.fht(a[i]) test = np.around(a, decimals=4) test_lable = (np.asarray(label)).reshape(-1) return train,train_lable,test,test_lable
def iterative(self, num_cols, num_iters, method, single_sketch=True, probs=None): """ Args: num_cols: int, number of sampled columns. num_iters: int, max. number of iterations. method: str, ssketching method to construct S. single_sketch: boolean, whether to use a single sketch throughtout the iterations or generate a new sketch at every iteration. probs: array(d), column-sampling probabilities. Not required for sketching-based methods (sampling == False). """ assert_ge(num_cols, self.n) n = self.n d = self.d c = self.c B = self.B L = np.zeros((num_iters, n, c)) G = np.zeros((num_iters, d, c)) Y = np.zeros((num_iters, n, c)) L[0] = self.P.todense() for i in xrange(num_iters): if i % 50 == 0: print "Iterative solver: iteration %d ..." % i if i > 0: L[i] = L[i - 1] - self.lmbd * Y[i - 1] - B.dot(G[i - 1]) if i == 0 or not single_sketch: # Generate new sketching matrix if method == 'sampling': assert probs is not None S = self.sampling_matrix(probs, num_cols) elif method == 'count-sketch': S = self.count_sketch(num_cols) elif method == 'SRHT': # Obtain the uniform-sampling and diagonal-sign matrices; # apply FHT later d1, D, S = self.subsampled_hadamard(num_cols) else: raise ValueError("Sampling method not recognized!\n") # Apply sketch if method == 'SRHT': B0 = np.c_[B, np.zeros((n, d1 - d))] # Pad by zeros BD = (D.dot(B0.T)).T # Multiply by (rescaled) signs # Apply fast Hadamard transform BDH = np.zeros((n, d1)) for k in xrange(n): # For each row a = deepcopy(BD[k, :]) fht(a) # Modifies in-place BDH[k, :] = deepcopy(a) assert not np.any(np.isnan(BDH)), BDH # Uniform sampling matrix SB = S.T.dot(BDH.T) else: SB = S.T.dot(B.T) assert_shape(SB, (num_cols, self.n)) # Direct inverse H = SB.T.dot(SB) + self.lmbd * np.identity(n) H_inv = inv(H) # # Implicit inverse via SVD # U, sigmas, _ = svd(SB.T, full_matrices=True) # Full SVD # Ds = diags(1./(sigmas**2. + self.lmbd)) # H_inv = U.dot(Ds.dot(U.T)) Y[i] = H_inv.dot(L[i]) G[i] = B.T.dot(Y[i]) G_opt = np.sum(G, axis=0) G_hist = np.cumsum(G, axis=0) return G_opt, G_hist
import numpy as np import ffht import timeit reps = 1000 n = 2**20 chunk_size = 1024 a = ffht.create_aligned(n, np.float32) np.copyto(a, np.random.randn(n)) t1 = timeit.default_timer() for i in range(reps): ffht.fht(a, chunk_size) t2 = timeit.default_timer() print(t2 - t1 + 0.0) / (reps + 0.0)
def structured_product(self, x, OMEGA, OMEGA_CONJ): y = np.copy(x) #print(self._d_pr, self._d) if self._d_pr <= self._d: for block in self._BBB[::-1]: # matrix-vector product if block._hash_func == 'random': y = np.dot(block._A, y) if block._hash_func == 'circulant': y = StructuredMatrices.circulant_product(block._col, y) if block._hash_func == 'skew_circulant': y = StructuredMatrices.skew_circulant_product( block._col, y, OMEGA, OMEGA_CONJ) if block._hash_func == 'toeplitz': y = StructuredMatrices.toeplitz_product( block._col, block._row, y) if block._hash_func == 'hadamard': y = StructuredMatrices.hadamard_cross_polytope_product( y, block._D, block._nblocks) if block._hash_func == 'hankel': y = StructuredMatrices.hankel_product( block._col, block._row, y) if block._hash_func == 'low_displacement_rank': y = StructuredMatrices.low_displacement_product( block._G, block._H, block._r, y, OMEGA, OMEGA_CONJ) if block._hash_func == 'kronecker': y = StructuredMatrices.kronecker_product_rec( block._A_list, y) if block._hash_func == 'pure_hadamard': #y = fht.fht1(y) a = ffht.create_aligned(y.shape[0], np.float32) np.copyto(a, y) ffht.fht(a, min(y.shape[0], 1024, 2048)) y = a if block._hash_func == 'diagonal' or block._hash_func == 'diagonal_kronecker' or block._hash_func == 'diagonal_gaussian': y = block._D * y #print(block, y.shape) # all blocks have been handled. # dimensionality reduction y = y[:self._d_pr] if self._normalization: y *= self._norm_factor_list return y else: if len(self._BBB) == 1 and self._BBB[0]._hash_func == 'random': y = np.dot(self._BBB[0]._A, y) return y y_global = np.zeros(self._d_pr) div = self._d_pr // self._d # number of parameters div_real = self._d_pr / self._d #print(div, div_real, self._d_pr, self._d) assert div == div_real # we handle only this case # /!\ + d_pr should be an integral power of 2!!! for i_div in xrange(div): #print('i_div', i_div) y = np.copy(x) # if div_real != div, the last mini block should be truncated => never happened because of the case of hadamard # split y into small vectors and reunit all results after for block in self._BBB[::-1]: #if i_div == div or i_div < div -1: # easy # matrix-vector product if block._hash_func == 'circulant': y = StructuredMatrices.circulant_product( block._col[i_div], y) if block._hash_func == 'skew_circulant': y = StructuredMatrices.skew_circulant_product( block._col[i_div], y, OMEGA, OMEGA_CONJ) if block._hash_func == 'toeplitz': y = StructuredMatrices.toeplitz_product( block._col[i_div], block._row[i_div], y) if block._hash_func == 'hadamard': y = StructuredMatrices.hadamard_cross_polytope_product( y, block._D[i_div], block._nblocks) if block._hash_func == 'hankel': y = StructuredMatrices.hankel_product( block._col[i_div], block._row[i_div], y) if block._hash_func == 'low_displacement_rank': y = StructuredMatrices.low_displacement_product( block._G[i_div], block._H[i_div], block._r, y, OMEGA, OMEGA_CONJ) if block._hash_func == 'kronecker': y = StructuredMatrices.kronecker_product_rec( block._A_list[i_div], y) if block._hash_func == 'pure_hadamard': a = ffht.create_aligned(y.shape[0], np.float32) np.copyto(a, y) ffht.fht(a, min(x.shape[0], 1024, 2048)) y = a if block._hash_func == 'diagonal' or block._hash_func == 'diagonal_kronecker' or block._hash_func == 'diagonal_gaussian': y = block._D[i_div] * y y_global[i_div * self._d:i_div * self._d + self._d] = y #print(y_global) if self._normalization: y_global *= self._norm_factor_list return y_global
import numpy as np import ffht import timeit reps = 1000 n = 2**20 chunk_size = 1024 a = ffht.create_aligned(n, np.float32) np.copyto(a, np.random.randn(n)) t1 = timeit.default_timer() for i in range(reps): ffht.fht(a, chunk_size) t2 = timeit.default_timer() print (t2 - t1 + 0.0) / (reps + 0.0)
def FastIHT_WHT(y, K, Q, d, Sigma, top_k_func=1): """ Fast iterative hard thresholding algorithm with partial Walsh-Hadamard Transform sensing matrices. y : numpy.ndarray the measurement vector K : int number of nonzero entries in the recovered signal Q : int dimension of y d : int dimension of the recovered signal, must be a power of 2 Sigma : numpy.ndarray a Q-dimensional array consisting of row indices of the partial WHT matrix top_k_fun : {0, 1, 2} indicates the function used for computing the top K indices of a vector 0 - numpy.argpartition 1 - tensorflow.math.top_k 2 - torch.topk Packages required: numpy, pandas, ffht, tensorflow, torch """ if (d & (d-1) != 0) or (d <= 0): print("The dimension d must be a power of 2.") return if not(top_k_func in (0, 1, 2)): top_k_func = 0 eps = 1e-4 max_iter = 25 res_norms = np.zeros(max_iter) # recording norms of y - Φ * w_vec wht_factor = 1 / np.sqrt(Q) # scaling factor for WHT g = np.zeros(d) # g: the current iterate g_prev = g # g_prev: the previous iterate res = np.zeros(d) # res <- transpose(Φ) * y res[Sigma] = y * wht_factor ffht.fht(res) Omega_prev = np.array([], dtype=np.int32) # Omega_prev: support of g_prev # Omega <- top K indices of transpose(Φ) * y Omega = TOP_K_ABS_FUNC[top_k_func](res, K) # Omega: support of g # initialize g as the top K projection of transpose(Φ) * y g[Omega] = res[Omega] for s in range(max_iter): if s == 0: w_vec = g Omega_w = Omega # Omega_w: support of w_vec else: g_wht = np.copy(g) ffht.fht(g_wht) # g_diff <- g - g_prev g_diff = np.copy(g) g_diff[Omega_prev] -= g_prev[Omega_prev] g_diff_wht = np.copy(g_diff) ffht.fht(g_diff_wht) g_diff_wht_partial = g_diff_wht[Sigma] * wht_factor # tau <- <y-Φg, Φ(g-g_prev)> / ||Φ(g-g_prev)||^2 tau = np.dot(y - g_wht[Sigma] * wht_factor, g_diff_wht_partial) / np.dot(g_diff_wht_partial, g_diff_wht_partial) # w_vec <- g + tau * (g-g_prev) # and update the support of w (i.e., Omega_w) Omega_w = pandas.unique(np.concatenate((Omega, Omega_prev))) w_vec = np.copy(g) w_vec[Omega_w] += tau * g_diff[Omega_w] w_vec_wht = np.copy(w_vec) ffht.fht(w_vec_wht) res_w = np.zeros(d) res_w[Sigma] = (y - w_vec_wht[Sigma] * wht_factor) * wht_factor # stopping criterion: # if the norm of y - Φ * w_vec remains nearly the same in the last 4 iterations # or if the current norm of y - Φ * w_vec is small enough res_norms[s] = np.linalg.norm(res_w[Sigma]) / wht_factor if s >= 3 and np.std(res_norms[s-3:s+1]) / np.mean(res_norms[s-3:s+1]) < 1e-2: break elif res_norms[s] < eps: break # res_w <- transpose(Φ) * (y - Φ * w_vec) ffht.fht(res_w) res_w_proj_wht = np.zeros(d) res_w_proj_wht[Omega_w] = res_w[Omega_w] * wht_factor ffht.fht(res_w_proj_wht) # alpha_tilde <- ||Proj(res_w, Omega_w)||^2 / ||Φ * Proj(res_w, Omega_w)||^2 alpha_tilde = np.dot(res_w[Omega_w], res_w[Omega_w]) / np.dot(res_w_proj_wht[Sigma], res_w_proj_wht[Sigma]) g_prev = g Omega_prev = Omega # h_vec <- w_vec + alpha_tilde * res_w h_vec = alpha_tilde * res_w h_vec[Omega_w] += w_vec[Omega_w] # set g as the top K projection of h_vec # and update the support of g (i.e., Omega) Omega = TOP_K_ABS_FUNC[top_k_func](h_vec, K) g = np.zeros(d) g[Omega] = h_vec[Omega] g_wht = np.copy(g) ffht.fht(g_wht) # res <- transpose(Φ) * (y - Φ * g) res = np.zeros(d) res[Sigma] = (y - g_wht[Sigma] * wht_factor) * wht_factor ffht.fht(res) # res_proj_wht <- Φ * Proj(res, Omega) res_proj_wht = np.zeros(d) res_proj_wht[Omega] = res[Omega] * wht_factor ffht.fht(res_proj_wht) # alpha <- ||Proj(res, Omega)||^2 / ||Φ * Proj(res, Omega)||^2 alpha = np.dot(res[Omega], res[Omega]) / np.dot(res_proj_wht[Sigma], res_proj_wht[Sigma]) # g <- g + alpha * Proj(res, Omega) g[Omega] += alpha * res[Omega] return g
def jl_chebyshev(X, lamb): #print(X[:,0].mean(0)) #assert X[:,0].mean(0) < 1e-4 X = X - X.mean(0, keepdim=True) n_data, feat_dim = X.size() X_scaled = X / n_data #if lamb=0 no scaling, so not to magnify bessel[i, 0] in the approximation. scale = int(dominant_eval_cov(np.sqrt(lamb) * X)[0]) if lamb > 0 else 1 if scale > 1: print('Scaling M! {}'.format(scale)) #scale down matrix if matrix norm >= 3, since later scale up when #odd power if scale % 2 == 0: scale -= 1 X_scaled /= scale else: scale = 1 subsample_freq = int(feat_dim / math.log(feat_dim, 2)) #100 k = math.ceil(feat_dim / subsample_freq) X_t = X.t() #fast Hadamard transform (ffht) vs transform by multiplication by Hadamard mx. ffht_b = False P, H, D = get_jl_mx(feat_dim, k, ffht_b) I_proj = torch.eye(feat_dim, feat_dim, device=X.device) M = D I_proj = torch.mm(D, I_proj) if ffht_b: #can be obtained from https://github.com/FALCONN-LIB/FFHT #place here so not everyone needs to install. import ffht M = M.t() #M1 = np.zeros((M.size(0), M.size(1)), dtype=np.double) M_np = M.cpu().numpy() I_np = I_proj.cpu().numpy() for i in range(M.size(0)): ffht.fht(M_np[i]) for i in range(I_proj.size(0)): ffht.fht(I_np[i]) #pdb.set_trace() M = torch.from_numpy(M_np).to(dtype=M.dtype, device=X.device).t() I_proj = torch.from_numpy(I_np).to(dtype=M.dtype, device=X.device) else: #right now form the matrix exponential here M = torch.mm(H, M) I_proj = torch.mm(H, I_proj) #apply P now so downstream multiplications are faster: kd instead of d^2. #subsample to get reduced dimension subsample = True if subsample: #random sampling performs well in practice and has lower complexity #select_idx = torch.randint(low=0, high=feat_dim, size=(feat_dim//5,)) <--this produces repeats if device == 'cuda': #pdb.set_trace() select_idx = torch.cuda.LongTensor( list(range(0, feat_dim, subsample_freq))) else: select_idx = torch.LongTensor( list(range(0, feat_dim, subsample_freq))) #M = torch.index_select(M, dim=0, index=select_idx) M = M[select_idx] #I_proj = torch.index_select(I_proj, dim=0, index=select_idx) I_proj = I_proj[select_idx] else: M = torch.sparse.mm(P, M) I_proj = torch.sparse.mm(P, I_proj) #M is now the projection mx A = M for _ in range(scale): #(k x d) A = sketch_and_apply(lamb, X, X_scaled, A, I_proj) #Compute tau1 scores #M = M / M.diag().sum() #M is (k x d) #compute tau1 scores (this M is previous M^{1/2}) tau1 = (torch.mm(A, X_t)**2).sum(0) return tau1
grad += e # Compute sparsity of grad sparsity_grad = (np.linalg.norm(grad, ord=1)**2) / ( np.linalg.norm(grad, ord=2)**2 * grad.shape[0]) sp_p(sparsity_grad) # Recover K-sparse signal g_rec = FIHT.FastIHT_WHT(z, k, Q, d_aug, Phi_row_idx, top_k_func=1)[0:d] e = grad - g_rec g_rec_wht = np.concatenate( (g_rec, np.zeros(d_aug - d))) / np.sqrt(Q) ffht.fht(g_rec_wht) z_rec = g_rec_wht[Phi_row_idx] r = z - z_rec # Unflatten g_rec_wht shapes = [ model.trainable_weights[i].shape for i in range(len(model.trainable_weights)) ] grad_tx = [] n_prev = 0 for i in range(len(shapes)): n = n_prev + tf.math.reduce_prod(shapes[i]) grad_tx.append(
def f(i): for j in range(i*1000,(i+1)*1000): ffht.fht(b[j]) return b[i*1000:(i+1)*1000],i
import numpy as np import ffht import timeit import sys reps = 1000 n = 2**20 chunk_size = 1024 a = np.random.randn(n).astype(np.float32) t1 = timeit.default_timer() for i in range(reps): ffht.fht(a) t2 = timeit.default_timer() if sys.version_info[0] == 2: print(t2 - t1 + 0.0) / (reps + 0.0) if sys.version_info[0] == 3: print('{}'.format((t2 - t1 + 0.0) / (reps + 0.0)))