def hadamard2(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1):
p = FLAGS.p
x_ = batch
n = x_.shape[0]
x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])),
'constant',
constant_values=(0, 0))
x_ = np.tile(x_, (1, p))
x_i = np.multiply(x_, B)
x_ = x_i.reshape(n * p, d)
for i in range(n * p):
ffht.fht(x_[i])
x_ = x_.reshape(n, p * d)
np.take(x_, PI_value, axis=1, mode='wrap', out=x_)
x_transformed = np.multiply(x_, G)
x_ = np.reshape(x_transformed, (n * p, d))
for i in range(n * p):
ffht.fht(x_[i])
x_ = x_.reshape(n, p * d)
x_value = np.multiply(x_, S)
x_value = x_value / (sigma * (np.sqrt(d)**3))
x_value = np.cos(x_value + FLAGS.b) + FLAGS.t
x_value = np.sign(x_value)
return x_value
def fastfood(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1):
start = time.time()
T = FLAGS.T
x_ = batch
n = x_.shape[0]
# x.shape [batch,d]
x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])),
'constant',
constant_values=(0, 0))
x_ = np.tile(x_, (1, T))
x_i = np.multiply(x_, S)
x_ = x_i.reshape(FLAGS.BATCHSIZE * T, d)
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
x_transformed = np.multiply(x_, G)
np.take(x_, PI_value, axis=1, mode='wrap', out=x_)
x_ = np.reshape(x_transformed, (FLAGS.BATCHSIZE * T, d))
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
# print(np.linalg.norm(x_[:2],axis=1))
# print(np.mean(x_[:,:2], axis=0))
x_value = np.multiply(x_, B)
# print(np.linalg.norm(x_value[:2], axis=1))
# print(np.any(FLAGS.b))
x_value = (sigma**2) * x_value / np.sqrt(d)
x_value = np.cos(x_value)
# print(time.time()-start)
return x_value
def transform(name):
np.set_printoptions(threshold=np.inf, suppress=True)
train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/train" % (name)
# data = load_svmlight_file(train_url)
train, tra_label = load_svmlight_file(train_url)
train = train.todense()
tra_label = np.mat(tra_label).T
b_num,n_feature = np.shape(train)
if n_feature < 4000:
train, n_feature = ped(train)
print(train.shape)
start = time.process_time()
else:
start = time.process_time()
sub = int(math.pow(2,12))
transformer = cwt_matrix(sub,n_feature)
train = scipy.sparse.csr_matrix.dot(train, transformer.T)
n_feature = sub
rng = check_random_state(None)
array = np.mat(rng.choice([1, -1], n_feature))
n_sample, d_feature = np.shape(train)
a = np.array(np.multiply(train, array))
for i in range(n_sample):
ffht.fht(a[i])
transform_time = time.process_time() - start
return transform_time,a,array
def Fastfood_updata(self):
p = FLAGS.p
'''
using the sigmoid
'''
self.gradient = np.dot((self.prediction-self.label),self.wb.T/self.scaling)/self.original.shape[0]
'''
using the softmax
'''
self.gradient = np.multiply(self.gradient,-np.sin(self.x_transfer + FLAGS.b))
self.gradient = np.array(self.gradient)
S = copy.deepcopy(self.S)
self.S -= self.update_S()
self.gradient= np.multiply(S,self.gradient) # gradient update after S
self.gradient = self.gradient.reshape(-1,self.dimension)
for i in range(self.gradient.shape[0]):
ffht.fht(self.gradient[i])
# self.gradient = self.gradient/(2**(self.dimension/2))# gradient update after H
self.gradient = self.gradient.reshape(-1, p*self.dimension)/(np.sqrt(self.dimension) )
G = copy.deepcopy(self.G)
self.G -= self.update_G()
def HD_x(D_by_sqrt_d, pad, x):
# (d^{-1/2}) * [ Dx 0 ... 0 ]
HDx = np.concatenate([D_by_sqrt_d * x, pad])
# (d^{-1/2}) * H [ Dx 0 ... 0 ]
fht(HDx)
return HDx
def HGPHD_x(D_by_sqrt_d, pad, P_seed, G, x):
# (1/d) * HDx
HDx = np.concatenate([x * D_by_sqrt_d, pad])
fht(HDx)
# (1/d) * GPHDx
HGPHDx = G * rperm(P_seed, HDx)
# (1/d) * HGPHDx
fht(HGPHDx)
return HGPHDx
def Forwardpropogation(self):
p = FLAGS.p
x_ = self.batch
n = x_.shape[0]
x_ = np.pad(x_, ((0, 0), (0, self.dimension - x_.shape[1])),
'constant',
constant_values=(0, 0))
x_ = np.tile(x_, (1, p))
self.original = x_
self.x_B = np.multiply(x_, self.B)
x_ = self.x_B.reshape(n * p, self.dimension)
for i in range(n * p):
ffht.fht(x_[i])
x_ = x_.reshape(n, p * self.dimension)
self.x_G = np.take(x_, self.PI_value, axis=1, mode='wrap')
x_transformed = np.multiply(self.x_G, self.G)
x_ = np.reshape(x_transformed, (n * p, self.dimension))
for i in range(n * p):
ffht.fht(x_[i])
self.x_S = x_.reshape(n, p * self.dimension)
x_ = np.multiply(self.x_S, self.S)
# self.x_transfer = np.sign(x_)
self.x_transfer = copy.deepcopy(x_)
# print(np.max(self.x_transfer),np.mean(self.x_transfer),np.min(self.x_transfer))
# exit()
x_transfer = np.cos(x_ + FLAGS.b) + FLAGS.t
x_ = np.sign(x_transfer)
self.wb = np.sign(self.coefficients)
'''
using the sigmoid function we will use the sigmoid activation for the multiclass problem
'''
prediction = x_.dot(np.sign(self.coefficients))
self.ywx = np.multiply(self.label, prediction)
self.prediction = np.where(prediction > 0, 1, -1)
self.loss = np.sum(np.clip(1 - self.ywx, 0, None)) / x_.shape[0]
# self.loss = sklearn.metrics.log_loss(self.label, prediction)
self.ywx = np.where(self.ywx < 1, 1, 0)
self.gradient = -np.multiply(np.mean(self.ywx, axis=0),
np.dot(x_.T, self.label))
# print(np.count_nonzero(self.gradient))
# print(self.gradient[0,:10])
# print(self.coefficients[0,:10])
'''
def hadamard(d, f_num, batch, G, B, PI_value, S):
T = FLAGS.T
x_ = batch
x_ = np.pad(x_, ((0, 0), (0, d - f_num)),
'constant',
constant_values=(0, 0)) # x.shape [batch,d]
x_ = np.tile(x_, (1, T))
x_i = np.multiply(x_, S)
x_ = x_i.reshape(FLAGS.BATCHSIZE, T, d)
h = 1
# while h < d:
# for i in range(0, d, h * 2):
# for j in range(i, i + h):
# a = x_[:, :, j]
# b = x_[:, :, j + h]
# temp = a - b
# x_[:, :, j] = a + b
# x_[:, :, j + h] = temp
# h *= 2
for i in range(x_.shape[0]):
for j in range(T):
ffht.fht(x_[i, j])
x_transformed = np.multiply(x_.reshape(FLAGS.BATCHSIZE, d * T), G)
x_transformed = np.reshape(x_transformed, (FLAGS.BATCHSIZE, T, d))
x_permutation = x_transformed[:, :, PI_value]
x_ = x_permutation
h = 1
while h < d:
for i in range(0, d, h * 2):
for j in range(i, i + h):
a = x_[:, :, j]
b = x_[:, :, j + h]
temp = a - b
x_[:, :, j] = a + b
x_[:, :, j + h] = temp
h *= 2
# for i in range(x_.shape[0]):
# for j in range(T):
# ffht.fht(x_[i,j])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
x_value = np.multiply(x_, B)
x_value = np.sign(x_value)
return x_value
def Hx(dim, reps) :
t = 0
for i in range(reps) :
x = np.random.normal(size=dim)
start = timeit.default_timer()
d = fht(x)
stop = timeit.default_timer()
t += (stop - start)
return t
def main():
d = 668426
K = 1000
Q = K * 12
d_aug = 2**np.ceil(np.log2(d)).astype('int32')
rng = np.random.default_rng(2)
Phi_row_idx = rng.choice(d_aug, Q, replace=False)
time_total = 0
for ii in range(100):
rng = np.random.default_rng()
supp = rng.choice(d, K, replace=False)
g0 = np.zeros(d)
g0[supp] = np.random.randn(K)
g0 += np.random.randn(d) * 0.01
supp = np.argpartition(np.abs(g0), -K)[-K:]
g0_sp = np.zeros(d)
g0_sp[supp] = g0[supp]
g0_wht = np.concatenate((g0, np.zeros(d_aug - d))) / np.sqrt(Q)
ffht.fht(g0_wht)
y = g0_wht[Phi_row_idx]
t1 = timeit.default_timer()
g_rec = FIHT.FastIHT_WHT(y, K, Q, d_aug, Phi_row_idx,
top_k_func=1)[0:d]
t2 = timeit.default_timer()
time_total += t2 - t1
rel_diff_rec = np.linalg.norm(g_rec - g0) / np.linalg.norm(g0)
rel_diff_bestK = np.linalg.norm(g0_sp - g0) / np.linalg.norm(g0)
print("||g_rec - g|| / ||g|| = ", rel_diff_rec)
if rel_diff_bestK != 0:
print("||g_bestK - g|| / ||g|| = ", rel_diff_bestK)
print("amplification factor = ", rel_diff_rec / rel_diff_bestK)
print(time_total / 100)
def Forwardpropogation(self):
p = FLAGS.p
x_ = self.batch
n = x_.shape[0]
x_ = np.pad(x_, ((0, 0), (0, self.dimension - x_.shape[1])), 'constant', constant_values=(0, 0))
x_ = np.tile(x_, (1, p))
self.original = x_
self.x_B = np.multiply(x_, self.B)
x_ = self.x_B.reshape(n * p, self.dimension)
for i in range(n * p):
ffht.fht(x_[i])
x_ = x_.reshape(n, p * self.dimension)/(np.sqrt(self.dimension) )
self.x_G = np.take(x_, self.PI_value, axis=1, mode='wrap')
x_transformed = np.multiply(self.x_G, self.G)
x_ = np.reshape(x_transformed, (n * p, self.dimension))
for i in range(n * p):
ffht.fht(x_[i])
self.x_S = x_.reshape(n, p * self.dimension)/(np.sqrt(self.dimension) )
x_= np.multiply(self.x_S, self.S)
# self.x_transfer = np.sign(x_)
self.x_transfer = copy.deepcopy(x_)
x_transfer= np.cos(x_+ FLAGS.b) + FLAGS.t
'''
control whether the feature and coefficients are binary
'''
x_= np.sign(x_transfer)
# x_ = x_transfer
self.wb = np.sign(self.coefficients)
# self.wb = copy.deepcopy(self.coefficients)
'''
using the sigmoid function we will use the sigmoid activation for the multiclass problem
'''
prediction = 1/(1+np.exp(-x_.dot(self.wb)/self.scaling) )
self.prediction = prediction
self.loss = sklearn.metrics.log_loss(self.label, prediction)
self.gradient = np.dot(x_.T/self.scaling, prediction - self.label) / x_.shape[0]
'''
def hadamard(d, f_num, batch, G, B, PI_value, S, FLAGS, sigma=1):
start = time.time()
T = FLAGS.T
kf = KFold(n_splits=2)
x_return = np.zeros((1, T * d))
for train_index, test_index in kf.split(batch):
x_ = batch[test_index]
# print(x_.shape)
n = x_.shape[0]
# x.shape [batch,d]
x_ = np.pad(x_, ((0, 0), (0, d - x_.shape[1])),
'constant',
constant_values=(0, 0))
x_ = np.tile(x_, (1, T))
x_i = np.multiply(x_, S)
x_ = x_i.reshape(n * T, d)
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(n, T * d)
x_transformed = np.multiply(x_, G)
np.take(x_, PI_value, axis=1, mode='wrap', out=x_)
x_ = np.reshape(x_transformed, (n * T, d))
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(n, T * d)
# print(np.linalg.norm(x_[:2],axis=1))
# print(np.mean(x_[:,:2], axis=0))
x_value = np.multiply(x_, B)
# print(np.linalg.norm(x_value[:2], axis=1))
# print(np.any(FLAGS.b))
if np.any(FLAGS.b):
# print(1)
x_value = x_value / (sigma * np.sqrt(d)**3)
x_value = np.cos(x_value + FLAGS.b) + FLAGS.t
# print(time.time()-start)
x_value = np.sign(x_value)
x_return = np.vstack((x_return, x_value))
return x_return[1:, :]
def hadamard(d, f_num, batch, G, B, PI_value, S):
T = FLAGS.T
x_ = batch
x_ = np.pad(x_, ((0, 0), (0, d - f_num)),
'constant',
constant_values=(0, 0)) # x.shape [batch,d], pad to d = 2^p
x_ = np.tile(x_, (1, T)) #
x_i = np.multiply(x_, S)
x_ = x_i.reshape(FLAGS.BATCHSIZE, T, d)
for i in range(x_.shape[0]):
for j in range(T):
ffht.fht(x_[i, j])
x_transformed = np.multiply(x_.reshape(FLAGS.BATCHSIZE, d * T), G)
x_transformed = np.reshape(x_transformed, (FLAGS.BATCHSIZE, T, d))
x_ = x_transformed
for i in range(x_.shape[0]):
for j in range(T):
ffht.fht(x_[i, j])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
x_value = np.multiply(x_, B)
x_value = np.sign(x_value)
return x_value
def hadamard(d, f_num, batch, G, B, PI_value, S, T):
x_ = batch
n = x_.shape[0]
x_ = np.pad(x_, ((0, 0), (0, d - f_num)),
'constant',
constant_values=(0, 0)) # x.shape [batch,d]
x_ = np.tile(x_, (1, T))
x_i = np.multiply(x_, S)
x_ = x_i.reshape(FLAGS.BATCHSIZE * T, d)
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
np.take(x_, PI_value, axis=1, mode='wrap', out=x_)
x_transformed = np.multiply(x_, G)
x_ = np.reshape(x_transformed, (FLAGS.BATCHSIZE * T, d))
for i in range(n * T):
ffht.fht(x_[i])
x_ = x_.reshape(FLAGS.BATCHSIZE, T * d)
x_value = np.multiply(x_, B)
x_value = np.sign(x_value)
# exit()
return x_value
def test_transform(name,array):
np.set_printoptions(threshold=np.inf, suppress=True)
train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/test" % (name)
# data = load_svmlight_file(train_url)
train, tra_label = load_svmlight_file(train_url)
train = train.todense()
tra_label = np.mat(tra_label).T
b_num,n_feature = np.shape(train)
if n_feature < 10000:
train, n_feature = ped(train)
start = time.process_time()
else:
start = time.process_time()
sub = int(math.pow(2,12))
transformer = cwt_matrix(sub,n_feature)
train = scipy.dot(train[:,:n_feature], transformer.T)
n_feature = sub
a = np.array(np.multiply(train, array))
for i in range(b_num):
ffht.fht(a[i])
transform_time = time.process_time() - start
return transform_time,a
def Fastfood_updata(self):
p = FLAGS.p
# print(self.label.shape)
self.gradient = -np.multiply(
np.mean(self.ywx, axis=1).reshape(-1, 1),
np.dot((self.label) / self.original.shape[0], self.wb.T))
# print(self.gradient.shape)
self.gradient = np.multiply(self.gradient,
-np.sin(self.x_transfer + FLAGS.b))
self.gradient = np.array(self.gradient)
S = copy.deepcopy(self.S)
self.S -= self.update_S()
self.gradient = np.multiply(S,
self.gradient) # gradient update after S
self.gradient = self.gradient.reshape(-1, self.dimension)
for i in range(self.gradient.shape[0]):
ffht.fht(self.gradient[i])
# self.gradient = self.gradient/(2**(self.dimension/2))# gradient update after H
self.gradient = self.gradient.reshape(-1, p * self.dimension)
G = copy.deepcopy(self.G)
self.G -= self.update_G()
def wht(msgin):
for i in range(msgin.shape[0]):
ffht.fht(msgin[i, :])
return msgin
psi_FFHT = np.zeros((2 * basis.Ns, ), dtype=np.float64)
psi_FFHT[:basis.Ns] = psi_0.real.copy()
psi_FFHT[basis.Ns:] = psi_0.imag.copy()
psi_FFHT_cpx = np.zeros((basis.Ns, ), dtype=np.complex128)
norm = basis.Ns #np.sqrt(2)**L
a = np.zeros_like(psi_0)
t1 = timeit.default_timer()
for j in range(nT):
# evolve state using FFHT
# apply FHT
ffht.fht(psi_FFHT[:basis.Ns])
ffht.fht(psi_FFHT[basis.Ns:])
#psi_FFHT/=norm
# apply Ux
''' (a+ib)(c+id)=ac-bd + i(ad+bc)
z=cosHx_diagonal*psi_FFHT[:basis.Ns] - sinHx_diagonal*psi_FFHT[basis.Ns:]
psi_FFHT[basis.Ns:] = sinHx_diagonal*psi_FFHT[:basis.Ns] + cosHx_diagonal*psi_FFHT[basis.Ns:]
psi_FFHT[:basis.Ns]=z[:]
'''
a = sinHx_diagonal * psi_FFHT[basis.Ns:]
psi_FFHT[basis.Ns:] *= cosHx_diagonal
psi_FFHT[basis.Ns:] += sinHx_diagonal * psi_FFHT[:basis.Ns]
psi_FFHT[:basis.Ns] *= cosHx_diagonal
psi_FFHT[:basis.Ns] -= a
def generater(name, what = 'train_0', what2 = 'test_0'):
np.set_printoptions(threshold=np.inf, suppress=True)
train_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/train" % (name)
# data = load_svmlight_file(train_url)
train, tra_label = load_svmlight_file(train_url)
tra_label = np.mat(tra_label).T
d_data,n_feature = np.shape(train)
train_shape = n_feature
# sub = int(math.pow(2,f_sub))
if n_feature>10000:
sub = int(math.pow(2,13))
transformer = cwt_matrix(sub,n_feature)
train = scipy.sparse.csr_matrix.dot(train, transformer.T)
n_feature = sub
else:
train, n_feature = ped(train)
array = np.mat(np.random.uniform(-1, 1, n_feature)) # -1 +1均匀采样
array[array < 0] = -1
array[array > 0] = 1
label = np.where(tra_label[:, 0] != 1, -1, 1)
n_sample, d_feature = np.shape(train)
train = train.todense()
# scaler = preprocessing.StandardScaler(with_std=False).fit(train)
scaler = preprocessing.StandardScaler(with_std=False).fit(train)
train = scaler.transform(train)
# print(0)
a = np.asarray(np.multiply(np.asarray(train), array))
for i in range(a.shape[0]):
# print(a[i].shape)
ffht.fht(a[i])
train = np.around(a,decimals=4)
train_lable = (np.asarray(label)).reshape(-1)
# dump_svmlight_file(np.around(a, decimals=4),(np.asarray(label)).reshape(-1),'svm/BudgetedSVM/original/%s/%s'%(name,what),zero_based=False)
del a
gc.collect()
test_url = "/home/comp/cszjlei/svm/BudgetedSVM/original/%s/test" % (name)
test, te_label = load_svmlight_file(test_url)
# test, n_feature = ped(test)
te_label = np.mat(te_label).T
label = np.where(te_label[:, 0] != 1, -1, 1)
n_number, n_feature = np.shape(test)
if n_feature > 10000:
if n_feature>train_shape:
# print(train_shape,n_feature)
test = scipy.sparse.csr_matrix.dot(test[:,:train_shape], transformer.T)
elif n_feature == train_shape:
test = scipy.sparse.csr_matrix.dot(test, transformer.T)
else:
pad = np.zeros((n_number,train_shape-n_feature))
test = scipy.sparse.hstack(test,pad)
test = scipy.sparse.csr_matrix.dot(test, transformer.T)
else:
test, n_feature = ped(test)
# n_feature = sub
test = test.todense()
test = scaler.transform(test)
a = np.asarray(np.multiply(np.asarray(test), array))
for i in range(a.shape[0]):
ffht.fht(a[i])
test = np.around(a, decimals=4)
test_lable = (np.asarray(label)).reshape(-1)
return train,train_lable,test,test_lable
def iterative(self,
num_cols,
num_iters,
method,
single_sketch=True,
probs=None):
"""
Args:
num_cols: int, number of sampled columns.
num_iters: int, max. number of iterations.
method: str, ssketching method to construct S.
single_sketch: boolean, whether to use a single sketch throughtout
the iterations or generate a new sketch at every iteration.
probs: array(d), column-sampling probabilities.
Not required for sketching-based methods (sampling == False).
"""
assert_ge(num_cols, self.n)
n = self.n
d = self.d
c = self.c
B = self.B
L = np.zeros((num_iters, n, c))
G = np.zeros((num_iters, d, c))
Y = np.zeros((num_iters, n, c))
L[0] = self.P.todense()
for i in xrange(num_iters):
if i % 50 == 0:
print "Iterative solver: iteration %d ..." % i
if i > 0:
L[i] = L[i - 1] - self.lmbd * Y[i - 1] - B.dot(G[i - 1])
if i == 0 or not single_sketch:
# Generate new sketching matrix
if method == 'sampling':
assert probs is not None
S = self.sampling_matrix(probs, num_cols)
elif method == 'count-sketch':
S = self.count_sketch(num_cols)
elif method == 'SRHT':
# Obtain the uniform-sampling and diagonal-sign matrices;
# apply FHT later
d1, D, S = self.subsampled_hadamard(num_cols)
else:
raise ValueError("Sampling method not recognized!\n")
# Apply sketch
if method == 'SRHT':
B0 = np.c_[B, np.zeros((n, d1 - d))] # Pad by zeros
BD = (D.dot(B0.T)).T # Multiply by (rescaled) signs
# Apply fast Hadamard transform
BDH = np.zeros((n, d1))
for k in xrange(n): # For each row
a = deepcopy(BD[k, :])
fht(a) # Modifies in-place
BDH[k, :] = deepcopy(a)
assert not np.any(np.isnan(BDH)), BDH
# Uniform sampling matrix
SB = S.T.dot(BDH.T)
else:
SB = S.T.dot(B.T)
assert_shape(SB, (num_cols, self.n))
# Direct inverse
H = SB.T.dot(SB) + self.lmbd * np.identity(n)
H_inv = inv(H)
# # Implicit inverse via SVD
# U, sigmas, _ = svd(SB.T, full_matrices=True) # Full SVD
# Ds = diags(1./(sigmas**2. + self.lmbd))
# H_inv = U.dot(Ds.dot(U.T))
Y[i] = H_inv.dot(L[i])
G[i] = B.T.dot(Y[i])
G_opt = np.sum(G, axis=0)
G_hist = np.cumsum(G, axis=0)
return G_opt, G_hist
import numpy as np
import ffht
import timeit
reps = 1000
n = 2**20
chunk_size = 1024
a = ffht.create_aligned(n, np.float32)
np.copyto(a, np.random.randn(n))
t1 = timeit.default_timer()
for i in range(reps):
ffht.fht(a, chunk_size)
t2 = timeit.default_timer()
print(t2 - t1 + 0.0) / (reps + 0.0)
def structured_product(self, x, OMEGA, OMEGA_CONJ):
y = np.copy(x)
#print(self._d_pr, self._d)
if self._d_pr <= self._d:
for block in self._BBB[::-1]:
# matrix-vector product
if block._hash_func == 'random':
y = np.dot(block._A, y)
if block._hash_func == 'circulant':
y = StructuredMatrices.circulant_product(block._col, y)
if block._hash_func == 'skew_circulant':
y = StructuredMatrices.skew_circulant_product(
block._col, y, OMEGA, OMEGA_CONJ)
if block._hash_func == 'toeplitz':
y = StructuredMatrices.toeplitz_product(
block._col, block._row, y)
if block._hash_func == 'hadamard':
y = StructuredMatrices.hadamard_cross_polytope_product(
y, block._D, block._nblocks)
if block._hash_func == 'hankel':
y = StructuredMatrices.hankel_product(
block._col, block._row, y)
if block._hash_func == 'low_displacement_rank':
y = StructuredMatrices.low_displacement_product(
block._G, block._H, block._r, y, OMEGA, OMEGA_CONJ)
if block._hash_func == 'kronecker':
y = StructuredMatrices.kronecker_product_rec(
block._A_list, y)
if block._hash_func == 'pure_hadamard':
#y = fht.fht1(y)
a = ffht.create_aligned(y.shape[0], np.float32)
np.copyto(a, y)
ffht.fht(a, min(y.shape[0], 1024, 2048))
y = a
if block._hash_func == 'diagonal' or block._hash_func == 'diagonal_kronecker' or block._hash_func == 'diagonal_gaussian':
y = block._D * y
#print(block, y.shape)
# all blocks have been handled.
# dimensionality reduction
y = y[:self._d_pr]
if self._normalization:
y *= self._norm_factor_list
return y
else:
if len(self._BBB) == 1 and self._BBB[0]._hash_func == 'random':
y = np.dot(self._BBB[0]._A, y)
return y
y_global = np.zeros(self._d_pr)
div = self._d_pr // self._d # number of parameters
div_real = self._d_pr / self._d
#print(div, div_real, self._d_pr, self._d)
assert div == div_real # we handle only this case
# /!\ + d_pr should be an integral power of 2!!!
for i_div in xrange(div):
#print('i_div', i_div)
y = np.copy(x)
# if div_real != div, the last mini block should be truncated => never happened because of the case of hadamard
# split y into small vectors and reunit all results after
for block in self._BBB[::-1]:
#if i_div == div or i_div < div -1:
# easy
# matrix-vector product
if block._hash_func == 'circulant':
y = StructuredMatrices.circulant_product(
block._col[i_div], y)
if block._hash_func == 'skew_circulant':
y = StructuredMatrices.skew_circulant_product(
block._col[i_div], y, OMEGA, OMEGA_CONJ)
if block._hash_func == 'toeplitz':
y = StructuredMatrices.toeplitz_product(
block._col[i_div], block._row[i_div], y)
if block._hash_func == 'hadamard':
y = StructuredMatrices.hadamard_cross_polytope_product(
y, block._D[i_div], block._nblocks)
if block._hash_func == 'hankel':
y = StructuredMatrices.hankel_product(
block._col[i_div], block._row[i_div], y)
if block._hash_func == 'low_displacement_rank':
y = StructuredMatrices.low_displacement_product(
block._G[i_div], block._H[i_div], block._r, y,
OMEGA, OMEGA_CONJ)
if block._hash_func == 'kronecker':
y = StructuredMatrices.kronecker_product_rec(
block._A_list[i_div], y)
if block._hash_func == 'pure_hadamard':
a = ffht.create_aligned(y.shape[0], np.float32)
np.copyto(a, y)
ffht.fht(a, min(x.shape[0], 1024, 2048))
y = a
if block._hash_func == 'diagonal' or block._hash_func == 'diagonal_kronecker' or block._hash_func == 'diagonal_gaussian':
y = block._D[i_div] * y
y_global[i_div * self._d:i_div * self._d + self._d] = y
#print(y_global)
if self._normalization:
y_global *= self._norm_factor_list
return y_global
import numpy as np
import ffht
import timeit
reps = 1000
n = 2**20
chunk_size = 1024
a = ffht.create_aligned(n, np.float32)
np.copyto(a, np.random.randn(n))
t1 = timeit.default_timer()
for i in range(reps):
ffht.fht(a, chunk_size)
t2 = timeit.default_timer()
print (t2 - t1 + 0.0) / (reps + 0.0)
def FastIHT_WHT(y, K, Q, d, Sigma, top_k_func=1):
"""
Fast iterative hard thresholding algorithm with partial Walsh-Hadamard Transform sensing matrices.
y : numpy.ndarray
the measurement vector
K : int
number of nonzero entries in the recovered signal
Q : int
dimension of y
d : int
dimension of the recovered signal, must be a power of 2
Sigma : numpy.ndarray
a Q-dimensional array consisting of row indices of the partial WHT matrix
top_k_fun : {0, 1, 2}
indicates the function used for computing the top K indices of a vector
0 - numpy.argpartition
1 - tensorflow.math.top_k
2 - torch.topk
Packages required: numpy, pandas, ffht, tensorflow, torch
"""
if (d & (d-1) != 0) or (d <= 0):
print("The dimension d must be a power of 2.")
return
if not(top_k_func in (0, 1, 2)):
top_k_func = 0
eps = 1e-4
max_iter = 25
res_norms = np.zeros(max_iter) # recording norms of y - Φ * w_vec
wht_factor = 1 / np.sqrt(Q) # scaling factor for WHT
g = np.zeros(d) # g: the current iterate
g_prev = g # g_prev: the previous iterate
res = np.zeros(d)
# res <- transpose(Φ) * y
res[Sigma] = y * wht_factor
ffht.fht(res)
Omega_prev = np.array([], dtype=np.int32) # Omega_prev: support of g_prev
# Omega <- top K indices of transpose(Φ) * y
Omega = TOP_K_ABS_FUNC[top_k_func](res, K) # Omega: support of g
# initialize g as the top K projection of transpose(Φ) * y
g[Omega] = res[Omega]
for s in range(max_iter):
if s == 0:
w_vec = g
Omega_w = Omega # Omega_w: support of w_vec
else:
g_wht = np.copy(g)
ffht.fht(g_wht)
# g_diff <- g - g_prev
g_diff = np.copy(g)
g_diff[Omega_prev] -= g_prev[Omega_prev]
g_diff_wht = np.copy(g_diff)
ffht.fht(g_diff_wht)
g_diff_wht_partial = g_diff_wht[Sigma] * wht_factor
# tau <- <y-Φg, Φ(g-g_prev)> / ||Φ(g-g_prev)||^2
tau = np.dot(y - g_wht[Sigma] * wht_factor, g_diff_wht_partial) / np.dot(g_diff_wht_partial, g_diff_wht_partial)
# w_vec <- g + tau * (g-g_prev)
# and update the support of w (i.e., Omega_w)
Omega_w = pandas.unique(np.concatenate((Omega, Omega_prev)))
w_vec = np.copy(g)
w_vec[Omega_w] += tau * g_diff[Omega_w]
w_vec_wht = np.copy(w_vec)
ffht.fht(w_vec_wht)
res_w = np.zeros(d)
res_w[Sigma] = (y - w_vec_wht[Sigma] * wht_factor) * wht_factor
# stopping criterion:
# if the norm of y - Φ * w_vec remains nearly the same in the last 4 iterations
# or if the current norm of y - Φ * w_vec is small enough
res_norms[s] = np.linalg.norm(res_w[Sigma]) / wht_factor
if s >= 3 and np.std(res_norms[s-3:s+1]) / np.mean(res_norms[s-3:s+1]) < 1e-2:
break
elif res_norms[s] < eps:
break
# res_w <- transpose(Φ) * (y - Φ * w_vec)
ffht.fht(res_w)
res_w_proj_wht = np.zeros(d)
res_w_proj_wht[Omega_w] = res_w[Omega_w] * wht_factor
ffht.fht(res_w_proj_wht)
# alpha_tilde <- ||Proj(res_w, Omega_w)||^2 / ||Φ * Proj(res_w, Omega_w)||^2
alpha_tilde = np.dot(res_w[Omega_w], res_w[Omega_w]) / np.dot(res_w_proj_wht[Sigma], res_w_proj_wht[Sigma])
g_prev = g
Omega_prev = Omega
# h_vec <- w_vec + alpha_tilde * res_w
h_vec = alpha_tilde * res_w
h_vec[Omega_w] += w_vec[Omega_w]
# set g as the top K projection of h_vec
# and update the support of g (i.e., Omega)
Omega = TOP_K_ABS_FUNC[top_k_func](h_vec, K)
g = np.zeros(d)
g[Omega] = h_vec[Omega]
g_wht = np.copy(g)
ffht.fht(g_wht)
# res <- transpose(Φ) * (y - Φ * g)
res = np.zeros(d)
res[Sigma] = (y - g_wht[Sigma] * wht_factor) * wht_factor
ffht.fht(res)
# res_proj_wht <- Φ * Proj(res, Omega)
res_proj_wht = np.zeros(d)
res_proj_wht[Omega] = res[Omega] * wht_factor
ffht.fht(res_proj_wht)
# alpha <- ||Proj(res, Omega)||^2 / ||Φ * Proj(res, Omega)||^2
alpha = np.dot(res[Omega], res[Omega]) / np.dot(res_proj_wht[Sigma], res_proj_wht[Sigma])
# g <- g + alpha * Proj(res, Omega)
g[Omega] += alpha * res[Omega]
return g
def jl_chebyshev(X, lamb):
#print(X[:,0].mean(0))
#assert X[:,0].mean(0) < 1e-4
X = X - X.mean(0, keepdim=True)
n_data, feat_dim = X.size()
X_scaled = X / n_data
#if lamb=0 no scaling, so not to magnify bessel[i, 0] in the approximation.
scale = int(dominant_eval_cov(np.sqrt(lamb) * X)[0]) if lamb > 0 else 1
if scale > 1:
print('Scaling M! {}'.format(scale))
#scale down matrix if matrix norm >= 3, since later scale up when
#odd power
if scale % 2 == 0:
scale -= 1
X_scaled /= scale
else:
scale = 1
subsample_freq = int(feat_dim / math.log(feat_dim, 2)) #100
k = math.ceil(feat_dim / subsample_freq)
X_t = X.t()
#fast Hadamard transform (ffht) vs transform by multiplication by Hadamard mx.
ffht_b = False
P, H, D = get_jl_mx(feat_dim, k, ffht_b)
I_proj = torch.eye(feat_dim, feat_dim, device=X.device)
M = D
I_proj = torch.mm(D, I_proj)
if ffht_b:
#can be obtained from https://github.com/FALCONN-LIB/FFHT
#place here so not everyone needs to install.
import ffht
M = M.t()
#M1 = np.zeros((M.size(0), M.size(1)), dtype=np.double)
M_np = M.cpu().numpy()
I_np = I_proj.cpu().numpy()
for i in range(M.size(0)):
ffht.fht(M_np[i])
for i in range(I_proj.size(0)):
ffht.fht(I_np[i])
#pdb.set_trace()
M = torch.from_numpy(M_np).to(dtype=M.dtype, device=X.device).t()
I_proj = torch.from_numpy(I_np).to(dtype=M.dtype, device=X.device)
else:
#right now form the matrix exponential here
M = torch.mm(H, M)
I_proj = torch.mm(H, I_proj)
#apply P now so downstream multiplications are faster: kd instead of d^2.
#subsample to get reduced dimension
subsample = True
if subsample:
#random sampling performs well in practice and has lower complexity
#select_idx = torch.randint(low=0, high=feat_dim, size=(feat_dim//5,)) <--this produces repeats
if device == 'cuda':
#pdb.set_trace()
select_idx = torch.cuda.LongTensor(
list(range(0, feat_dim, subsample_freq)))
else:
select_idx = torch.LongTensor(
list(range(0, feat_dim, subsample_freq)))
#M = torch.index_select(M, dim=0, index=select_idx)
M = M[select_idx]
#I_proj = torch.index_select(I_proj, dim=0, index=select_idx)
I_proj = I_proj[select_idx]
else:
M = torch.sparse.mm(P, M)
I_proj = torch.sparse.mm(P, I_proj)
#M is now the projection mx
A = M
for _ in range(scale):
#(k x d)
A = sketch_and_apply(lamb, X, X_scaled, A, I_proj)
#Compute tau1 scores
#M = M / M.diag().sum()
#M is (k x d)
#compute tau1 scores (this M is previous M^{1/2})
tau1 = (torch.mm(A, X_t)**2).sum(0)
return tau1
grad += e
# Compute sparsity of grad
sparsity_grad = (np.linalg.norm(grad, ord=1)**2) / (
np.linalg.norm(grad, ord=2)**2 * grad.shape[0])
sp_p(sparsity_grad)
# Recover K-sparse signal
g_rec = FIHT.FastIHT_WHT(z, k, Q, d_aug, Phi_row_idx,
top_k_func=1)[0:d]
e = grad - g_rec
g_rec_wht = np.concatenate(
(g_rec, np.zeros(d_aug - d))) / np.sqrt(Q)
ffht.fht(g_rec_wht)
z_rec = g_rec_wht[Phi_row_idx]
r = z - z_rec
# Unflatten g_rec_wht
shapes = [
model.trainable_weights[i].shape
for i in range(len(model.trainable_weights))
]
grad_tx = []
n_prev = 0
for i in range(len(shapes)):
n = n_prev + tf.math.reduce_prod(shapes[i])
grad_tx.append(
def f(i):
for j in range(i*1000,(i+1)*1000):
ffht.fht(b[j])
return b[i*1000:(i+1)*1000],i
import numpy as np
import ffht
import timeit
import sys
reps = 1000
n = 2**20
chunk_size = 1024
a = np.random.randn(n).astype(np.float32)
t1 = timeit.default_timer()
for i in range(reps):
ffht.fht(a)
t2 = timeit.default_timer()
if sys.version_info[0] == 2:
print(t2 - t1 + 0.0) / (reps + 0.0)
if sys.version_info[0] == 3:
print('{}'.format((t2 - t1 + 0.0) / (reps + 0.0)))
