def cvx_online_dict_learning(X,
y_true,
n_hat,
k_cluster,
T,
lmda,
eps,
flag=True,
version='Rr'):
'''
X: R^(n * m)
y_true: str^n
W_0: R^(n_hat * k)
x_i : R^m
alpha: R^k
cvx_online problem
min||x_i - X.T * W * alpha|| + lambda * ||alpha||
in the online setting, there is no X in (n * m),
instead, we need to store a candidate set and solve the subproblem:
min ||x_i - X_hat * W_hat * alpha|| + lambda * ||alpha||
X_hat : R^(m * n_hat)
W_hat : R^(n_hat * k)
version: Rr, restricted, heuristic approach
Ru, uniform, random assignment
'''
n_dim, m_dim = X.shape
A_t = np.zeros((k_cluster, k_cluster))
B_t = np.zeros((m_dim, k_cluster))
x_sum = 0
alpha_sum = 0
# step 1: sample n_hat * k_cluster points as initial X_hat.
X_0 = np.zeros((m_dim, n_hat))
for idx in range(n_hat):
sample_idx = np.random.randint(0, n_dim)
x_sample = X[sample_idx, :]
X_0[:, idx] = x_sample
# step 1: initialization, get X_hat (including clusters info)
# and W_hat from X_0, using same init as in CNMF.
# here representative_size_count is the n_1_hat, n_2_hat, ..., n_k_hat.
t1 = time.time()
X_hat, W_hat, representative_size_count = initialize_X_W_hat(
X_0, k_cluster)
X_0, W_0 = X_hat.copy(), W_hat.copy()
t2 = time.time()
# print('init cost {:.4f}'.format(t2 - t1))
# step 2: after initialization of X_hat, update alpha, W_hat and X_hat alternatively.
t_start = time.time()
print(lmda, _NF, eps)
g_hat_list = []
error_eval_list = []
t_cur = 0
for t in range(T):
# t_start_online = time.time()
g_hat_i = get_g_hat_value(t, W_hat, X_hat, A_t, B_t, x_sum, alpha_sum)
D_tmp_dummy = X_hat @ W_hat
error_i = eval_g_hat_with_DnX(X, D_tmp_dummy.T, n_dim, m_dim)
g_hat_list.append((t_cur, g_hat_i))
error_eval_list.append((t_cur, error_i))
if t % 50 == 0 and flag:
D_t = np.matmul(X_hat, W_hat)
tmp_assignment = get_clustering_assignment_1(X, D_t, k_cluster)
tmp_acc, tmp_AMI = evaluation_clustering(tmp_assignment, y_true)
print('1)iteration {}, distance acc = {:.4f}, AMI = {:.4f}'.format(
t, tmp_acc, tmp_AMI))
tmp_assignment = get_clustering_assignment_2(
X, D_t, k_cluster, lmda)
tmp_acc, tmp_AMI = evaluation_clustering(tmp_assignment, y_true)
print(
'2)iteration {}, kmeans of weights acc = {:.4f}, AMI = {:.4f}'.
format(t, tmp_acc, tmp_AMI))
t_end = time.time()
print('time elapse = {:.4f}s'.format(t_end - t_start))
t_start = t_end
print('-' * 7)
sample_idx = np.random.randint(0, n_dim)
x_sample = X[sample_idx, :]
# update alpha
t1 = time.time()
lars_lasso = LassoLars(alpha=lmda, max_iter=500)
D_t = np.matmul(X_hat, W_hat)
lars_lasso.fit(D_t, x_sample)
alpha_t = lars_lasso.coef_
# t2 = time.time()
# print('lasso cost {:.4f}s'.format(t2 - t1))
# using different clustering assignment
# t1 = time.time()
if version == 'Rr':
cluster_of_x_i = np.argmax(alpha_t)
# elif version == 'Ru':
else:
cluster_of_x_i = int(np.random.uniform(0, k_cluster))
# t2 = time.time()
# print('argmax alpha cost {:.4f}s'.format(t2 - t1))
# t1 = time.time()
A_t += np.matmul(alpha_t.reshape(k_cluster, 1),
alpha_t.reshape(1, k_cluster))
B_t += np.matmul(x_sample.reshape(m_dim, 1),
alpha_t.reshape(1, k_cluster))
x_sum += (np.linalg.norm(x_sample)**2)
alpha_sum += lmda * np.linalg.norm(alpha_t, 1)
# t2 = time.time()
# print('update At, Bt cost {:.4f}s'.format(t2 - t1))
# update X_hat
# t1 = time.time()
W_hat, X_hat = update_W_X_hat(W_hat, X_hat, representative_size_count,
x_sample, cluster_of_x_i, A_t, B_t,
x_sum, alpha_sum, t, eps)
t2 = time.time()
t_cur += (t2 - t1)
# print('update X_hat, W_hat cost {:.4f}s'.format(t2 - t1))
print('Dcitionary update done! Time elapse {:.04f}s'.format(time.time() -
t_start))
return W_hat, X_hat, representative_size_count, X_0, W_0, g_hat_list, error_eval_list
def update_W_X_hat(W_hat, X_hat, repre_size_count, x_sample, cluster_of_x_i,
A_t, B_t, x_sum, alpha_sum, t, eps):
# add W_hat block diagonal constraint,
# using projection.
# linalg.init()
# W_hat_gpu = gpuarray.to_gpu(W_hat.astype(np.float64))
# tmp_x = np.ascontiguousarray(X_hat)
# X_hat_gpu = gpuarray.to_gpu(tmp_x.astype(np.float64))
# A_t_gpu = gpuarray.to_gpu(A_t.astype(np.float64))
# B_t_gpu = gpuarray.to_gpu(B_t.astype(np.float64))
cluster_seperation_idx = np.cumsum(repre_size_count)
end_idx = cluster_seperation_idx[cluster_of_x_i]
start_idx = end_idx - repre_size_count[cluster_of_x_i]
A_t_inv = np.linalg.pinv(A_t)
# W_opt_old_X = opt_cal_W_hat_numpy(W_hat, X_hat, A_t, B_t, x_sum, alpha_sum, eps, t)
W_opt_old_X = opt_cal_W_hat_solve(W_hat, X_hat, A_t_inv, B_t, x_sum, alpha_sum, eps, t)
g_hat_old_X = get_g_hat_value(t, W_opt_old_X, X_hat, A_t, B_t, x_sum, alpha_sum)
# W_opt_old_X = update_W_hat_skcuda(W_hat_gpu, X_hat_gpu, A_t_gpu, B_t_gpu,
# x_sum, alpha_sum, eps, t)
# g_hat_old_X = get_g_hat_value(t, W_opt_old_X.get(), X_hat, A_t, B_t, x_sum, alpha_sum)
list_of_W_opt_new_X = [W_opt_old_X]
list_of_g_hat_new_X = [g_hat_old_X]
list_of_new_X = [X_hat]
# print('starting loop in update_W_X, total {}'.format(end_idx - start_idx))
for idx in range(start_idx, end_idx):
# print('iter # {}'.format(idx))
t1 = time.time()
X_hat_new = X_hat.copy()
X_hat_new[:, idx] = x_sample
list_of_new_X.append(X_hat_new)
# tmp_x = np.ascontiguousarray(X_hat_new)
# X_hat_new_gpu = gpuarray.to_gpu(tmp_x.astype(np.float64))
t2 = time.time()
# print('\t update X_hat cost {:.4f}s'.format(t2 - t1))
t1 = time.time()
# W_opt_new_X = opt_cal_W_hat_numpy(W_hat, X_hat_new, A_t, B_t, x_sum, alpha_sum, eps, t)
# W_opt_new_X = update_W_hat_numpy(W_hat, X_hat_new, A_t, B_t, x_sum, alpha_sum, eps, t)
W_opt_new_X = opt_cal_W_hat_solve(W_hat, X_hat_new, A_t_inv, B_t, x_sum, alpha_sum, eps, t)
g_hat_new_X = get_g_hat_value(t, W_opt_new_X, X_hat_new, A_t, B_t, x_sum, alpha_sum)
# W_opt_new_X = update_W_hat_skcuda(W_hat_gpu, X_hat_new_gpu, A_t_gpu, B_t_gpu,
# x_sum, alpha_sum, eps, t)
# g_hat_new_X = get_g_hat_value(t, W_opt_new_X.get(), X_hat_new, A_t, B_t, x_sum, alpha_sum)
t2 = time.time()
# print('\t update W_hat_new cost {:.4f}'.format(t2 - t1))
t1 = time.time()
list_of_W_opt_new_X.append(W_opt_new_X)
list_of_g_hat_new_X.append(g_hat_new_X)
t2 = time.time()
# print('appending W_opt list cost {:.4f}s'.format(t2 - t1))
min_g_idx = np.argmin(list_of_g_hat_new_X)
X_hat_new = list_of_new_X[min_g_idx]
W_hat_new = list_of_W_opt_new_X[min_g_idx]
# if list_of_g_hat_new_X[min_g_idx] <= g_hat_old_X:
# X_hat_new = X_hat.copy()
# X_hat_new[:, start_idx + min_g_idx] = x_sample
# # W_hat_new = list_of_W_opt_new_X[min_g_idx].get()
# W_hat_new = list_of_W_opt_new_X[min_g_idx].copy()
# else:
# X_hat_new = X_hat.copy()
# # W_hat_new = W_opt_old_X.get()
# W_hat_new = W_opt_old_X.copy()
return W_hat_new, X_hat_new
x_sum += (np.linalg.norm(x_sample)**2)
alpha_sum += lmda * np.linalg.norm(alpha_i, 1)
# T1 = 1/2 * (np.linalg.norm(x_sample.reshape(m_dim, 1) -
# X_hat @ W_hat @ alpha_i.reshape(k, 1)) ** 2)
# T2 = lmda * np.linalg.norm(alpha_i, 1)
# g_val_culmulative += (T1 + T2)
t1 = time.time()
for _ in range(10):
W_cuda = update_W_hat_skcuda(W_hat, X_hat, A_t, B_t, x_sum, alpha_sum,
eps, T)
t2 = time.time()
print('pycuda cost {:.4f}s'.format(t2 - t1))
# print('total number of loop {}, each cost {:.4f}s'.format(k, (t2-t1)/k))
g_hat_val_cuda = get_g_hat_value(T, W_cuda.get(), X_hat, A_t, B_t, x_sum,
alpha_sum)
print('g value = {:.4f}'.format(g_hat_val_cuda))
print('-' * 7)
t1 = time.time()
for _ in range(10):
W_numpy = update_W_hat_numpy(W_hat, X_hat, A_t, B_t, x_sum, alpha_sum,
eps, T)
t2 = time.time()
print('numpy dense cost {:.4f}s'.format(t2 - t1))
# print('total number of loop {}, each cost {:.4f}s'.format(k, (t2-t1)/k))
g_hat_val_np = get_g_hat_value(T, W_numpy, X_hat, A_t, B_t, x_sum,
alpha_sum)
print('g value = {:.4f}'.format(g_hat_val_np))
print('-' * 7)