def load_eigenfunctions(self,m,which_lap='sym',D=None,remove_first_eig=False):
""" Extract ``m`` eigenvectors and eigenvalues of the laplacian, in non-decreasing order.
Args:
which_lap (str) : Chooses the type of laplacian. One of `sym`,`comb` or `rw`.
m (int) : number of eigenvectors to extract
D (`[NDArray[float].shape[N,N]`) : extra matrix for generalized eigenvalue problem
Returns:
Pair[NDArray[float].shape[M,N],NDArray[float].shape[M]] : matrix of eigenvectors, and vector of eigenvalues
"""
if not self.cache_dir is None:
eigvec_path = osp.join(self.cache_dir,f'eigvec_{which_lap}.npy')
eigval_path = osp.join(self.cache_dir,f'eigval_{which_lap}.npy')
files_exist = (osp.isfile(eigvec_path)) and (osp.isfile(eigval_path))
if files_exist:
LOG.info(f"Loading eigenfunctions in {eigvec_path} ...")
EIGVAL = np.load(eigval_path)
EigVec = np.load(eigvec_path)
if EIGVAL.shape[0] >= m:
return EigVec[:,:m], EIGVAL[:m]
L = lap_matrix(self,which_lap)
print(f"Extracting {m} eigenvectors for matrix L (shape: {L.shape}, #edges= {L.data.shape}")
m = min(m,L.shape[0]-1)
eigVec, eigVal = extract_lap_eigvec(L,m,D,remove_first_eig)
if (not self.cache_dir is None):
LOG.info(f"Saving eigenfunctions to {eigvec_path} ...")
np.save(eigvec_path,eigVec)
np.save(eigval_path,eigVal)
return eigVec, eigVal
def run_all(self):
CSV_PATH = os.path.join(CSV_FOLDER, self.get_spec_name() + '.csv')
JOINED_CSV_PATH = os.path.join(CSV_FOLDER,
self.get_spec_name() + '_joined.csv')
cfgs = self.get_all_configs()
cfgs_keys = set()
for x in cfgs:
cfgs_keys.update(x.keys())
#List of produced output dicts
output_dicts = list()
cfgs_size = len(cfgs)
has_written_already = False
bar = progressbar.ProgressBar(maxval=cfgs_size)
counter = 0
bar.start()
bar.update(0)
for i in range(cfgs_size):
print("PROGRESS: {}".format(i / cfgs_size))
#Maybe suppress output
nullwrite = open(os.devnull, 'w')
oldstdout = sys.stdout
if not self.DEBUG_MODE:
sys.stdout = nullwrite
output_dicts.append(self.run(cfgs[i]))
sys.stdout = oldstdout
#Append to csv if conditions are met
if i == cfgs_size - 1 or i % self.WRITE_FREQ == 0:
LOG.info("appending csv...", LOG.ll.SPECIFICATION)
csv_exists = os.path.isfile(CSV_PATH)
if self.OVERWRITE:
if csv_exists and has_written_already:
f_mode = 'a'
else:
f_mode = 'w'
else:
if csv_exists:
f_mode = 'a'
else:
f_mode = 'w'
LOG.debug("f_mode={}".format(f_mode), LOG.ll.SPECIFICATION)
self._append_to_csv(output_dicts, CSV_PATH, f_mode, cfgs_keys)
has_written_already = True
output_dicts.clear()
bar.update(i + 1)
LOG.info(f"CSV saved at f{CSV_PATH}", LOG.ll.SPECIFICATION)
aggregate_csv([CSV_PATH], JOINED_CSV_PATH)
def handle_adaptive_sigma(self,K):
if not scipy.sparse.issparse(K):
M = K
M[M==0] = np.infty
M = np.sort(M, axis=1)
self.sigma = np.mean(M[:,9])/3
LOG.info("Adaptive sigma is {}".format(self.sigma),LOG.ll.MATRIX)
else:
self.sigma = np.mean([np.sort(K.getrow(i).data)[9]/3 for i in range(K.shape[0])])
return partial(lambda d: np.exp(-(d*d)/(2*self.sigma*self.sigma)))
def createVideo(self):
if not self.create_video:
return
if self.steps_taken == 0:
return
LOG.info("Creating video...", LOG.ll.HOOK)
video_command = "ffmpeg -r {} -y -pattern_type glob -i '{}' -c:v libx264 -vf fps=25 -pix_fmt yuv420p '{}'".format(\
self.steps_taken/(15.0*5.0),
os.path.join(self.filename_dir,self.temp_subfolder_name,"*.png".format(self.str_len)),
os.path.join(self.filename_dir,self.video_path)
)
LOG.debug(video_command, LOG.ll.HOOK)
os.system(video_command)
LOG.info("Created video...", LOG.ll.HOOK)
def _faiss_knn(X,k, mode='mut', inner_prod = False):
# kNN search for the graph
X = np.ascontiguousarray(X)
print("Number of GPUS detected by FAISS: {}".format(faiss.get_num_gpus() ))
d = X.shape[1]
res = faiss.StandardGpuResources()
flat_config = faiss.GpuIndexFlatConfig()
flat_config.useFloat16 = False
flat_config.device = 0
c = time.time()
if inner_prod:
faiss.normalize_L2(X)
index = faiss.GpuIndexFlatIP(res,d,flat_config)
else:
index = faiss.GpuIndexFlatL2(res,d,flat_config) # build the index
#normalize_L2(X)
elapsed = time.time() - c
LOG.info(f'kNN Index built in {elapsed:.3f} seconds',LOG.ll.UTILS)
index.add(X)
N = X.shape[0]
Nidx = index.ntotal
c = time.time()
D, I = index.search(X, k + 1)
elapsed = time.time() - c
LOG.info(f'kNN Search done in {elapsed:.3f} seconds',LOG.ll.UTILS)
# Create the graph
D = np.sqrt(D[:,1:])
I = I[:,1:]
row_idx = np.arange(N)
row_idx_rep = np.tile(row_idx,(k,1)).T
W = scipy.sparse.csr_matrix((D.flatten('F'), (row_idx_rep.flatten('F'), I.flatten('F'))), shape=(N, N))
W = __symmetrize_KNN(W,mode=mode)
return W
def LGC_iter_TF(X,W,Y,labeledIndexes, alpha = 0.1,num_iter = 1000, hook=None):
c = time.time()
""" Set W to sparse if necessary, make copy of Y """
W = sparse.csr_matrix(W)
Y = np.copy(Y)
""" Convert W to tensor """
W = convert_sparse_matrix_to_sparse_tensor(W)
LOG.debug(W,LOG.ll.CLASSIFIER)
""" Get degree Matrix """
D = tf.sparse.reduce_sum(W,axis=1)
""" F_0 is a copy of the label matrix, but we erase the information on labeled Indexes """
F_0 = np.copy(Y).astype(np.float32)
F_0[np.logical_not(labeledIndexes),:] = 0.0
"""
CREATE S - Needed for LGC propagation
"""
S = get_S_fromtensor(W)
"""
CREATE F variable
"""
F = tf.Variable(np.copy(F_0).astype(np.float32),name="F")
F_0 = tf.Variable(F_0)
TOTAL_ITER = tf.constant(int(num_iter))
for _ in range(num_iter):
F = (1-alpha)*F_0 + alpha*tf.sparse.sparse_dense_matmul(S,F)
elapsed = time.time() - c
LOG.info('Label Prop done in {:.2} seconds'.format(elapsed),
LOG.ll.CLASSIFIER)
return F.numpy()
def generateAffMat(self,X,Y=None,labeledIndexes=None,hook=None):
""" Generates the Affinity Matrix.
Returns:
`tflabelprop.gssl.graph.gssl_affmat.AffMat`: An affinity matrix
"""
"""
Return Cached matrix, if cache directory exists
"""
X = X.astype(np.float32)
if AffMat.cache_mat_exists(self.cache_dir):
LOG.info(f"Loading Affinity Matrix from {self.cache_dir}...",LOG.ll.MATRIX)
return AffMat(W=None,cache_dir=self.cache_dir)
LOG.info("Creating Affinity Matrix...",LOG.ll.MATRIX)
if not hook is None:
hook._begin(X=X,Y=Y,labeledIndexes=labeledIndexes,W=None)
K = self.get_or_calc_Mask(X)
if self.sigma == "mean":
self.dist_func = self.handle_adaptive_sigma(K)
if not K.shape[0] == X.shape[0]:
raise ValueError("Shapes do not match for X,K")
W = self.W_from_K(X,K)
if self.row_normalize == True:
W = gutils.deg_matrix(W, pwr=-1.0, NA_replace_val=1.0) @ W
del K
LOG.info("Creating Affinity Matrix...Done!",LOG.ll.MATRIX)
assert(W.shape == (X.shape[0],X.shape[0]))
if np.max(W)==0:
raise Exception("Affinity matrix cannot have all entries equal to zero.")
if not hook is None:
hook._end(X=X,Y=Y,W=W)
return AffMat(W=W.astype(np.float32),cache_dir=self.cache_dir)
def __MR(self, X, W, Y, labeledIndexes, p, optimize_labels, hook=None):
"""
-------------------------------------------------------------
INITIALIZATION
--------------------------------------------------------------
"""
ORACLE_Y = Y.copy()
Y = np.copy(Y)
if Y.ndim == 1:
Y = gutils.init_matrix(Y, labeledIndexes)
Y[np.logical_not(labeledIndexes), :] = 0
if not W.shape[0] == Y.shape[0]:
raise ValueError("W,Y shape not compatible")
l = np.reshape(np.array(np.where(labeledIndexes)), (-1))
num_lab = l.shape[0]
if not isinstance(p, int):
p = int(p * num_lab)
if p > Y.shape[0]:
p = Y.shape[0]
LOG.warn("Warning: p greater than the number of labeled indexes",
LOG.ll.CLASSIFIER)
#W = gutils.scipy_to_np(W)
#W = 0.5* (W + W.T)
L = gutils.lap_matrix(W, which_lap='sym')
D = gutils.deg_matrix(W, flat=True, pwr=-1.0)
L = 0.5 * (L + L.T)
def check_symmetric(a, tol=1e-8):
return np.allclose(a, a.T, atol=tol)
def is_pos_sdef(x):
return np.all(np.linalg.eigvals(x) >= -1e-06)
import scipy.sparse
sym_err = L - L.T
sym_check_res = np.all(np.abs(sym_err.data) < 1e-7) # tune this value
assert sym_check_res
"""---------------------------------------------------------------------------------------------------
EIGENFUNCTION EXTRACTION
---------------------------------------------------------------------------------------------------
"""
import time
start_time = time.time()
import os.path as osp
from tf_labelprop.settings import INPUT_FOLDER
cache_eigvec = osp.join(INPUT_FOLDER, 'eigenVectors.npy')
cache_eigval = osp.join(INPUT_FOLDER, 'eigenValues.npy')
if False:
eigenValues, eigenVectors = np.load(cache_eigval), np.load(
cache_eigvec)
eigenVectors = eigenVectors[:, :p]
eigenValues = eigenValues[:p]
else:
eigenVectors, eigenValues = W.load_eigenfunctions(p)
time_elapsed = time.time() - start_time
LOG.info("Took {} seconds to calculate eigenvectors".format(
int(time_elapsed)))
idx = eigenValues.argsort()
eigenValues = eigenValues[idx]
LOG.debug(eigenValues)
assert eigenValues[0] <= eigenValues[eigenValues.shape[0] - 1]
eigenVectors = eigenVectors[:, idx]
np.save(cache_eigval, arr=eigenValues)
np.save(cache_eigvec, arr=eigenVectors)
U = eigenVectors
LAMBDA = eigenValues
U = U[:, np.argsort(LAMBDA)]
LAMBDA = LAMBDA[np.argsort(LAMBDA)]
import tensorflow as tf
gpus = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*8)])
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
"""
-------------------------------------------------------------------------
Define Constants on GPU
------------------------------------------------------------------------------
"""
U, X, Y = [tf.constant(x.astype(np.float32)) for x in [U, X, Y]]
_U_times_U = tf.multiply(U, U)
N = X.shape[0]
def to_sp_diag(x):
n = tf.cast(x.shape[0], tf.int64)
indices = tf.concat([
tf.range(n, dtype=tf.int64)[None, :],
tf.range(n, dtype=tf.int64)[None, :]
],
axis=0)
return tf.sparse.SparseTensor(indices=tf.transpose(indices),
values=x,
dense_shape=[n, n])
@tf.function
def smooth_labels(labels, factor=0.001):
# smooth the labels
labels = tf.cast(labels, tf.float32)
labels *= (1 - factor)
labels += (factor / tf.cast(tf.shape(labels)[0], tf.float32))
# returned the smoothed labels
return labels
@tf.function
def divide_by_row(x, eps=1e-07):
x = tf.maximum(x, 0 * x)
x = x + eps # [N,C] [N,1]
return x / (tf.reduce_sum(x, axis=-1)[:, None])
def spd_matmul(x, y):
return tf.sparse.sparse_dense_matmul(x, y)
def mult_each_row_by(X, by):
""" Elementwise multiplies each row by a given row vector.
For a 2D tensor, also correponds to multiplying each column by the respective scalar in the given row vector
Args:
X (Tensor)
by (Tensor[shape=(N,)]): row vector
"""
#[N,C] [N,1]
return X * by[None, :]
def mult_each_col_by(X, by):
#[N,C] [1,C]
return X * by[:, None]
@tf.function
def accuracy(y_true, y_pred):
acc = tf.cast(
tf.equal(tf.argmax(y_true, axis=-1),
tf.argmax(y_pred, axis=-1)), tf.float32)
acc = tf.cast(acc, tf.float32)
return tf.reduce_mean(acc)
"""
-----------------------------------------------------------------------------
DEFINE VARS
--------------------------------------------------------------------------------
"""
MU = tf.Variable(0.1, name="MU")
LAMBDA = tf.constant(LAMBDA.astype(np.float32), name="LAMBDA")
PI = tf.Variable(tf.ones(shape=(tf.shape(Y)[0], ), dtype=tf.float32),
name="PI")
_l = LAMBDA.numpy()
CUTOFF = tf.Variable(0.0, name='CUTOFF')
CUTOFF_K = tf.Variable(1.0)
@tf.function
def get_alpha(MU):
return tf.pow(2.0, -tf.math.reciprocal(tf.abs(100 * MU)))
@tf.function
def to_prob(x):
return tf.nn.softmax(x, axis=1)
@tf.function
def cutoff(x):
return 1.0 / (1.0 + tf.exp(-CUTOFF_K * (CUTOFF - x)))
model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv1D(8, kernel_size=5, padding='same'))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Conv1D(8, kernel_size=5, padding='same'))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Conv1D(1, kernel_size=3, padding='same'))
model.add(tf.keras.layers.Flatten())
"""
-----------------------------------------------------------------------------
DEFINE FORWARD
--------------------------------------------------------------------------------
"""
@tf.function
def forward(Y, U, PI, mode='train', remove_diag=True):
if mode == 'train':
U = tf.gather(U, indices=np.where(labeledIndexes)[0], axis=0)
Y = tf.gather(Y, indices=np.where(labeledIndexes)[0], axis=0)
#F = tf.gather(F,indices=np.where(labeledIndexes)[0],axis=0)
PI = tf.gather(PI, indices=np.where(labeledIndexes)[0], axis=0)
pi_Y = spd_matmul(to_sp_diag(tf.abs(PI)), Y)
alpha = get_alpha(MU)
"""
Maybe apply custom convolution to LAMBDA, otherwise just fit LGC's alpha using the corresponding filter 1/(1-alpha + alpha*lambda)
"""
if not self.custom_conv:
lambda_tilde = tf.math.reciprocal(1 - alpha + alpha * LAMBDA)
else:
#lambda_tilde = tf.math.reciprocal(1-alpha + alpha*LAMBDA)
_lambda = (LAMBDA -
tf.reduce_mean(LAMBDA)) / tf.math.reduce_std(LAMBDA)
lambda_tilde = tf.clip_by_value(
2 * tf.nn.sigmoid(
tf.reshape(model(_lambda[None, :, None]), (-1, ))), 0,
1)
lambda_tilde = tf.sort(lambda_tilde, direction='DESCENDING')
lambda_tilde = tf.reshape(divide_by_row(lambda_tilde[None, :]),
(-1, ))
_self_infl = mult_each_row_by(
tf.square(U), by=lambda_tilde
) #Square each element of U, then dot product of each row with lambda_tilde
_self_infl = tf.reduce_sum(_self_infl, axis=1)
_P_op = U @ (mult_each_col_by(
(tf.transpose(U) @ pi_Y), by=lambda_tilde))
if not remove_diag:
_diag_P_op = tf.zeros_like(
mult_each_col_by(pi_Y, by=_self_infl))
else:
_diag_P_op = mult_each_col_by(pi_Y, by=_self_infl)
return divide_by_row(_P_op - _diag_P_op), lambda_tilde, pi_Y
"""
-----------------------------------------------------------------------------
DEFINE LOSSES and learning schedule
--------------------------------------------------------------------------------
"""
losses = {
'xent':
lambda y_, y: tf.reduce_mean(-tf.reduce_sum(y_ * tf.cast(
tf.math.log(smooth_labels(y, factor=0.01)), tf.float32),
axis=[1])),
'sq_loss':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.square(y_ - y), axis=[1])),
'abs_loss':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.abs(y_ - y), axis=[1])),
'hinge':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.maximum(1. - y_ * y, tf.zeros_like(y)),
axis=1))
}
NUM_ITER = 700
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
0.5, decay_steps=200, decay_rate=0.9, staircase=False)
opt = tf.keras.optimizers.Adam(0.05)
Y_l = tf.gather(Y, indices=np.where(labeledIndexes)[0], axis=0)
#import matplotlib.pyplot as plt
#import matplotlib
#matplotlib.use('tkagg')
import pandas as pd
"""
-----------------------------------------------------------------------------
LEARNING
--------------------------------------------------------------------------------
"""
L = []
df = pd.DataFrame()
max_acc, min_loss = [0, np.inf]
for i in range(NUM_ITER):
#MU.assign(i)
with tf.GradientTape() as t:
# no need to watch a variable:
# trainable variables are always watched
pred_L, lambda_tilde, pi_Y = forward(Y, U, PI, mode='train')
loss_sq = losses['sq_loss'](pred_L, Y_l)
loss = losses['xent'](pred_L, Y_l)
loss_xent = losses['xent'](pred_L, Y_l)
acc = accuracy(Y_l, pred_L)
_not_lab = np.where(np.logical_not(labeledIndexes))[0]
acc_true = accuracy(
tf.gather(ORACLE_Y, indices=_not_lab, axis=0),
tf.gather(forward(Y, U, PI, mode='eval')[0],
indices=_not_lab,
axis=0))
L.append(
np.array([i, loss_sq, loss, loss_xent, acc,
acc_true])[None, :])
"""
TRAINABLE VARIABLES GO HERE
"""
if self.custom_conv:
trainable_variables = model.weights
else:
trainable_variables = [MU]
if optimize_labels:
trainable_variables.append(PI)
if acc > max_acc:
print(max_acc)
best_trainable_variables = [
k.numpy() for k in trainable_variables
]
max_acc = acc
min_loss = loss
counter_since_best = 0
elif acc <= max_acc:
counter_since_best += 1
if counter_since_best > 2000:
break
"""
Apply gradients
"""
gradients = t.gradient(loss, trainable_variables)
opt.apply_gradients(zip(gradients, trainable_variables))
"""
Project labels such that they sum up to the original amount
"""
pi = PI.numpy()
pi[labeledIndexes] = np.sum(
labeledIndexes) * pi[labeledIndexes] / (np.sum(
pi[labeledIndexes]))
PI.assign(pi)
if i % 10 == 0:
""" Print info """
if not hook is None:
if self.hook_iter_mode == "labeled":
plot_y = np.zeros_like(Y)
plot_y[labeledIndexes] = Y_l.numpy()
else:
plot_y = tf.clip_by_value(
forward(Y, U, PI, mode='eval')[0], 0,
999999).numpy()
hook._step(step=i,
X=X,
W=W,
Y=plot_y,
labeledIndexes=labeledIndexes)
alpha = get_alpha(MU)
PI_l = tf.gather(PI,
indices=np.where(labeledIndexes)[0],
axis=0)
LOG.info(
f"Acc: {acc.numpy():.3f}; ACC_TRUE:{acc_true.numpy():.3f} Loss: {loss.numpy():.3f}; alpha = {alpha.numpy():.3f}; PI mean = {tf.reduce_mean(PI_l).numpy():.3f} "
)
#plt.scatter(range(lambda_tilde.shape[0]),np.log10(lambda_tilde/LAMBDA),s=2)
#plt.show()
for k in range(len(trainable_variables)):
trainable_variables[k].assign(best_trainable_variables[k])
return tf.clip_by_value(forward(Y, U, PI, mode='eval')[0], 0,
999999).numpy()
def __MR(self, X, W, Y, labeledIndexes, p, optimize_labels, hook=None):
"""
-------------------------------------------------------------
INITIALIZATION
--------------------------------------------------------------
"""
ORACLE_Y = Y.copy()
Y = np.copy(Y)
if Y.ndim == 1:
Y = gutils.init_matrix(Y, labeledIndexes)
Y[np.logical_not(labeledIndexes), :] = 0
if not W.shape[0] == Y.shape[0]:
raise ValueError("W,Y shape not compatible")
l = np.reshape(np.array(np.where(labeledIndexes)), (-1))
num_lab = l.shape[0]
if not isinstance(p, int):
p = int(p * num_lab)
if p > Y.shape[0]:
p = Y.shape[0]
LOG.warn("Warning: p greater than the number of labeled indexes",
LOG.ll.CLASSIFIER)
#W = gutils.scipy_to_np(W)
#W = 0.5* (W + W.T)
L = gutils.lap_matrix(W)
D = gutils.deg_matrix(W, flat=True, pwr=-1.0)
L = 0.5 * (L + L.T)
def check_symmetric(a, tol=1e-8):
return np.allclose(a, a.T, atol=tol)
def is_pos_sdef(x):
return np.all(np.linalg.eigvals(x) >= -1e-06)
import scipy.sparse
sym_err = L - L.T
sym_check_res = np.all(np.abs(sym_err.data) < 1e-7) # tune this value
assert sym_check_res
"""---------------------------------------------------------------------------------------------------
EIGENFUNCTION EXTRACTION
---------------------------------------------------------------------------------------------------
"""
import time
start_time = time.time()
eigenVectors, eigenValues = W.load_eigenfunctions(p)
time_elapsed = time.time() - start_time
LOG.info("Took {} seconds to calculate eigenvectors".format(
int(time_elapsed)))
U = eigenVectors
LAMBDA = eigenValues
"""
-------------------------------------------------------------------------
Import and setup Tensorflow
------------------------------------------------------------------------------
"""
import tensorflow as tf
import tf_labelprop.gssl.classifiers.lgc_lvo_aux as aux
gpus = tf.config.experimental.list_physical_devices('GPU')
#tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*8)])
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
"""
-------------------------------------------------------------------------
Define Constants on GPU
------------------------------------------------------------------------------
"""
U, X, Y = [tf.constant(x.astype(np.float32)) for x in [U, X, Y]]
_U_times_U = tf.multiply(U, U)
N = X.shape[0]
"""
-----------------------------------------------------------------------------
DEFINE VARS
--------------------------------------------------------------------------------
"""
MU = tf.Variable(0.1, name="MU")
LAMBDA = tf.constant(LAMBDA.astype(np.float32), name="LAMBDA")
PI = tf.Variable(tf.ones(shape=(tf.shape(Y)[0], ), dtype=tf.float32),
name="PI")
_l = LAMBDA.numpy()
"""
-----------------------------------------------------------------------------
DEFINE FORWARD
--------------------------------------------------------------------------------
"""
def forward(Y, U, PI, mode='train', p=None, remove_diag=True):
if p is None:
p = 99999
pi_Y = aux.spd_matmul(aux.to_sp_diag(tf.abs(PI)), Y)
alpha = self.get_alpha(MU)
"""
Maybe apply custom convolution to LAMBDA, otherwise just fit LGC's alpha using the corresponding filter 1/(1-alpha + alpha*lambda)
"""
#tf.print(alpha)
a = alpha - alpha * LAMBDA
lambda_tilde = 1 / (1 - a)
""" Set entries corresponding to eigvector e_i to zero for i > p """
lambda_tilde = tf.where(
tf.less_equal(tf.range(0, lambda_tilde.shape[0]), p),
lambda_tilde, 0 * lambda_tilde)
_self_infl = aux.mult_each_row_by(
tf.square(U), by=lambda_tilde
) #Square each element of U, then dot product of each row with lambda_tilde
B = _self_infl
_self_infl = tf.reduce_sum(_self_infl, axis=1)
A = aux.mult_each_col_by((tf.transpose(U) @ pi_Y), by=lambda_tilde)
_P_op = U @ (A)
if not remove_diag:
_diag_P_op = tf.zeros_like(
aux.mult_each_col_by(pi_Y, by=_self_infl))
else:
_diag_P_op = aux.mult_each_col_by(pi_Y, by=_self_infl)
if mode == 'eval':
return aux.divide_by_row(_P_op - _diag_P_op)
else:
return A, B, aux.divide_by_row(_P_op - _diag_P_op)
def forward_eval(Y, U, PI, mode='train', p=None, remove_diag=True):
if p is None:
p = 99999
pi_Y = aux.spd_matmul(aux.to_sp_diag(tf.abs(PI)), Y)
alpha = self.get_alpha(MU)
"""
Maybe apply custom convolution to LAMBDA, otherwise just fit LGC's alpha using the corresponding filter 1/(1-alpha + alpha*lambda)
"""
#tf.print(alpha)
a = alpha - alpha * LAMBDA
lambda_tilde = 1 / (1 - a)
""" Set entries corresponding to eigvector e_i to zero for i > p """
lambda_tilde = tf.where(
tf.less_equal(tf.range(0, lambda_tilde.shape[0]), p),
lambda_tilde, 0 * lambda_tilde)
_self_infl = aux.mult_each_row_by(
tf.square(U), by=lambda_tilde
) #Square each element of U, then dot product of each row with lambda_tilde
_self_infl = tf.reduce_sum(_self_infl, axis=1)
A = aux.mult_each_col_by((tf.transpose(U) @ pi_Y), by=lambda_tilde)
_P_op = U @ (A)
if not remove_diag:
_diag_P_op = tf.zeros_like(
aux.mult_each_col_by(pi_Y, by=_self_infl))
else:
_diag_P_op = aux.mult_each_col_by(pi_Y, by=_self_infl)
return aux.divide_by_row(_P_op - _diag_P_op)
"""
-----------------------------------------------------------------------------
DEFINE LOSSES and learning schedule
--------------------------------------------------------------------------------
"""
losses = {
'xent':
lambda y_, y: tf.reduce_mean(-tf.reduce_sum(y_ * tf.cast(
tf.math.log(aux.smooth_labels(y, factor=0.01)), tf.float32),
axis=[1])),
'sq_loss':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.square(y_ - y), axis=[1])),
'abs_loss':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.abs(y_ - y), axis=[1])),
'hinge':
lambda y_, y: tf.reduce_mean(
tf.reduce_sum(tf.maximum(1. - y_ * y, tf.zeros_like(y)),
axis=1))
}
NUM_ITER = 10
Y_l = tf.gather(Y, indices=np.where(labeledIndexes)[0], axis=0)
U_l = tf.gather(U, indices=np.where(labeledIndexes)[0], axis=0)
PI_l = tf.gather(PI, indices=np.where(labeledIndexes)[0], axis=0)
"""
-----------------------------------------------------------------------------
LEARNING
--------------------------------------------------------------------------------
"""
L = []
df = pd.DataFrame()
max_acc, min_loss = [0, np.inf]
best_p = np.inf
for i in range(NUM_ITER, 0, -1):
MU.assign(i)
A, B, _ = forward(Y_l, U_l, PI_l, mode='train')
a1 = np.zeros_like(Y_l)
a2 = np.zeros_like(Y_l)
for i1 in range(p):
a2 += mult_each_col_by(X=Y_l, by=B[:, i1])
a1 += mult_each_col_by(
np.tile(A[i1, :][None, :], [a1.shape[0], 1]), U_l[:, i1])
pred_L = aux.divide_by_row(a1 - a2)
loss_sq = losses['sq_loss'](pred_L, Y_l)
loss = losses['xent'](pred_L, Y_l)
loss_xent = losses['xent'](pred_L, Y_l)
acc = aux.accuracy(Y_l, pred_L)
_not_lab = np.where(np.logical_not(labeledIndexes))[0]
if self.DEBUG:
acc_true = aux.accuracy(
tf.gather(ORACLE_Y, indices=_not_lab, axis=0),
tf.gather(forward_eval(Y, U, PI, mode='eval', p=i1),
indices=_not_lab,
axis=0))
prop = np.max(
pd.value_counts(tf.argmax(pred_L, 1).numpy(),
normalize=True).values)
else:
acc_true = 0
prop = 0
L.append(
np.array(
[i, i1, loss_sq, loss, loss_xent, acc, acc_true,
prop])[None, :])
if (max_acc < acc) or (acc == max_acc and min_loss > loss):
print(
f"acc: {acc},p:{i1},Mu:{int(MU.numpy())}alpha:{self.get_alpha(MU.numpy()).numpy()}"
)
best_p = int(i1)
best_MU = int(MU.numpy())
max_acc = acc
min_loss = loss.numpy()
"""
if self.DEBUG:
alpha = self.get_alpha(MU)
I = np.identity(Y.shape[0], dtype = np.float32)
match_true = tf.gather(np.linalg.inv(I- alpha*(I - gutils.lap_matrix(W,'sym')))@Y,_not_lab,axis=0)
F = forward_eval(Y,U,PI,mode='eval',p=best_p)
match_approx = tf.gather(F,indices=_not_lab,axis=0)
match = aux.accuracy(match_true, match_approx)
print(f"Match rate {np.round(100*match,3)} ")
print(f"LGC_acc = {np.round(100*aux.accuracy(match_true,tf.gather(ORACLE_Y,indices=_not_lab,axis=0)),3)} ")
print(f"LGCLVO_acc = {np.round(100*aux.accuracy(match_approx,tf.gather(ORACLE_Y,indices=_not_lab,axis=0)),3)} ")
"""
if i % 1 == 0:
""" Print info """
if not hook is None:
if self.hook_iter_mode == "labeled":
plot_y = np.zeros_like(Y)
plot_y[labeledIndexes] = Y_l.numpy()
else:
MU.assign(best_MU)
plot_y = tf.clip_by_value(
forward(Y, U, PI, p=best_p, mode='eval'), 0,
999999).numpy()
hook._step(step=i,
X=X,
W=W,
Y=plot_y,
labeledIndexes=labeledIndexes)
alpha = self.get_alpha(MU)
LOG.info(
f"Acc: {max_acc.numpy():.3f}; Loss: {loss.numpy():.3f}; alpha = {alpha.numpy():.3f};"
)
if self.DEBUG:
df = pd.DataFrame(np.concatenate(L, axis=0),
index=range(len(L)),
columns=[
'i', 'p', 'loss_sq', 'loss', 'loss_xent',
'acc', 'acc_true', 'prop'
])
self.create_3d_mesh(df)
print(f"BEst mu: {best_MU}; best p: {best_p}")
MU.assign(best_MU)
print(MU)
return forward_eval(Y, U, PI, mode='eval', p=None).numpy()
"""
----------------------------------------------------
PART 2
-------------------------------------------------
"""
opt = tf.keras.optimizers.Adam(0.05)
max_acc = 0
for i in range(7000):
#MU.assign(i)
with tf.GradientTape() as t:
_, _, pred_L = forward(Y_l,
U_l,
tf.gather(
PI,
indices=np.where(labeledIndexes)[0],
axis=0),
mode='train',
p=best_p)
loss_sq = losses['sq_loss'](pred_L, Y_l)
loss = losses['xent'](pred_L, Y_l)
loss_xent = losses['xent'](pred_L, Y_l)
acc = aux.accuracy(Y_l, pred_L)
_not_lab = np.where(np.logical_not(labeledIndexes))[0]
acc_true = aux.accuracy(
tf.gather(ORACLE_Y, indices=_not_lab, axis=0),
tf.gather(forward(Y, U, PI, mode='eval')[0],
indices=_not_lab,
axis=0))
L.append(
np.array([i, loss_sq, loss, loss_xent, acc,
acc_true])[None, :])
"""
Project labels such that they sum up to the original amount
"""
pi = PI.numpy()
pi[labeledIndexes] = np.sum(
labeledIndexes) * pi[labeledIndexes] / (np.sum(
pi[labeledIndexes]))
PI.assign(pi)
"""
TRAINABLE VARIABLES GO HERE
"""
trainable_variables = []
if optimize_labels:
trainable_variables.append(PI)
"""
Apply gradients
"""
gradients = t.gradient(loss, trainable_variables)
opt.apply_gradients(zip(gradients, trainable_variables))
if acc > max_acc:
print(max_acc)
best_trainable_variables = [
k.numpy() for k in trainable_variables
]
max_acc = acc
min_loss = loss
counter_since_best = 0
for k in range(len(trainable_variables)):
trainable_variables[k].assign(best_trainable_variables[k])
return forward(Y, U, PI, mode='eval', p=None).numpy()
"""
for c in df.columns:
if c.startswith('loss'):
df[c] = (df[c] - df[c].min())/(df[c].max()-df[c].min())
for c in df.columns:
if not c in 'i':
plt.plot(df['i'],df[c],label=c)
plt.legend()
plt.show()
#plt.scatter(range(lambda_tilde.shape[0]),np.log10(lambda_tilde/LAMBDA),s=2)
#plt.show()
"""
return tf.clip_by_value(forward(Y, U, PI, mode='eval')[0], 0,
999999).numpy()
def intcomp_demo():
LOG.info("Demonstração para Inteligência Computacional")
ds, alg = "", ""
while not ds in ['mnist','isolet','g241c']:
LOG.info("Qual DATASET? (MNIST ou ISOLET)")
ds = input().lower()
while not alg in ['L','D','N']:
LOG.info("Qual LGCLVO? (L ou D ou N[enhum])")
alg = input().upper()
import tf_labelprop.experiment.specification.specification_bits as spec
from tf_labelprop.experiment.specification.specification_bits import allPermutations as P
from tf_labelprop.experiment.specification.specification_skeleton import EmptySpecification
class ExpIntComp(EmptySpecification):
CACHE_AFFMAT = False
def __init__(self,ds,alg):
self.ds = ds
self.alg = alg
def get_spec_name(self):
return "INTCOMP"
def generalConfig(self):
s = spec.GENERAL_DEFAULT
s['id'] = [1]
return P(s)
def inputConfig(self):
if self.ds == 'mnist':
s = spec.INPUT_MNIST
s['labeled_percent'] = [100/70000]
elif self.ds=="isolet":
s = spec.INPUT_ISOLET
else:
s = spec.INPUT_CHAPELLE_A
s["use_chapelle_splits"] = [True]
s['num_labeled'] = [10]
s['benchmark'] = ['digit1']
return P(s)
def filterConfig(self):
def alpha_to_mu(alpha):
return (1-alpha)/alpha
if self.alg in ["L"]:
s = spec.FILTER_LGC_LVO_AUTO
s['LGC_iter'] = [1000]
else:
s = spec.FILTER_NOFILTER
return P(s)
def noiseConfig(self):
s = spec.NOISE_UNIFORM_DET_SOME
s["corruption_level"] = [0.0 if self.ds == "mnist" else 0.0]
return P(s)
def affmatConfig(self):
if self.ds == "g241c":
s = spec.AFFMAT_DEFAULT
s['k'] = [500]
elif self.ds=="mnist":
s = spec.AFFMAT_DEFAULT
s['k'] = [15]
elif self.ds == "isolet":
s = spec.AFFMAT_ISOLET
else:
s = spec.AFFMAT_DEFAULT
return P(s)
def algConfig(self):
if self.alg=="D":
s =spec.ALGORITHM_LGCLOO_DEFAULT
s["optimize_labels"] = [False]
s["custom_conv"] = [False]
s['p'] = [1500 if self.ds == 'isolet' else 50]
return P(s)
s = spec.ALGORITHM_SIIS_DEFAULT
s['m'] = [100]
#s['mu'] = [(1-0.9)/0.9]
#s['alpha'] = [0.9]
#s['num_iter'] = [1000.0]
def alpha_to_mu(alpha):
return (1-alpha)/alpha
#s["num_iter"] = [1000]
return P(s)
opt = ExpIntComp(ds=ds,alg=alg).get_all_configs()[0]
ExperimentRun(opt).run(hook_list=[])
def _log(msg):
LOG.info(msg,LOG.ll.EXPERIMENT)
def run(self,hook_list=PLOT_HOOKS):
for k,v in self.args.items():
LOG.info("{}:{}".format(k,v),LOG.ll.EXPERIMENT)
#Multiplex the arguments, allocating each to the correct step
mplex = postprocess(keys_multiplex(self.args))
#Get Hooks:
hooks = select_and_add_hook(hook_list, mplex, self)
LOG.info("Step 1: Read Dataset",LOG.ll.EXPERIMENT)
#Select Input
self.X,self.W, self.Y_true, self.labeledIndexes = select_input(**mplex["INPUT"])
if self.W is None:
self.W = select_affmat(**mplex["AFFMAT"]).generateAffMat(self.X,hook=hooks["AFFMAT"])
if "know_estimated_freq" in mplex["ALG"].keys():
if mplex["ALG"]['know_estimated_freq']:
mplex["ALG"]["use_estimated_freq"] = np.sum(self.Y_true,axis=0) / self.Y_true.shape[0]
mplex["ALG"].pop("know_estimated_freq")
if "know_estimated_freq" in mplex["FILTER"].keys():
if mplex["ALG"]['know_estimated_freq']:
mplex["FILTER"]["use_estimated_freq"] = np.sum(self.Y_true,axis=0) / self.Y_true.shape[0]
mplex["FILTER"].pop("know_estimated_freq")
LOG.info("Step 2: Apply Noise",LOG.ll.EXPERIMENT)
#Apply Noise
self.Y_noisy = select_noise(**mplex["NOISE"]).corrupt(self.Y_true, self.labeledIndexes,hook=hooks["NOISE"])
LOG.info("Step 3: Create Affinity Matrix",LOG.ll.EXPERIMENT)
#Generate Affinity Matrix
self.W = select_affmat(**mplex["AFFMAT"]).generateAffMat(self.X,hook=hooks["AFFMAT"])
LOG.info("Step 4: Filtering",LOG.ll.EXPERIMENT)
#Create Filter
ft = select_filter(**mplex["FILTER"])
self.ft = ft
noisyIndexes = (np.argmax(self.Y_true,axis=1) != np.argmax(self.Y_noisy,axis=1))
self.Y_filtered, self.labeledIndexes_filtered = ft.fit(self.X, self.Y_noisy, self.labeledIndexes, self.W, hook=hooks["FILTER"])
LOG.info("Step 5: Classification",LOG.ll.EXPERIMENT)
#Select Classifier
alg = select_classifier(**mplex["ALG"])
#Get Classification
self.F = alg.fit(self.X,self.W,self.Y_filtered,self.labeledIndexes_filtered,hook=hooks["ALG"])
LOG.info("Step 6: Evaluation",LOG.ll.EXPERIMENT)
LOG.debug("ALGORITHM settings:{}".format(mplex["ALG"]["algorithm"]),LOG.ll.EXPERIMENT)
""" Accuracy. """
acc = gutils.accuracy(gutils.get_pred(self.F), gutils.get_pred(self.Y_true))
acc_unlabeled = gutils.accuracy(gutils.get_pred(self.F)[np.logical_not(self.labeledIndexes)],\
gutils.get_pred(self.Y_true)[np.logical_not(self.labeledIndexes)])
acc_labeled = gutils.accuracy(gutils.get_pred(self.F)[self.labeledIndexes],\
gutils.get_pred(self.Y_true)[self.labeledIndexes])
CMN_acc = gutils.accuracy(gutils.get_pred(gutils.class_mass_normalization(self.F,self.Y_filtered,self.labeledIndexes,normalize_rows=True)), gutils.get_pred(self.Y_true))
"""
Log accuracy results and update output dictionary
"""
def _log(msg):
LOG.info(msg,LOG.ll.EXPERIMENT)
_log("Accuracy: {:.3%} | {:.3%}".format(acc,1-acc))
_log("Accuracy (unlabeled): {:.3%} |{:.3%}".format(acc_unlabeled,1-acc_unlabeled))
_log("Accuracy (labeled): {:.3%} | {:.3%}".format(acc_labeled,1-acc_labeled))
_log("Accuracy w/ CMN: {:.3%} | {:.3%}".format(CMN_acc,1-CMN_acc))
self.out_dict.update({OUTPUT_PREFIX + "acc" :acc})
self.out_dict.update({OUTPUT_PREFIX + "acc_unlabeled" :acc_unlabeled})
self.out_dict.update({OUTPUT_PREFIX + "acc_labeled" :acc_labeled})
self.out_dict.update({OUTPUT_PREFIX + "CMN_acc" :CMN_acc})
return self.out_dict
def transition_count_mat(Y, A):
""" Obtains a transition count matrix for uniform noise,
indicating how many instances should have their label flipped and to which class.
Specifically, this returns a matrix M such that :math:`M[i,j]` is the number of instances of i-th class to be swapped
to the j-th class.
Args:
Y (`[NDArray[int].shape[N,C]`) : Matrix encoding initial beliefs.
A (`[NDArray[int].shape[C,C]`): Transition probabilities between each class.
Returns:
`[NDArray[int].shape[C,C]`: the transition count matrix.
Raises:
ValueError: if `p` is an invalid percentage.
"""
c = Y.shape[1]
class_freq = [int(round(sum(Y[:, i]))) for i in range(c)]
num_clean = int(np.round(sum([class_freq[i] * A[i, i] for i in range(c)])))
LOG.info(
"NUM CLEAN:{};NUM NOISY:{};TOTAL:{}".format(
num_clean,
np.sum(class_freq) - num_clean, np.sum(class_freq)), LOG.ll.NOISE)
""" Little procedure that allocates clean labels according to prob. of each diagonal entry """
import heapq
B = np.zeros((c, )) #Soon to be our diagonal
H = [(-A[i, i] * class_freq[i], i) for i in range(c)]
heapq.heapify(H)
for i in range(num_clean):
x = heapq.heappop(H)
id = x[1]
val = x[0]
B[id] += 1
heapq.heappush(H, (val + 1, id))
""" We approximate the "most commonly observed" scenario by rounding """
#Fix possible isues
for i in range(c):
A[i, :] = np.round(A[i, :] * (class_freq[i] - num_clean))
A[i, i] = B[i]
A = A.astype(np.int32)
observed_class_counts = np.asarray([A[i, i] for i in range(c)])
expected_class_counts = np.asarray(
[np.sum(np.argmax(Y, axis=1) == i) for i in range(c)])
INFTY = A.shape[0]**2
for i in range(c):
S = sum(A[i, :]) - class_freq[i]
if S > 0:
for s in range(S):
diff = observed_class_counts - expected_class_counts
diff[i] = -INFTY
diff[A[i, :] == 0] = -2 * INFTY
j = np.random.choice(np.flatnonzero(diff == diff.max()))
observed_class_counts[j] -= 1
A[i, j] -= 1
elif S < 0:
for s in range(-S):
diff = expected_class_counts - observed_class_counts
diff[i] = -INFTY
j = np.random.choice(np.flatnonzero(diff == diff.max()))
observed_class_counts[j] += 1
A[i, j] += 1
assert sum(A[i, :]) - class_freq[i] == 0
return A
def apply_noise(Y, labeledIndexes, A, seed=None, deterministic=True):
""" Corrupts a set percentage of initial labels with noise.
Args:
Y (`[NDArray[int].shape[N,C]`) : Matrix encoding initial beliefs.
A (`[NDArray[int].shape[C,C]`): Transition probabilities between each class.
labeledIndexes (`NDArray[bool].shape[N]`) : determines which indices are to be considered as labeled.
seed (float) : Optional. Used to reproduce results.
Returns:
`NDArray[int].shape[N,C]` : Belief matrix after corruption.
"""
np.random.seed(seed)
old_A = np.copy(np.asarray(A))
if not np.all(old_A <= 1):
LOG.debug(old_A, LOG.ll.NOISE)
raise Exception("trans. mat has value >1")
old_Y = np.copy(Y)
is_flat = np.ndim(Y) == 1
if is_flat:
Y = gutils.init_matrix(Y, labeledIndexes)
c = Y.shape[1]
n = Y.shape[0]
Y = Y[labeledIndexes, :]
Y_flat = np.argmax(Y, axis=1)
vec = np.random.RandomState(seed).permutation(Y.shape[0])
assert not vec is None
cursor = np.zeros((c), dtype=np.int32)
if deterministic == True:
A = transition_count_mat(Y, A)
else:
class_freq = [int(np.sum(Y[:, i])) for i in range(c)]
num_clean = np.sum(labeledIndexes) * sum(
[old_A[i, i] for i in range(c)]) / c
num_clean = int(np.round(num_clean))
num_noisy = np.sum(labeledIndexes) - num_clean
##########3
perm = np.random.permutation(Y.shape[0])[0:num_noisy]
A = np.zeros((c, c))
for i in range(c):
A[i, i] = class_freq[i]
for my_id in perm:
j = np.argmax(Y[my_id, :])
A[j, j] -= 1
new_j = j
while new_j == j:
new_j = np.random.choice(c)
A[j, new_j] += 1
assert np.sum(A) == np.sum(labeledIndexes)
LOG.debug(A, LOG.ll.NOISE)
###############
for i in np.arange(Y_flat.shape[0]):
current_class = Y_flat[vec[i]]
while A[current_class, cursor[current_class]] == 0:
cursor[current_class] += 1
assert cursor[current_class] < c
Y_flat[vec[i]] = cursor[current_class]
A[current_class, cursor[current_class]] -= 1
noisy_Y = np.zeros(shape=(n, c))
labeledIndexes_where = np.where(labeledIndexes)[0]
for l in range(Y_flat.shape[0]):
noisy_Y[labeledIndexes_where[l], Y_flat[l]] = 1
noisy_Y[np.logical_not(labeledIndexes), :] = old_Y[
np.logical_not(labeledIndexes), :]
LOG.info(
"Changed {} percent of entries".format(
np.round(1 - gutils.accuracy(np.argmax(Y, axis=1), Y_flat), 6)),
LOG.ll.NOISE)
if is_flat:
old_Y[labeledIndexes] = np.argmax(noisy_Y[labeledIndexes], axis=1)
return old_Y
else:
return noisy_Y
def __LGC(self,
X,
W,
Y,
labeledIndexes,
alpha=0.1,
useEstimatedFreq=None,
hook=None):
""" Init """
import scipy.sparse
if scipy.sparse.issparse(W):
W = W.todense()
Y = self.CLEAN_UNLABELED_ROWS(Y, labeledIndexes)
if not W.shape[0] == Y.shape[0]:
raise ValueError("W,Y shape not compatible")
""" Estimate frequency of classes"""
num_labeled = Y[labeledIndexes].shape[0]
num_classes = Y.shape[1]
if not useEstimatedFreq is None:
if isinstance(useEstimatedFreq, bool):
estimatedFreq = np.sum(Y[labeledIndexes], axis=0) / num_labeled
else:
estimatedFreq = useEstimatedFreq
else:
estimatedFreq = np.repeat(1 / num_classes, num_classes)
omega = estimatedFreq
""" """
mu = (1 - alpha) / alpha
n = Y.shape[0]
c = Y.shape[1]
I = np.identity(Y.shape[0])
S = I - gutils.lap_matrix(W, which_lap='sym')
""" stuff that has matrix multiplication with theta """
theta = (1 / mu) * np.asarray(np.linalg.inv(I - alpha * S))
F_lgc = (theta @ Y) * mu
theta_1n = np.sum(theta, axis=1).flatten()
theta_1n_ratio = (theta_1n /
(np.sum(theta_1n)))[:, np.newaxis] #Shape: nx1
print(theta_1n_ratio.shape)
""" Intermediate calc """
zeta = n * omega - np.sum(F_lgc, axis=0) #Shape: 1xc
zeta = np.reshape(zeta, (1, c))
ypsilon = np.ones(shape=(n,1)) - np.sum(F_lgc,axis=1)[:,np.newaxis] -\
theta_1n_ratio * (n - np.sum(F_lgc.flatten())) #Shape: nx1
F = F_lgc
F += theta_1n_ratio @ zeta
F += (1 / c) * (ypsilon @ np.ones((1, c)))
log_args = [
np.round(x, 3)
for x in [np.sum(F, axis=1)[0:10],
np.sum(F, axis=0), n * omega]
]
LOG.info(
"F sum on rows: {} (expected 1,1,...,1); F sum col: {} (expected {})"
.format(*log_args))
return F
def LGCLVO(self,
X,
W,
Y,
labeledIndexes,
mu=99.0,
useEstimatedFreq=True,
tuning_iter=0,
hook=None,
constant_prop=False,
useZ=True,
normalize_rows=True):
labeledIndexes, noisyIndexes = labeledIndexes
Y = np.copy(Y)
#We make a deep copy of labeledindexes
labeledIndexes = np.array(labeledIndexes)
lids = np.where(labeledIndexes)[0]
if Y.ndim == 1:
Y = gutils.init_matrix(Y, labeledIndexes)
Y[np.logical_not(labeledIndexes), :] = 0
if not W.shape[0] == Y.shape[0]:
raise ValueError("W,Y shape not compatible")
W = 0.5 * (W + W.transpose())
num_labeled = Y[labeledIndexes].shape[0]
num_unlabeled = Y.shape[0] - num_labeled
num_classes = Y.shape[1]
D = gutils.deg_matrix(W, flat=True)
if not useEstimatedFreq is None:
if isinstance(useEstimatedFreq, bool):
estimatedFreq = np.sum(Y[labeledIndexes], axis=0) / num_labeled
else:
estimatedFreq = useEstimatedFreq
else:
estimatedFreq = np.repeat(1 / num_classes, num_classes)
if scipy.sparse.issparse(W):
l = np.sum(labeledIndexes)
itertool_prod = [[i, j] for i in range(l) for j in range(l)]
row = np.asarray([lids[i] for i in range(l)])
col = np.asarray([i for i in range(l)])
data = np.asarray([1.0] * l)
temp_Y = _to_np(
scipy.sparse.coo_matrix((data, (row, col)),
shape=(W.shape[0], l)))
PL = LGC_iter_TF(X,
W,
Y=temp_Y,
labeledIndexes=labeledIndexes,
alpha=1 / (1 + mu),
num_iter=1000)
PL = PL[labeledIndexes, :]
PL[range(PL.shape[0]), range(PL.shape[0])] = 0 #Set diagonal to 0
PL = PL
del temp_Y
row = np.asarray(
[lids[x[0]] for x in itertool_prod if x[0] != x[1]])
col = np.asarray(
[lids[x[1]] for x in itertool_prod if x[0] != x[1]])
data = [PL[x[0], x[1]] for x in itertool_prod if x[0] != x[1]]
P = scipy.sparse.coo_matrix((data, (row, col)),
shape=W.shape).tocsr()
P = P
else:
#Identity matrix
I = np.identity(W.shape[0])
#Get graph laplacian
L = gutils.lap_matrix(W, which_lap='sym')
#Propagation matrix
P = np.zeros(W.shape)
P[np.ix_(labeledIndexes,
labeledIndexes)] = np.linalg.inv(I + 0.5 *
(L + L.transpose()) /
mu)[np.ix_(
labeledIndexes,
labeledIndexes)]
P[labeledIndexes, labeledIndexes] = 0
P[np.ix_(labeledIndexes, labeledIndexes)] = P[np.ix_(
labeledIndexes, labeledIndexes)] / np.sum(P[np.ix_(
labeledIndexes, labeledIndexes)],
axis=0,
keepdims=False)
W = scipy.sparse.csr_matrix(W)
Z = []
detected_noisylabels = []
suggested_labels = []
where_noisylabels = []
Q_values = []
Y_flat = np.argmax(Y, axis=1)
def divide_row_by_sum(e):
e = _to_np(e)
if normalize_rows:
e = e / np.sum(e + 1e-100, axis=1, keepdims=True)
return e
else:
return e
def find_argmin(Q, class_to_unlabel):
id_min_line = np.argmin(Q[:, class_to_unlabel])
id_min_col = class_to_unlabel
return id_min_line, id_min_col, Q[id_min_line, id_min_col]
#######################################################################################
'''BEGIN iterations'''
Q = None
cleanIndexes = np.copy(labeledIndexes)
for i_iter in range(tuning_iter):
found_noisy = True
if np.sum(labeledIndexes) > 0 and found_noisy:
'''Z matrix - The binary values of current Y are replaced with their corresponding D entries.
Then, we normalize each row so that row sums to its estimated influence
'''
useZ = False
if i_iter >= 0:
if (not self.use_baseline) or Q is None:
if useZ:
Z = gutils.calc_Z(Y,
labeledIndexes,
D,
estimatedFreq,
weigh_by_degree=False)
F = P @ Z
if scipy.sparse.issparse(F):
F = np.asarray(F.toarray())
#Compute graph gradient
Q = (divide_row_by_sum(F) - divide_row_by_sum(Z))
else:
F = P @ Y
if scipy.sparse.issparse(F):
F = np.asarray(F.toarray())
Q = (divide_row_by_sum(F) - divide_row_by_sum(Y))
#import scipy.stats
#During label tuning, we'll also 'unlabel' the argmax
unlabeledIndexes = np.logical_not(cleanIndexes)
if self.early_stop:
Q[np.sum(F, axis=1) == 0.0, :] = 9999
Q[unlabeledIndexes, :] = np.inf
#Find minimum unlabeled index
if constant_prop:
expectedNumLabels = estimatedFreq * np.sum(labeledIndexes)
actualNumLabels = np.sum(Y[labeledIndexes, :], axis=0)
temp = expectedNumLabels - actualNumLabels
class_priority = np.argsort(temp)
found_noisy = False
for class_to_unlabel in class_priority:
id_min_line, id_min_col, val = find_argmin(
Q, class_to_unlabel)
if val < 0:
#This means that the class would have a different label under the modified label prop
found_noisy = True
break
else:
id_min = np.argmin(Q)
id_min_line = id_min // num_classes
id_min_col = id_min % num_classes #The class previously assigned to instance X_{id_min_line}
found_noisy = Q[id_min_line, id_min_col] < 0
if found_noisy:
id_max_col = np.argmax(
Q[id_min_line, :]) #The new, suggested class
detected_noisylabels.append(id_min_col)
where_noisylabels.append(id_min_line)
suggested_labels.append(id_max_col)
Q_values.append(1 + Q[id_min_line, id_min_col])
#Unlabel OP
if labeledIndexes[id_min_line] == False:
raise Exception(
"Error: unlabeled instance was selected")
if not Y[id_min_line, id_min_col] == 1:
raise Exception("Error: picked wrong class to unlabel")
labeledIndexes[id_min_line] = False
cleanIndexes[id_min_line] = False
if not Y[id_min_line, id_min_col] == 1:
raise Exception(
"Tried to remove label from unlabeled instance")
Y[id_min_line, id_min_col] = 0
if self.relabel:
labeledIndexes[id_min_line] = True
Y[id_min_line, :] = 0
Y[id_min_line, id_max_col] = 1
if not hook is None:
hook._step(step=(i_iter + 1),
X=X,
W=W,
Y=Y,
labeledIndexes=labeledIndexes)
'''
MATPLOTLIB stuff
'''
import cv2 as cv
#ret2,th2 = cv.threshold(255*np.asarray(Q_values).astype(np.uint8),0,255,cv.THRESH_BINARY+cv.THRESH_OTSU)
import matplotlib
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(5 * 3, 2 * 3))
ax = fig.add_subplot()
#ax.plot(np.arange(len(Q_values)),Q_values)
ax.scatter(np.arange(len(Q_values)),
Q_values,
c=noisyIndexes[where_noisylabels])
ax.set_xlabel("#Labels Removed", fontsize=22)
ax.set_ylabel("Consistency with LGC", fontsize=22)
ax.axvline(np.sum(noisyIndexes), color='red')
# We change the fontsize of minor ticks label
ax.tick_params(axis='both', which='major', labelsize=18)
ax.tick_params(axis='both', which='minor', labelsize=18)
# For the minor ticks, use no labels; default NullFormatter.
ax.yaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(0.1))
fig.tight_layout()
plt.axhline(np.max(Q_values[0:(1 + np.sum(noisyIndexes))]),
color='green')
plt.grid(True, axis='y', linestyle='-', alpha=0.5, which='major')
plt.grid(True, axis='y', linestyle='--', alpha=0.5, which='minor')
#plt.axvline(th2,color='purple')
plt.savefig(
'/home/klaus/eclipse-workspace/NoisyGSSL/results/python_plotly/' +
'mnist_alpha=0.99_noise=0.3_thresh_static.png')
#print(th2)
plt.show()
'''END iterations'''
LOG.info(
"NUMBER OF DETECTED NOISY INSTANCES:{}".format(
len(detected_noisylabels)), LOG.ll.FILTER)
return Y, labeledIndexes
def __LGC_iter_TF(self,
X,
W,
Y,
labeledIndexes,
alpha=0.1,
useEstimatedFreq=True,
num_iter=1000,
hook=None):
from tf_labelprop.gssl.classifiers._lgc_tf import LGC_iter_TF
""" Init """
import scipy.sparse
if not scipy.sparse.issparse(W):
W = scipy.sparse.csr_matrix(W)
Y = np.copy(Y)
if Y.ndim == 1:
Y = gutils.init_matrix(Y, labeledIndexes)
Y[np.logical_not(labeledIndexes), :] = 0
if not W.shape[0] == Y.shape[0]:
raise ValueError("W,Y shape not compatible")
""" Estimate frequency of classes"""
num_labeled = Y[labeledIndexes].shape[0]
num_classes = Y.shape[1]
if not useEstimatedFreq is None:
if isinstance(useEstimatedFreq, bool):
estimatedFreq = np.sum(Y[labeledIndexes], axis=0) / num_labeled
else:
estimatedFreq = useEstimatedFreq
else:
estimatedFreq = np.repeat(1 / num_classes, num_classes)
omega = estimatedFreq
""" """
mu = (1 - alpha) / alpha
n = Y.shape[0]
c = Y.shape[1]
print(np.concatenate([Y, np.ones((n, 1))], axis=1))
""" stuff that has matrix multiplication with theta """
PY1 = LGC_iter_TF(X, W, np.concatenate([Y, np.ones((n, 1))], axis=1),
labeledIndexes, alpha, num_iter, hook)
PY1 = np.asarray(PY1)
F_lgc, theta_1n = (1 / mu) * PY1[:, :-1], (1 / mu) * PY1[:, -1]
theta_1n_ratio = (theta_1n /
(np.sum(theta_1n)))[:, np.newaxis] #Shape: nx1
""" Intermediate calc """
zeta = n * omega - np.sum(F_lgc, axis=0) #Shape: 1xc
zeta = np.reshape(zeta, (1, c))
ypsilon = np.ones(shape=(n,1)) - np.sum(F_lgc,axis=1)[:,np.newaxis] -\
theta_1n_ratio * (n - np.sum(F_lgc.flatten())) #Shape: nx1
F = F_lgc
F += theta_1n_ratio @ zeta
F += (1 / c) * (ypsilon @ np.ones((1, c)))
import pandas as pd
print(pd.Series(np.argmax(F, axis=1)).value_counts() / n)
log_args = [
np.round(x, 3)
for x in [np.sum(F, axis=1)[0:10],
np.sum(F, axis=0), n * omega]
]
LOG.info(
"F sum on rows: {} (expected 1,1,...,1); F sum col: {} (expected {})"
.format(*log_args))
return F
def plotGraph(X,
W,
labeledIndexes,
vertex_opt,
plot_filepath=None,
online=False,
interactive=False,
title="",
plot_size=_plot_big,
edge_width=0.5,
labeled_only=False):
""" Plots a GSSL graph.
Creates a plot showing vertices connected by edges from the affinity matrix in 2D/3D.
A number of different configurations is possible by the use of a vertexplotOpt object.
Args:
X (`NDArray[float].shape[N,D]`): A 2D or 3D matrix containing the vertex positions.
W (`NDArray[float].shape[N,N]`): Optional. The affinity matrix defining the graph.
vertex_opt (vertexOptObject) : The size/color/group vertex configuration object.
title (string, default = ``''``) : The title to be printed above the image.
online (bool, default = ``False``) : whether to create an online plot
interactive (bool, default = ``False``) : whether to open an interactive plot on the browser
plot_size (`List[int].shape[2]`, default = ``[1000,1000]``) : size of the canvas for the plotting operation.
edge_width (float, default = ``0.5``) : thickness of the edges.
Raises:
ValueError: ``if X.shape[1] not in [2, 3]``
Returns:
None
"""
if plot_filepath is None:
plot_filepath = path.join(PLOT_FOLDER,
str(datetime.datetime.now()) + ".png")
plot_dim = X.shape[1]
if plot_dim < 2 or plot_dim > 3:
raise ValueError("plot_dim must be either 2 or 3")
if (not W is None) and W.shape[0] != X.shape[0]:
raise ValueError("Shapes of W, X do not match")
if plot_dim > X.shape[1]:
#Add missing dimensions
temp = np.zeros((X.shape[0], plot_dim))
temp[0:X.shape[0], 0:X.shape[1]] = X
X = temp
if not os.path.exists(os.path.dirname(plot_filepath)):
os.makedirs(os.path.dirname(plot_filepath))
def axis(dim_num):
M = np.max(X[:, dim_num])
m = np.min(X[:, dim_num])
x = (M - m) / 2
M = M + (M - x) * 0.2
m = m + (m - x) * 0.2
axis = dict(showline=False,
zeroline=False,
showgrid=False,
showticklabels=False,
visible=True,
title='',
range=[m, M])
return (axis)
scene = {}
scene["xaxis"] = dict(axis(0))
scene["yaxis"] = dict(axis(1))
if plot_dim == 3:
scene["zaxis"] = dict(axis(2))
layout = go.Layout(
paper_bgcolor='rgb(0,0,0,0)',
plot_bgcolor='rgb(255,255,255)',
title=title,
font=dict(family='Courier New, bold', size=30, color='black'),
width=plot_size[0],
height=plot_size[1],
legend=dict(x=0,
y=1,
traceorder='normal',
font=dict(family='sans-serif', size=30, color='#000'),
bgcolor='#E2E2E2',
bordercolor='#FFFFFF',
borderwidth=2),
showlegend=True,
xaxis=scene["xaxis"],
yaxis=scene["yaxis"],
margin=dict(t=100),
hovermode='closest',
annotations=[
dict(showarrow=False,
text="</a>",
xref='paper',
yref='paper',
x=0,
y=0.1,
xanchor='left',
yanchor='bottom',
font=dict(size=14))
],
)
if "zaxis" in scene.keys():
layout.update(go.Layout(zaxis=scene["zaxis"]))
#Create Traces
data = []
if not W is None:
trace_edge = _traceEdges(X=X,
W=W,
plot_dim=plot_dim,
edge_width=edge_width)
data = trace_edge
trace_vertex = _traceVertex(X=X,
labeledIndexes=labeledIndexes,
plot_dim=plot_dim,
v_opt=vertex_opt)
data += trace_vertex
#Create figure
fig = go.Figure(data=data, layout=layout)
LOG.info("Plotting graph ({}) ...".format(title), LOG.ll.OUTPUT)
if online:
try:
py.iplot(fig)
except:
LOG.warn("Warning: Could not plot online", LOG.ll.OUTPUT)
if interactive:
pyoff.offline.plot(fig)
pio.write_image(fig, plot_filepath)
LOG.info("Plotting graph({})...Done!".format(title), LOG.ll.OUTPUT)
""" Gets the color values for a discrete palette. """
#palette = "husl"
if Y.shape[0] == -1:
raise ""
Y = Y - np.min(Y)
pal = sns.color_palette(palette, np.max(Y) + 10)
#pal = [pal[2],pal[6]]
res = 255 * np.array(list(map(lambda k: (pal[int(k)]), Y)))
#print(res)
return (res)
def color_scale_continuous(Y, palette="coolwarm", num_palette=70):
""" Gets the color values for a continuous palette. """
Y = (Y - np.min(Y))
Y = Y / np.max(Y)
Y[np.isnan(Y)] = 0.5
pal = ig.AdvancedGradientPalette(sns.color_palette(palette, 7),
n=num_palette + 1)
res = 255 * np.array(
list(map(lambda k: (pal.get(int(num_palette * k))), Y)))
res = res.astype(np.int64)
return res
if __name__ == "__main__":
LOG.info(PLOT_FOLDER, LOG.ll.OTHER)
def rmFolders(self):
if self.keep_images:
return
shutil.rmtree(os.path.join(self.filename_dir,
self.temp_subfolder_name))
LOG.info("Deleted images....", LOG.ll.HOOK)
def _end(self, **kwargs):
LOG.info("Best F1 score: {}".format(self.best_f1), LOG.ll.HOOK)
pass
def get_all_configs(self):
"""Gets the configuration for every experiment. The corresponding prefix is added for each stage.
Returns:
`List[dict]` A list of all possible configs.
"""
g = self.generalConfig()
for x in g:
x["spec_name"] = self.get_spec_name()
Z = [(g,GENERAL_PREFIX),
(self.inputConfig(),INPUT_PREFIX),
(self.noiseConfig(),NOISE_PREFIX),
(self.filterConfig(),FILTER_PREFIX),
(self.affmatConfig(),AFFMAT_PREFIX),
(self.algConfig(),ALG_PREFIX)\
]
l = [[spec.add_key_prefix(y, elem) for elem in x] for x, y in Z]
res = list(reduce(lambda x, y: spec.comb(x, y), l))
res = self.remove_undesirable_configs(res)
LOG.info("Number of configurations: {}".format(len(res)))
if self.CACHE_AFFMAT:
res_df = pd.DataFrame(res)
_cache_col = AFFMAT_PREFIX + "cache_dir"
res_df[_cache_col] = '/tmp/{}'.format(self.get_spec_name())
relevant_keys = self.get_keys_relevant_to_affmat()
class Counter():
def __init__(self):
self.counter = 0
def inc(self):
self.counter += 1
def value(self):
return self.counter
COUNTER = Counter()
def add_cache(df):
df[_cache_col] = [
os.path.join(x, str(COUNTER.value()))
for x in df[_cache_col].values
]
COUNTER.inc()
print(COUNTER.value())
return df
res_df = res_df.fillna('__nan')
for g, df in res_df.groupby(relevant_keys):
print(g)
res_df = res_df.groupby(relevant_keys).apply(add_cache)
res_df = res_df.sort_values(axis=0,
by=[_cache_col, GENERAL_PREFIX + 'id'])
"""
----------------------------------------
Cache Directory handling
-----------------------------------------
"""
if self.CLEAN_AFFMAT_DIR:
for _dir in pd.unique(res_df[_cache_col]):
print(_dir)
if os.path.isdir(_dir):
shutil.rmtree(_dir)
print(list(res_df.index))
res = list(np.array(res)[list(res_df.index)])
_caches = res_df[_cache_col].values
for i, dct in enumerate(res):
dct[_cache_col] = _caches[i]
return res