def y_and_J(self, x):
x_torch = utils.convert_into_at_least_2d_pytorch_tensor(
x).requires_grad_().to(self.device)
with torch.enable_grad():
y_torch = self.y_torch(x_torch)
J_torch = torch.stack([
torch.autograd.grad(y_torch[:, d].sum(),
x_torch,
retain_graph=True,
create_graph=self.is_training)[0]
for d in range(y_torch.shape[1])
],
dim=1)
return y_torch.cpu().detach().numpy(), J_torch.cpu().detach().numpy()
test_successes = np.zeros((n_iter, len(runs)))
for run_idx, run in enumerate(runs):
result_path = '../plot/ecmnn/' + task + '/r{:02d}/'.format(run)
aug_dataloader_input_file = result_path + 'aug_dataloader_' + task + '.pkl'
with open(aug_dataloader_input_file, 'rb') as aug_dataloader:
data = pickle.load(aug_dataloader)
[
batch_train_loader, _, _, all_train_dataset, all_valid_dataset,
all_test_dataset, all_dataset
] = data.get_train_valid_test_dataloaders(batch_size=128,
train_fraction=0.85,
valid_fraction=0.0)
train_inputs = convert_into_at_least_2d_pytorch_tensor(
all_train_dataset['data'])
test_inputs = convert_into_at_least_2d_pytorch_tensor(
all_test_dataset['data'])
train_norm_level = all_train_dataset['norm_level_data'].flatten()
train_inputs = train_inputs[train_norm_level == 0.0]
train_inputs = train_inputs[:n_data, :].detach().numpy()
test_norm_level = all_test_dataset['norm_level_data'].flatten()
test_inputs = test_inputs[test_norm_level == 0.0]
test_inputs = test_inputs[:n_data, :].detach().numpy()
q_max = np.max(train_inputs, axis=0)
q_min = np.min(train_inputs, axis=0)
for iter in range(n_iter):
def __init__(self, dataset_filepath, *args, **kwargs):
rand_seed = kwargs.get('rand_seed', 38)
is_performing_data_augmentation = kwargs.get(
'is_performing_data_augmentation', True)
is_computing_all_cost_components = kwargs.get(
'is_computing_all_cost_components', True)
is_using_level_dependent_weight = kwargs.get(
'is_using_level_dependent_weight', False)
N_normal_space_traversal = kwargs.get('N_normal_space_traversal', 9)
is_optimizing_signed_siamese_pairs = kwargs.get(
'is_optimizing_signed_siamese_pairs', True)
clean_aug_data = kwargs.get('clean_aug_data', True)
aug_clean_thresh = kwargs.get('aug_clean_thresh', 1e-1)
is_aligning_lpca_normal_space_eigvecs = kwargs.get(
'is_aligning_lpca_normal_space_eigvecs', True)
N_normal_space_eigvecs_alignment_repetition = kwargs.get(
'N_normal_space_eigvecs_alignment_repetition', 1)
is_augmenting_w_rand_comb_of_normaleigvecs = kwargs.get(
'is_augmenting_w_rand_comb_of_normaleigvecs', True)
N_siam_same_levelvec = kwargs.get('N_siam_same_levelvec', 5)
N_local_neighborhood_mult = kwargs.get('N_local_neighborhood_mult', 1)
if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components:
onmanif_siam_same_levelvecs_list = list()
else:
is_aligning_lpca_normal_space_eigvecs = False
dataset_dict = {}
dataset_dict['data'] = np.load(dataset_filepath + '.npy')
N_data = dataset_dict['data'].shape[0]
dim_ambient = dataset_dict['data'].shape[1]
print('N_data = %d' % N_data)
dataset_dict['norm_level_data'] = np.zeros((N_data, 1))
dataset_dict['norm_level_weight'] = np.ones((N_data, 1))
dataset_dict['augmenting_vector'] = np.zeros_like(dataset_dict['data'])
# N_local_neighborhood = int(round(N_data/1000))
N_local_neighborhood = N_local_neighborhood_mult * 2 * (2**dim_ambient)
print('N_local_neighborhood = %d (%f percent)' %
(N_local_neighborhood, (100.0 * N_local_neighborhood) / N_data))
start_time = time.clock()
S_list = []
kd_tree = cKDTree(data=dataset_dict['data'])
for i in tqdm.tqdm(range(N_data)):
# compute nearest neighbors of size N_local_neighborhood (exclude self):
[_, nearneigh_indices] = kd_tree.query(dataset_dict['data'][i],
k=N_local_neighborhood + 1)
nearneigh_indices = nearneigh_indices[1:]
vecs_from_curr_pt_to_nearneighs = (
dataset_dict['data'][nearneigh_indices] -
dataset_dict['data'][i])
assert (vecs_from_curr_pt_to_nearneighs.shape[0] ==
N_local_neighborhood)
if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components:
# pick N_siam_same_levelvec random (on-manifold) configurations (exclude self/i-th config)
# which all supposed to have the same implicit vector-value/level vector/ecmnn output vector
# as the current/i-th configuration (i.e. zero vector):
all_idx_but_self = list(range(N_data))
all_idx_but_self.pop(i)
onmanif_siam_same_levelvec_indices = np.random.permutation(
all_idx_but_self)[:N_siam_same_levelvec]
onmanif_siam_same_levelvecs = dataset_dict['data'][
onmanif_siam_same_levelvec_indices, :]
onmanif_siam_same_levelvecs_list.append(
onmanif_siam_same_levelvecs)
# compute the sample covariance matrix of the nearest neighbors (for Local PCA):
X = vecs_from_curr_pt_to_nearneighs
XTX = X.T @ X
S = XTX / (N_local_neighborhood - 1)
S_list.append(S)
dataset_dict['cov'] = np.stack(S_list)
print("Local PCA Covariance Matrices are computed in %f seconds." %
(time.clock() - start_time))
if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components:
dataset_dict['siam_reflection_data'] = copy.deepcopy(
dataset_dict['data'])
dataset_dict['siam_reflection_weight'] = np.ones((N_data, 1))
dataset_dict['siam_same_levelvec_data'] = np.stack(
onmanif_siam_same_levelvecs_list)
# (on-manifold) siamese same-level-vectors loss' weight is one:
dataset_dict['siam_same_levelvec_weight'] = np.ones(
(N_data, N_siam_same_levelvec))
# while the on-manifold siamese augmentation fraction loss' weight is zero
# (because the ecmnn output vector/implicit vector-value/level vector
# is expected to be a zero vector here, hence it CANNOT be normalized into a unit vector,
# and technically there is NO distinctive siamese augmentation fraction pair of it):
dataset_dict['siam_frac_aug_weight'] = np.zeros((N_data, 1))
del onmanif_siam_same_levelvecs_list
# pre-compute the SVD of the local sample covariance matrices, i.e. performing Local PCA:
cov_torch = convert_into_at_least_2d_pytorch_tensor(
dataset_dict['cov'])
[_, cov_torch_svd_s, cov_torch_svd_V] = torch.svd(cov_torch,
some=False)
dataset_dict['cov_svd_s'] = cov_torch_svd_s.cpu().detach().numpy()
dataset_dict['cov_svd_V'] = cov_torch_svd_V.cpu().detach().numpy()
# determine the intrinsic dimensionality of the Constraint Manifold,
# based on the index of the maximum drop of Eigenvalue
# (idea from Shay Deutsch's PhD Dissertation manuscript):
diff_eigval = dataset_dict['cov_svd_s'][:, :(
dim_ambient - 1)] - dataset_dict['cov_svd_s'][:, 1:]
argmax_diff_eigval = np.argmax(diff_eigval, axis=1)
[dim_tangent_space, _] = stats.mode(argmax_diff_eigval, axis=None)
dim_tangent_space = dim_tangent_space[0] + 1
dim_normal_space = dim_ambient - dim_tangent_space
print("Tangent Space Dimensionality = %d" % dim_tangent_space)
print("Normal Space Dimensionality = %d" % dim_normal_space)
assert (dim_normal_space >= 1)
N_aug_rand_normal_space_vecs = (4**dim_normal_space)
# some Eigenvalue statistics of the Local PCA:
max_eigval = np.max(dataset_dict['cov_svd_s'])
min_eigval = np.min(dataset_dict['cov_svd_s'])
mean_eigval = np.mean(dataset_dict['cov_svd_s'])
max_tangent_eigval = np.max(
dataset_dict['cov_svd_s'][:, :dim_tangent_space])
min_tangent_eigval = np.min(
dataset_dict['cov_svd_s'][:, :dim_tangent_space])
mean_tangent_eigval = np.mean(
dataset_dict['cov_svd_s'][:, :dim_tangent_space])
epsilon = N_local_neighborhood_mult * np.sqrt(mean_tangent_eigval)
print("Maximum Eigenvalue = %f" % max_eigval)
print("Minimum Eigenvalue = %f" % min_eigval)
print("Mean Eigenvalue = %f" % mean_eigval)
print("Maximum Tangent Space Eigenvalue = %f" % max_tangent_eigval)
print("Minimum Tangent Space Eigenvalue = %f" % min_tangent_eigval)
print("Mean Tangent Space Eigenvalue = %f" % mean_tangent_eigval)
print("Epsilon = %f" % epsilon)
# extract Tangent Space Eigenvectors and Normal Space Eigenvectors:
dataset_dict['cov_rowspace'] = dataset_dict[
'cov_svd_V'][:, :, :dim_tangent_space]
dataset_dict['cov_nullspace'] = dataset_dict[
'cov_svd_V'][:, :, dim_tangent_space:]
tangent_space_eigvecs = dataset_dict['cov_rowspace']
if is_aligning_lpca_normal_space_eigvecs:
for n_normal_space_eigvecs_alignment_repetition in range(
N_normal_space_eigvecs_alignment_repetition):
print('Performing Normal Space Eigenvector Bases iter %d/%d' %
(n_normal_space_eigvecs_alignment_repetition + 1,
N_normal_space_eigvecs_alignment_repetition))
normal_space_eigvecs = align_normal_space_eigvecs(
dataset_dict, rand_seed=rand_seed)
dataset_dict['cov_nullspace'] = copy.deepcopy(
normal_space_eigvecs)
else:
normal_space_eigvecs = dataset_dict['cov_nullspace']
if is_performing_data_augmentation:
# append more points based on the Normal Space Eigenvectors and the computed epsilon:
dataset_list_dict = {
key: [dataset_dict[key]]
for key in dataset_dict
}
for i in range(1, N_normal_space_traversal + 1):
print('Data Augmentation %d/%d' %
(i, N_normal_space_traversal))
unsigned_level_mult = epsilon * i
level_dependent_weight = 1.0
if is_using_level_dependent_weight:
level_dependent_weight /= (unsigned_level_mult**2)
dset_n_nspacetrv_list_dict = {
key: list()
for key in dataset_dict
}
if is_augmenting_w_rand_comb_of_normaleigvecs:
if dim_normal_space > 1:
aug_modes = ['deterministic', 'randomized']
else:
aug_modes = ['deterministic']
else:
aug_modes = ['deterministic']
level_mult_eigvec_list = list()
for aug_mode in aug_modes:
if aug_mode == 'deterministic':
for d in range(dim_normal_space):
for sign in [-1, 1]:
level_mult_eigvec = sign * unsigned_level_mult * normal_space_eigvecs[:, :,
d]
level_mult_eigvec_list.append(
level_mult_eigvec)
elif aug_mode == 'randomized':
# randomized combination of the normal space eigenvectors:
for _ in range(N_aug_rand_normal_space_vecs):
rand_combinator = np.tile(
np.random.normal(size=(1, dim_normal_space,
1)), (N_data, 1, 1))
rand_normal_space_vec = np.squeeze(
normal_space_eigvecs @ rand_combinator)
rand_normal_space_unitvec = (
rand_normal_space_vec / np.expand_dims(
npla.norm(rand_normal_space_vec, axis=1),
axis=1))
level_mult_eigvec = unsigned_level_mult * rand_normal_space_unitvec
level_mult_eigvec_list.append(level_mult_eigvec)
for level_mult_eigvec in level_mult_eigvec_list:
new_data = dataset_dict['data'] + level_mult_eigvec
# delete indices from augmented data if they do not fulfill the neighborhood condition
valid_idx = list(range(N_data))
if clean_aug_data:
del_idx = []
for idx, x in enumerate(new_data):
[_, idx_near] = kd_tree.query(x)
if (npla.norm(new_data[idx_near] - x) >
(aug_clean_thresh * unsigned_level_mult)):
del_idx += [idx]
[valid_idx.remove(idx) for idx in del_idx]
N_aug = len(valid_idx)
print('Accepted aug points ', N_aug, ' / ', N_data)
if N_aug == 0:
continue
valid_data = new_data[valid_idx]
dset_n_nspacetrv_list_dict['data'].append(valid_data)
dset_n_nspacetrv_list_dict['augmenting_vector'].append(
level_mult_eigvec[valid_idx])
if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components:
new_siam_reflection_data = dataset_dict[
'data'] - level_mult_eigvec
dset_n_nspacetrv_list_dict[
'siam_reflection_data'].append(
new_siam_reflection_data[valid_idx])
new_siam_reflection_weight = np.ones(
(N_data, 1)) * level_dependent_weight
for val_idx in valid_idx:
x = new_siam_reflection_data[val_idx]
[_, idx_near] = kd_tree.query(x)
if (npla.norm(new_siam_reflection_data[idx_near] -
x) >
(aug_clean_thresh * unsigned_level_mult)):
new_siam_reflection_weight[
val_idx,
0] = 0.0 # siam_reflection_loss will NOT be imposed
dset_n_nspacetrv_list_dict[
'siam_reflection_weight'].append(
new_siam_reflection_weight[valid_idx])
N_valid_siam_reflection_pairs = int(
round(
np.sum(
new_siam_reflection_weight[valid_idx] > 0.0
)))
print('Valid siamese reflection pairs ',
N_valid_siam_reflection_pairs, ' / ',
len(valid_idx))
dset_n_nspacetrv_list_dict[
'siam_frac_aug_weight'].append(
np.ones((N_aug, 1)) * level_dependent_weight)
siam_same_levelvecs_list = list()
siam_same_levelvecs_weight_list = list()
for j in range(N_aug):
# pick N_siam_same_levelvec random (augmented) configurations (exclude self/j-th config)
# which all supposed to have the same implicit vector-value/level vector/ecmnn output vector
# as the current/j-th configuration:
all_idx_but_self = list(range(N_aug))
all_idx_but_self.pop(j)
siam_same_levelvec_indices = np.random.permutation(
all_idx_but_self)[:N_siam_same_levelvec]
# siam_same_levelvec_indices = np.array([(int(round(j +
# ((N_aug/(N_siam_same_levelvec+1.0))*ns)))
# % N_aug)
# for ns in range(1, N_siam_same_levelvec+1)])
siam_same_levelvecs = np.zeros(
(N_siam_same_levelvec, dim_ambient))
siam_same_levelvecs_weight = np.zeros(
N_siam_same_levelvec)
N_siam_same_levelvec_indices = siam_same_levelvec_indices.shape[
0]
if N_siam_same_levelvec_indices > 0:
siam_same_levelvecs[:N_siam_same_levelvec_indices, :] = valid_data[
siam_same_levelvec_indices, :]
siam_same_levelvecs_weight[:N_siam_same_levelvec_indices] = np.ones(
N_siam_same_levelvec_indices)
siam_same_levelvecs_list.append(
siam_same_levelvecs)
siam_same_levelvecs_weight_list.append(
siam_same_levelvecs_weight)
dset_n_nspacetrv_list_dict[
'siam_same_levelvec_data'].append(
np.stack(siam_same_levelvecs_list))
dset_n_nspacetrv_list_dict[
'siam_same_levelvec_weight'].append(
np.stack(siam_same_levelvecs_weight_list) *
level_dependent_weight)
dset_n_nspacetrv_list_dict['norm_level_data'].append(
unsigned_level_mult * np.ones((N_aug, 1)))
dset_n_nspacetrv_list_dict['norm_level_weight'].append(
np.ones((N_aug, 1)) * level_dependent_weight)
dset_n_nspacetrv_list_dict['cov'].append(
copy.deepcopy(dataset_dict['cov'][valid_idx]))
dset_n_nspacetrv_list_dict['cov_svd_s'].append(
copy.deepcopy(dataset_dict['cov_svd_s'][valid_idx]))
dset_n_nspacetrv_list_dict['cov_svd_V'].append(
copy.deepcopy(dataset_dict['cov_svd_V'][valid_idx]))
dset_n_nspacetrv_list_dict['cov_rowspace'].append(
copy.deepcopy(dataset_dict['cov_rowspace'][valid_idx]))
dset_n_nspacetrv_list_dict['cov_nullspace'].append(
copy.deepcopy(
dataset_dict['cov_nullspace'][valid_idx]))
dset_n_nspacetrv_dict = {
key: np.concatenate(dset_n_nspacetrv_list_dict[key],
axis=0)
for key in dset_n_nspacetrv_list_dict
}
for key in dataset_dict:
dataset_list_dict[key].append(dset_n_nspacetrv_dict[key])
for key in dataset_dict:
dataset_dict[key] = np.concatenate(dataset_list_dict[key],
axis=0)
# some checks to ensure validity of cov_rowspace and cov_nullspace:
cov_rowspace = dataset_dict['cov_rowspace']
cov_nullspace = dataset_dict['cov_nullspace']
cov_rowspace_projector = cov_rowspace @ np.transpose(cov_rowspace,
axes=(0, 2, 1))
cov_nullspace_projector = cov_nullspace @ np.transpose(cov_nullspace,
axes=(0, 2, 1))
# projection from cov_nullspace to cov_rowspace and the otherwise shall be zero
cov_nullspace_proj_to_rowspace = (
cov_rowspace_projector @ cov_nullspace)
npt.assert_almost_equal(cov_nullspace_proj_to_rowspace,
np.zeros_like(cov_nullspace_proj_to_rowspace),
decimal=3)
cov_rowspace_proj_to_nullspace = (
cov_nullspace_projector @ cov_rowspace)
npt.assert_almost_equal(cov_rowspace_proj_to_nullspace,
np.zeros_like(cov_rowspace_proj_to_nullspace),
decimal=3)
# inner-product of each of cov_nullspace to cov_rowspace shall be identity:
inner_product_cov_rowspace = np.transpose(
cov_rowspace, axes=(0, 2, 1)) @ cov_rowspace
npt.assert_almost_equal(
inner_product_cov_rowspace,
np.tile(np.eye(dim_tangent_space),
(inner_product_cov_rowspace.shape[0], 1, 1)),
decimal=3)
inner_product_cov_nullspace = np.transpose(
cov_nullspace, axes=(0, 2, 1)) @ cov_nullspace
npt.assert_almost_equal(
inner_product_cov_nullspace,
np.tile(np.eye(dim_normal_space),
(inner_product_cov_nullspace.shape[0], 1, 1)),
decimal=3)
if is_performing_data_augmentation:
for key, value in dataset_dict.items():
create_dir_if_not_exist('../data/augmented/')
np.save('../data/augmented/' + key, value)
self.dataset = GeneralDataset(dataset_dict)
self.dim_ambient = dim_ambient
self.dim_tangent_space = dim_tangent_space
self.dim_normal_space = dim_normal_space
def J(self, x):
x_torch = utils.convert_into_at_least_2d_pytorch_tensor(
x).requires_grad_().to(self.device)
J_torch = torch.squeeze(self.J_torch(x_torch), dim=0)
return J_torch.cpu().detach().numpy()
def y(self, x):
x_torch = utils.convert_into_at_least_2d_pytorch_tensor(
x).requires_grad_().to(self.device)
y_torch = self.y_torch(x_torch)
return y_torch.cpu().detach().numpy()
def get_loss_components(self, data_dict):
loss_components = dict()
N_data = data_dict['data'].shape[0]
if ('siam_reflection_data' in data_dict) and ('siam_same_levelvec_data'
in data_dict):
is_computing_signed_siamese_pairs = True
else:
is_computing_signed_siamese_pairs = False
data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['data']).requires_grad_().to(self.device)
norm_level_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['norm_level_data']).requires_grad_().to(self.device)
norm_level_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['norm_level_weight']).requires_grad_().to(self.device)
[y_torch, J_torch] = self.y_torch_and_J_torch(data_torch)
# (level set) prediction error
norm_level_wnmse_per_dim = utils.compute_wnmse_per_dim(
prediction=torch.norm(y_torch, dim=1).unsqueeze(1),
ground_truth=norm_level_data_torch,
weight=norm_level_weight_torch)
if is_computing_signed_siamese_pairs:
siam_reflection_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_reflection_data']).requires_grad_().to(
self.device)
siam_reflection_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_reflection_weight']).requires_grad_().to(
self.device)
siam_same_levelvec_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_same_levelvec_data']).requires_grad_().to(
self.device)
siam_same_levelvec_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_same_levelvec_weight']).requires_grad_().to(
self.device)
augmenting_vector_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['augmenting_vector']).requires_grad_().to(
self.device)
siam_frac_aug_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_frac_aug_weight']).requires_grad_().to(
self.device)
siam_reflection_y_torch = self.y_torch(siam_reflection_data_torch)
siam_reflection_wnmse_torch = utils.compute_wnmse_per_dim(
prediction=(-siam_reflection_y_torch),
ground_truth=y_torch,
weight=siam_reflection_weight_torch)
N_siam_same_levelvec = siam_same_levelvec_data_torch.shape[1]
siam_same_levelvec_y_torch_list = [
self.y_torch(
siam_same_levelvec_data_torch[:, n_siam_same_levelvec, :])
for n_siam_same_levelvec in range(N_siam_same_levelvec)
]
siam_same_levelvec_wnmse_torch = torch.stack([
utils.compute_wnmse_per_dim(
prediction=siam_same_levelvec_y_torch_list[
n_siam_same_levelvec],
ground_truth=y_torch,
weight=siam_same_levelvec_weight_torch[:,
n_siam_same_levelvec]
).squeeze()
for n_siam_same_levelvec in range(N_siam_same_levelvec)
],
dim=0).mean(axis=0)
N_siam_frac_augs = 4
normalized_y_torch = utils.normalize(y_torch, data_axis=1)
siam_frac_aug_normalized_y_torch_list = [
utils.normalize(self.y_torch(data_torch - (
((1.0 * n_siam_frac_augs) / N_siam_frac_augs) *
augmenting_vector_torch)),
data_axis=1)
for n_siam_frac_augs in range(1, N_siam_frac_augs)
]
siam_frac_aug_wnmse_torch = torch.stack([
utils.compute_wnmse_per_dim(
prediction=siam_frac_aug_normalized_y_torch_list[
n_siam_frac_augs],
ground_truth=normalized_y_torch,
weight=siam_frac_aug_weight_torch).squeeze()
for n_siam_frac_augs in range(N_siam_frac_augs - 1)
],
dim=0).mean(axis=0)
# Local PCA's Rowspace and Nullspace Eigenvectors extraction
# (originally from the V Matrix of the Local PCA's Covariance Matrix's SVD (cov_svd_V)):
cov_torch_rowspace = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['cov_rowspace']).to(self.device)
assert (cov_torch_rowspace.shape[2] == (self.dim_ambient -
self.N_constraints))
cov_torch_nullspace = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['cov_nullspace']).to(self.device)
assert (cov_torch_nullspace.shape[2] == self.N_constraints)
cov_torch_rowspace_projector = (
cov_torch_rowspace @ cov_torch_rowspace.transpose(-2, -1))
cov_torch_nullspace_projector = (
cov_torch_nullspace @ cov_torch_nullspace.transpose(-2, -1))
# Constraint Manifold Neural Network's Jacobian
# Rowspace and Nullspace eigenvectors extraction:
[_, _, J_torch_svd_V] = torch.svd(J_torch, some=False)
J_torch_rowspace = J_torch_svd_V[:, :, :self.N_constraints]
J_torch_nullspace = J_torch_svd_V[:, :, self.N_constraints:]
J_torch_rowspace_projector = (
J_torch_rowspace @ J_torch_rowspace.transpose(-2, -1))
J_torch_nullspace_projector = (
J_torch_nullspace @ J_torch_nullspace.transpose(-2, -1))
# we want to align so that J_torch_nullspace == cov_torch_rowspace,
# so here is the projection loss (I - A^{+}A)b_i whose norm is to be minimized during training:
J_nspace_proj_error_per_dim = cov_torch_nullspace_projector @ J_torch_nullspace
# we want to align so that cov_torch_nullspace == J_torch_rowspace,
# so here is the projection loss (I - B^{+}B)a_j whose norm is to be minimized during training:
cov_nspace_proj_error_per_dim = J_torch_nullspace_projector @ cov_torch_nullspace
# similarly we want to align so that J_torch_rowspace == cov_torch_nullspace:
J_rspace_proj_error_per_dim = cov_torch_rowspace_projector @ J_torch_rowspace
# similarly we want to align so that cov_torch_rowspace == J_torch_nullspace:
cov_rspace_proj_error_per_dim = J_torch_rowspace_projector @ cov_torch_rowspace
# Local Tangent Space Alignment (LTSA) and Local PCA (eigenvectors) alignment errors:
J_nspace_proj_loss_per_dim = (J_nspace_proj_error_per_dim**2).mean(
axis=0)
cov_nspace_proj_loss_per_dim = (cov_nspace_proj_error_per_dim**2).mean(
axis=0)
J_rspace_proj_loss_per_dim = (J_rspace_proj_error_per_dim**2).mean(
axis=0)
cov_rspace_proj_loss_per_dim = (cov_rspace_proj_error_per_dim**2).mean(
axis=0)
loss_components['norm_level_wnmse_per_dim'] = norm_level_wnmse_per_dim
if is_computing_signed_siamese_pairs:
loss_components[
'siam_reflection_wnmse_torch'] = siam_reflection_wnmse_torch
loss_components[
'siam_same_levelvec_wnmse_torch'] = siam_same_levelvec_wnmse_torch
loss_components[
'siam_frac_aug_wnmse_torch'] = siam_frac_aug_wnmse_torch
loss_components[
'J_nspace_proj_loss_per_dim'] = J_nspace_proj_loss_per_dim
loss_components[
'cov_nspace_proj_loss_per_dim'] = cov_nspace_proj_loss_per_dim
loss_components[
'J_rspace_proj_loss_per_dim'] = J_rspace_proj_loss_per_dim
loss_components[
'cov_rspace_proj_loss_per_dim'] = cov_rspace_proj_loss_per_dim
return loss_components
x_torch = utils.convert_into_at_least_2d_pytorch_tensor(
x).requires_grad_().to(self.device)
J_torch = self.J_torch(x_torch)
return J_torch.cpu().detach().numpy()
if __name__ == "__main__":
rand_seed = 47
np.random.seed(rand_seed)
torch.random.manual_seed(rand_seed)
dfeat = DummyDifferentiableFeature()
x = np.random.rand(2, 5)
print("x = ")
print(x)
J = dfeat.J(x)
print("J = ")
print(J)
J_torch = utils.convert_into_at_least_2d_pytorch_tensor(
J).requires_grad_().to('cpu')
[J_svd_U_torch, J_svd_s_torch, J_svd_V_torch] = torch.svd(J_torch,
some=True)
print("J_svd_U_torch = ", J_svd_U_torch)
print("J_svd_s_torch = ", J_svd_s_torch)
print("J_svd_V_torch = ", J_svd_V_torch)
# print("J_svd_U_torch @ J_svd_U_torch.transpose() = ", J_svd_U_torch @ J_svd_U_torch.transpose(-2, -1))
# print("J_svd_U_torch.transpose() @ J_svd_U_torch = ", J_svd_U_torch.transpose(-2, -1) @ J_svd_U_torch)
# print("J_svd_V_torch @ J_svd_V_torch.transpose() = ", J_svd_V_torch @ J_svd_V_torch.transpose(-2, -1))
# print("J_svd_V_torch.transpose() @ J_svd_V_torch = ", J_svd_V_torch.transpose(-2, -1) @ J_svd_V_torch)
# print("det(J_svd_U_torch) = ", torch.det(J_svd_U_torch))