def y_and_J(self, x): x_torch = utils.convert_into_at_least_2d_pytorch_tensor( x).requires_grad_().to(self.device) with torch.enable_grad(): y_torch = self.y_torch(x_torch) J_torch = torch.stack([ torch.autograd.grad(y_torch[:, d].sum(), x_torch, retain_graph=True, create_graph=self.is_training)[0] for d in range(y_torch.shape[1]) ], dim=1) return y_torch.cpu().detach().numpy(), J_torch.cpu().detach().numpy()
test_successes = np.zeros((n_iter, len(runs))) for run_idx, run in enumerate(runs): result_path = '../plot/ecmnn/' + task + '/r{:02d}/'.format(run) aug_dataloader_input_file = result_path + 'aug_dataloader_' + task + '.pkl' with open(aug_dataloader_input_file, 'rb') as aug_dataloader: data = pickle.load(aug_dataloader) [ batch_train_loader, _, _, all_train_dataset, all_valid_dataset, all_test_dataset, all_dataset ] = data.get_train_valid_test_dataloaders(batch_size=128, train_fraction=0.85, valid_fraction=0.0) train_inputs = convert_into_at_least_2d_pytorch_tensor( all_train_dataset['data']) test_inputs = convert_into_at_least_2d_pytorch_tensor( all_test_dataset['data']) train_norm_level = all_train_dataset['norm_level_data'].flatten() train_inputs = train_inputs[train_norm_level == 0.0] train_inputs = train_inputs[:n_data, :].detach().numpy() test_norm_level = all_test_dataset['norm_level_data'].flatten() test_inputs = test_inputs[test_norm_level == 0.0] test_inputs = test_inputs[:n_data, :].detach().numpy() q_max = np.max(train_inputs, axis=0) q_min = np.min(train_inputs, axis=0) for iter in range(n_iter):
def __init__(self, dataset_filepath, *args, **kwargs): rand_seed = kwargs.get('rand_seed', 38) is_performing_data_augmentation = kwargs.get( 'is_performing_data_augmentation', True) is_computing_all_cost_components = kwargs.get( 'is_computing_all_cost_components', True) is_using_level_dependent_weight = kwargs.get( 'is_using_level_dependent_weight', False) N_normal_space_traversal = kwargs.get('N_normal_space_traversal', 9) is_optimizing_signed_siamese_pairs = kwargs.get( 'is_optimizing_signed_siamese_pairs', True) clean_aug_data = kwargs.get('clean_aug_data', True) aug_clean_thresh = kwargs.get('aug_clean_thresh', 1e-1) is_aligning_lpca_normal_space_eigvecs = kwargs.get( 'is_aligning_lpca_normal_space_eigvecs', True) N_normal_space_eigvecs_alignment_repetition = kwargs.get( 'N_normal_space_eigvecs_alignment_repetition', 1) is_augmenting_w_rand_comb_of_normaleigvecs = kwargs.get( 'is_augmenting_w_rand_comb_of_normaleigvecs', True) N_siam_same_levelvec = kwargs.get('N_siam_same_levelvec', 5) N_local_neighborhood_mult = kwargs.get('N_local_neighborhood_mult', 1) if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components: onmanif_siam_same_levelvecs_list = list() else: is_aligning_lpca_normal_space_eigvecs = False dataset_dict = {} dataset_dict['data'] = np.load(dataset_filepath + '.npy') N_data = dataset_dict['data'].shape[0] dim_ambient = dataset_dict['data'].shape[1] print('N_data = %d' % N_data) dataset_dict['norm_level_data'] = np.zeros((N_data, 1)) dataset_dict['norm_level_weight'] = np.ones((N_data, 1)) dataset_dict['augmenting_vector'] = np.zeros_like(dataset_dict['data']) # N_local_neighborhood = int(round(N_data/1000)) N_local_neighborhood = N_local_neighborhood_mult * 2 * (2**dim_ambient) print('N_local_neighborhood = %d (%f percent)' % (N_local_neighborhood, (100.0 * N_local_neighborhood) / N_data)) start_time = time.clock() S_list = [] kd_tree = cKDTree(data=dataset_dict['data']) for i in tqdm.tqdm(range(N_data)): # compute nearest neighbors of size N_local_neighborhood (exclude self): [_, nearneigh_indices] = kd_tree.query(dataset_dict['data'][i], k=N_local_neighborhood + 1) nearneigh_indices = nearneigh_indices[1:] vecs_from_curr_pt_to_nearneighs = ( dataset_dict['data'][nearneigh_indices] - dataset_dict['data'][i]) assert (vecs_from_curr_pt_to_nearneighs.shape[0] == N_local_neighborhood) if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components: # pick N_siam_same_levelvec random (on-manifold) configurations (exclude self/i-th config) # which all supposed to have the same implicit vector-value/level vector/ecmnn output vector # as the current/i-th configuration (i.e. zero vector): all_idx_but_self = list(range(N_data)) all_idx_but_self.pop(i) onmanif_siam_same_levelvec_indices = np.random.permutation( all_idx_but_self)[:N_siam_same_levelvec] onmanif_siam_same_levelvecs = dataset_dict['data'][ onmanif_siam_same_levelvec_indices, :] onmanif_siam_same_levelvecs_list.append( onmanif_siam_same_levelvecs) # compute the sample covariance matrix of the nearest neighbors (for Local PCA): X = vecs_from_curr_pt_to_nearneighs XTX = X.T @ X S = XTX / (N_local_neighborhood - 1) S_list.append(S) dataset_dict['cov'] = np.stack(S_list) print("Local PCA Covariance Matrices are computed in %f seconds." % (time.clock() - start_time)) if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components: dataset_dict['siam_reflection_data'] = copy.deepcopy( dataset_dict['data']) dataset_dict['siam_reflection_weight'] = np.ones((N_data, 1)) dataset_dict['siam_same_levelvec_data'] = np.stack( onmanif_siam_same_levelvecs_list) # (on-manifold) siamese same-level-vectors loss' weight is one: dataset_dict['siam_same_levelvec_weight'] = np.ones( (N_data, N_siam_same_levelvec)) # while the on-manifold siamese augmentation fraction loss' weight is zero # (because the ecmnn output vector/implicit vector-value/level vector # is expected to be a zero vector here, hence it CANNOT be normalized into a unit vector, # and technically there is NO distinctive siamese augmentation fraction pair of it): dataset_dict['siam_frac_aug_weight'] = np.zeros((N_data, 1)) del onmanif_siam_same_levelvecs_list # pre-compute the SVD of the local sample covariance matrices, i.e. performing Local PCA: cov_torch = convert_into_at_least_2d_pytorch_tensor( dataset_dict['cov']) [_, cov_torch_svd_s, cov_torch_svd_V] = torch.svd(cov_torch, some=False) dataset_dict['cov_svd_s'] = cov_torch_svd_s.cpu().detach().numpy() dataset_dict['cov_svd_V'] = cov_torch_svd_V.cpu().detach().numpy() # determine the intrinsic dimensionality of the Constraint Manifold, # based on the index of the maximum drop of Eigenvalue # (idea from Shay Deutsch's PhD Dissertation manuscript): diff_eigval = dataset_dict['cov_svd_s'][:, :( dim_ambient - 1)] - dataset_dict['cov_svd_s'][:, 1:] argmax_diff_eigval = np.argmax(diff_eigval, axis=1) [dim_tangent_space, _] = stats.mode(argmax_diff_eigval, axis=None) dim_tangent_space = dim_tangent_space[0] + 1 dim_normal_space = dim_ambient - dim_tangent_space print("Tangent Space Dimensionality = %d" % dim_tangent_space) print("Normal Space Dimensionality = %d" % dim_normal_space) assert (dim_normal_space >= 1) N_aug_rand_normal_space_vecs = (4**dim_normal_space) # some Eigenvalue statistics of the Local PCA: max_eigval = np.max(dataset_dict['cov_svd_s']) min_eigval = np.min(dataset_dict['cov_svd_s']) mean_eigval = np.mean(dataset_dict['cov_svd_s']) max_tangent_eigval = np.max( dataset_dict['cov_svd_s'][:, :dim_tangent_space]) min_tangent_eigval = np.min( dataset_dict['cov_svd_s'][:, :dim_tangent_space]) mean_tangent_eigval = np.mean( dataset_dict['cov_svd_s'][:, :dim_tangent_space]) epsilon = N_local_neighborhood_mult * np.sqrt(mean_tangent_eigval) print("Maximum Eigenvalue = %f" % max_eigval) print("Minimum Eigenvalue = %f" % min_eigval) print("Mean Eigenvalue = %f" % mean_eigval) print("Maximum Tangent Space Eigenvalue = %f" % max_tangent_eigval) print("Minimum Tangent Space Eigenvalue = %f" % min_tangent_eigval) print("Mean Tangent Space Eigenvalue = %f" % mean_tangent_eigval) print("Epsilon = %f" % epsilon) # extract Tangent Space Eigenvectors and Normal Space Eigenvectors: dataset_dict['cov_rowspace'] = dataset_dict[ 'cov_svd_V'][:, :, :dim_tangent_space] dataset_dict['cov_nullspace'] = dataset_dict[ 'cov_svd_V'][:, :, dim_tangent_space:] tangent_space_eigvecs = dataset_dict['cov_rowspace'] if is_aligning_lpca_normal_space_eigvecs: for n_normal_space_eigvecs_alignment_repetition in range( N_normal_space_eigvecs_alignment_repetition): print('Performing Normal Space Eigenvector Bases iter %d/%d' % (n_normal_space_eigvecs_alignment_repetition + 1, N_normal_space_eigvecs_alignment_repetition)) normal_space_eigvecs = align_normal_space_eigvecs( dataset_dict, rand_seed=rand_seed) dataset_dict['cov_nullspace'] = copy.deepcopy( normal_space_eigvecs) else: normal_space_eigvecs = dataset_dict['cov_nullspace'] if is_performing_data_augmentation: # append more points based on the Normal Space Eigenvectors and the computed epsilon: dataset_list_dict = { key: [dataset_dict[key]] for key in dataset_dict } for i in range(1, N_normal_space_traversal + 1): print('Data Augmentation %d/%d' % (i, N_normal_space_traversal)) unsigned_level_mult = epsilon * i level_dependent_weight = 1.0 if is_using_level_dependent_weight: level_dependent_weight /= (unsigned_level_mult**2) dset_n_nspacetrv_list_dict = { key: list() for key in dataset_dict } if is_augmenting_w_rand_comb_of_normaleigvecs: if dim_normal_space > 1: aug_modes = ['deterministic', 'randomized'] else: aug_modes = ['deterministic'] else: aug_modes = ['deterministic'] level_mult_eigvec_list = list() for aug_mode in aug_modes: if aug_mode == 'deterministic': for d in range(dim_normal_space): for sign in [-1, 1]: level_mult_eigvec = sign * unsigned_level_mult * normal_space_eigvecs[:, :, d] level_mult_eigvec_list.append( level_mult_eigvec) elif aug_mode == 'randomized': # randomized combination of the normal space eigenvectors: for _ in range(N_aug_rand_normal_space_vecs): rand_combinator = np.tile( np.random.normal(size=(1, dim_normal_space, 1)), (N_data, 1, 1)) rand_normal_space_vec = np.squeeze( normal_space_eigvecs @ rand_combinator) rand_normal_space_unitvec = ( rand_normal_space_vec / np.expand_dims( npla.norm(rand_normal_space_vec, axis=1), axis=1)) level_mult_eigvec = unsigned_level_mult * rand_normal_space_unitvec level_mult_eigvec_list.append(level_mult_eigvec) for level_mult_eigvec in level_mult_eigvec_list: new_data = dataset_dict['data'] + level_mult_eigvec # delete indices from augmented data if they do not fulfill the neighborhood condition valid_idx = list(range(N_data)) if clean_aug_data: del_idx = [] for idx, x in enumerate(new_data): [_, idx_near] = kd_tree.query(x) if (npla.norm(new_data[idx_near] - x) > (aug_clean_thresh * unsigned_level_mult)): del_idx += [idx] [valid_idx.remove(idx) for idx in del_idx] N_aug = len(valid_idx) print('Accepted aug points ', N_aug, ' / ', N_data) if N_aug == 0: continue valid_data = new_data[valid_idx] dset_n_nspacetrv_list_dict['data'].append(valid_data) dset_n_nspacetrv_list_dict['augmenting_vector'].append( level_mult_eigvec[valid_idx]) if is_optimizing_signed_siamese_pairs or is_computing_all_cost_components: new_siam_reflection_data = dataset_dict[ 'data'] - level_mult_eigvec dset_n_nspacetrv_list_dict[ 'siam_reflection_data'].append( new_siam_reflection_data[valid_idx]) new_siam_reflection_weight = np.ones( (N_data, 1)) * level_dependent_weight for val_idx in valid_idx: x = new_siam_reflection_data[val_idx] [_, idx_near] = kd_tree.query(x) if (npla.norm(new_siam_reflection_data[idx_near] - x) > (aug_clean_thresh * unsigned_level_mult)): new_siam_reflection_weight[ val_idx, 0] = 0.0 # siam_reflection_loss will NOT be imposed dset_n_nspacetrv_list_dict[ 'siam_reflection_weight'].append( new_siam_reflection_weight[valid_idx]) N_valid_siam_reflection_pairs = int( round( np.sum( new_siam_reflection_weight[valid_idx] > 0.0 ))) print('Valid siamese reflection pairs ', N_valid_siam_reflection_pairs, ' / ', len(valid_idx)) dset_n_nspacetrv_list_dict[ 'siam_frac_aug_weight'].append( np.ones((N_aug, 1)) * level_dependent_weight) siam_same_levelvecs_list = list() siam_same_levelvecs_weight_list = list() for j in range(N_aug): # pick N_siam_same_levelvec random (augmented) configurations (exclude self/j-th config) # which all supposed to have the same implicit vector-value/level vector/ecmnn output vector # as the current/j-th configuration: all_idx_but_self = list(range(N_aug)) all_idx_but_self.pop(j) siam_same_levelvec_indices = np.random.permutation( all_idx_but_self)[:N_siam_same_levelvec] # siam_same_levelvec_indices = np.array([(int(round(j + # ((N_aug/(N_siam_same_levelvec+1.0))*ns))) # % N_aug) # for ns in range(1, N_siam_same_levelvec+1)]) siam_same_levelvecs = np.zeros( (N_siam_same_levelvec, dim_ambient)) siam_same_levelvecs_weight = np.zeros( N_siam_same_levelvec) N_siam_same_levelvec_indices = siam_same_levelvec_indices.shape[ 0] if N_siam_same_levelvec_indices > 0: siam_same_levelvecs[:N_siam_same_levelvec_indices, :] = valid_data[ siam_same_levelvec_indices, :] siam_same_levelvecs_weight[:N_siam_same_levelvec_indices] = np.ones( N_siam_same_levelvec_indices) siam_same_levelvecs_list.append( siam_same_levelvecs) siam_same_levelvecs_weight_list.append( siam_same_levelvecs_weight) dset_n_nspacetrv_list_dict[ 'siam_same_levelvec_data'].append( np.stack(siam_same_levelvecs_list)) dset_n_nspacetrv_list_dict[ 'siam_same_levelvec_weight'].append( np.stack(siam_same_levelvecs_weight_list) * level_dependent_weight) dset_n_nspacetrv_list_dict['norm_level_data'].append( unsigned_level_mult * np.ones((N_aug, 1))) dset_n_nspacetrv_list_dict['norm_level_weight'].append( np.ones((N_aug, 1)) * level_dependent_weight) dset_n_nspacetrv_list_dict['cov'].append( copy.deepcopy(dataset_dict['cov'][valid_idx])) dset_n_nspacetrv_list_dict['cov_svd_s'].append( copy.deepcopy(dataset_dict['cov_svd_s'][valid_idx])) dset_n_nspacetrv_list_dict['cov_svd_V'].append( copy.deepcopy(dataset_dict['cov_svd_V'][valid_idx])) dset_n_nspacetrv_list_dict['cov_rowspace'].append( copy.deepcopy(dataset_dict['cov_rowspace'][valid_idx])) dset_n_nspacetrv_list_dict['cov_nullspace'].append( copy.deepcopy( dataset_dict['cov_nullspace'][valid_idx])) dset_n_nspacetrv_dict = { key: np.concatenate(dset_n_nspacetrv_list_dict[key], axis=0) for key in dset_n_nspacetrv_list_dict } for key in dataset_dict: dataset_list_dict[key].append(dset_n_nspacetrv_dict[key]) for key in dataset_dict: dataset_dict[key] = np.concatenate(dataset_list_dict[key], axis=0) # some checks to ensure validity of cov_rowspace and cov_nullspace: cov_rowspace = dataset_dict['cov_rowspace'] cov_nullspace = dataset_dict['cov_nullspace'] cov_rowspace_projector = cov_rowspace @ np.transpose(cov_rowspace, axes=(0, 2, 1)) cov_nullspace_projector = cov_nullspace @ np.transpose(cov_nullspace, axes=(0, 2, 1)) # projection from cov_nullspace to cov_rowspace and the otherwise shall be zero cov_nullspace_proj_to_rowspace = ( cov_rowspace_projector @ cov_nullspace) npt.assert_almost_equal(cov_nullspace_proj_to_rowspace, np.zeros_like(cov_nullspace_proj_to_rowspace), decimal=3) cov_rowspace_proj_to_nullspace = ( cov_nullspace_projector @ cov_rowspace) npt.assert_almost_equal(cov_rowspace_proj_to_nullspace, np.zeros_like(cov_rowspace_proj_to_nullspace), decimal=3) # inner-product of each of cov_nullspace to cov_rowspace shall be identity: inner_product_cov_rowspace = np.transpose( cov_rowspace, axes=(0, 2, 1)) @ cov_rowspace npt.assert_almost_equal( inner_product_cov_rowspace, np.tile(np.eye(dim_tangent_space), (inner_product_cov_rowspace.shape[0], 1, 1)), decimal=3) inner_product_cov_nullspace = np.transpose( cov_nullspace, axes=(0, 2, 1)) @ cov_nullspace npt.assert_almost_equal( inner_product_cov_nullspace, np.tile(np.eye(dim_normal_space), (inner_product_cov_nullspace.shape[0], 1, 1)), decimal=3) if is_performing_data_augmentation: for key, value in dataset_dict.items(): create_dir_if_not_exist('../data/augmented/') np.save('../data/augmented/' + key, value) self.dataset = GeneralDataset(dataset_dict) self.dim_ambient = dim_ambient self.dim_tangent_space = dim_tangent_space self.dim_normal_space = dim_normal_space
def J(self, x): x_torch = utils.convert_into_at_least_2d_pytorch_tensor( x).requires_grad_().to(self.device) J_torch = torch.squeeze(self.J_torch(x_torch), dim=0) return J_torch.cpu().detach().numpy()
def y(self, x): x_torch = utils.convert_into_at_least_2d_pytorch_tensor( x).requires_grad_().to(self.device) y_torch = self.y_torch(x_torch) return y_torch.cpu().detach().numpy()
def get_loss_components(self, data_dict): loss_components = dict() N_data = data_dict['data'].shape[0] if ('siam_reflection_data' in data_dict) and ('siam_same_levelvec_data' in data_dict): is_computing_signed_siamese_pairs = True else: is_computing_signed_siamese_pairs = False data_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['data']).requires_grad_().to(self.device) norm_level_data_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['norm_level_data']).requires_grad_().to(self.device) norm_level_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['norm_level_weight']).requires_grad_().to(self.device) [y_torch, J_torch] = self.y_torch_and_J_torch(data_torch) # (level set) prediction error norm_level_wnmse_per_dim = utils.compute_wnmse_per_dim( prediction=torch.norm(y_torch, dim=1).unsqueeze(1), ground_truth=norm_level_data_torch, weight=norm_level_weight_torch) if is_computing_signed_siamese_pairs: siam_reflection_data_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['siam_reflection_data']).requires_grad_().to( self.device) siam_reflection_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['siam_reflection_weight']).requires_grad_().to( self.device) siam_same_levelvec_data_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['siam_same_levelvec_data']).requires_grad_().to( self.device) siam_same_levelvec_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['siam_same_levelvec_weight']).requires_grad_().to( self.device) augmenting_vector_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['augmenting_vector']).requires_grad_().to( self.device) siam_frac_aug_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['siam_frac_aug_weight']).requires_grad_().to( self.device) siam_reflection_y_torch = self.y_torch(siam_reflection_data_torch) siam_reflection_wnmse_torch = utils.compute_wnmse_per_dim( prediction=(-siam_reflection_y_torch), ground_truth=y_torch, weight=siam_reflection_weight_torch) N_siam_same_levelvec = siam_same_levelvec_data_torch.shape[1] siam_same_levelvec_y_torch_list = [ self.y_torch( siam_same_levelvec_data_torch[:, n_siam_same_levelvec, :]) for n_siam_same_levelvec in range(N_siam_same_levelvec) ] siam_same_levelvec_wnmse_torch = torch.stack([ utils.compute_wnmse_per_dim( prediction=siam_same_levelvec_y_torch_list[ n_siam_same_levelvec], ground_truth=y_torch, weight=siam_same_levelvec_weight_torch[:, n_siam_same_levelvec] ).squeeze() for n_siam_same_levelvec in range(N_siam_same_levelvec) ], dim=0).mean(axis=0) N_siam_frac_augs = 4 normalized_y_torch = utils.normalize(y_torch, data_axis=1) siam_frac_aug_normalized_y_torch_list = [ utils.normalize(self.y_torch(data_torch - ( ((1.0 * n_siam_frac_augs) / N_siam_frac_augs) * augmenting_vector_torch)), data_axis=1) for n_siam_frac_augs in range(1, N_siam_frac_augs) ] siam_frac_aug_wnmse_torch = torch.stack([ utils.compute_wnmse_per_dim( prediction=siam_frac_aug_normalized_y_torch_list[ n_siam_frac_augs], ground_truth=normalized_y_torch, weight=siam_frac_aug_weight_torch).squeeze() for n_siam_frac_augs in range(N_siam_frac_augs - 1) ], dim=0).mean(axis=0) # Local PCA's Rowspace and Nullspace Eigenvectors extraction # (originally from the V Matrix of the Local PCA's Covariance Matrix's SVD (cov_svd_V)): cov_torch_rowspace = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['cov_rowspace']).to(self.device) assert (cov_torch_rowspace.shape[2] == (self.dim_ambient - self.N_constraints)) cov_torch_nullspace = utils.convert_into_at_least_2d_pytorch_tensor( data_dict['cov_nullspace']).to(self.device) assert (cov_torch_nullspace.shape[2] == self.N_constraints) cov_torch_rowspace_projector = ( cov_torch_rowspace @ cov_torch_rowspace.transpose(-2, -1)) cov_torch_nullspace_projector = ( cov_torch_nullspace @ cov_torch_nullspace.transpose(-2, -1)) # Constraint Manifold Neural Network's Jacobian # Rowspace and Nullspace eigenvectors extraction: [_, _, J_torch_svd_V] = torch.svd(J_torch, some=False) J_torch_rowspace = J_torch_svd_V[:, :, :self.N_constraints] J_torch_nullspace = J_torch_svd_V[:, :, self.N_constraints:] J_torch_rowspace_projector = ( J_torch_rowspace @ J_torch_rowspace.transpose(-2, -1)) J_torch_nullspace_projector = ( J_torch_nullspace @ J_torch_nullspace.transpose(-2, -1)) # we want to align so that J_torch_nullspace == cov_torch_rowspace, # so here is the projection loss (I - A^{+}A)b_i whose norm is to be minimized during training: J_nspace_proj_error_per_dim = cov_torch_nullspace_projector @ J_torch_nullspace # we want to align so that cov_torch_nullspace == J_torch_rowspace, # so here is the projection loss (I - B^{+}B)a_j whose norm is to be minimized during training: cov_nspace_proj_error_per_dim = J_torch_nullspace_projector @ cov_torch_nullspace # similarly we want to align so that J_torch_rowspace == cov_torch_nullspace: J_rspace_proj_error_per_dim = cov_torch_rowspace_projector @ J_torch_rowspace # similarly we want to align so that cov_torch_rowspace == J_torch_nullspace: cov_rspace_proj_error_per_dim = J_torch_rowspace_projector @ cov_torch_rowspace # Local Tangent Space Alignment (LTSA) and Local PCA (eigenvectors) alignment errors: J_nspace_proj_loss_per_dim = (J_nspace_proj_error_per_dim**2).mean( axis=0) cov_nspace_proj_loss_per_dim = (cov_nspace_proj_error_per_dim**2).mean( axis=0) J_rspace_proj_loss_per_dim = (J_rspace_proj_error_per_dim**2).mean( axis=0) cov_rspace_proj_loss_per_dim = (cov_rspace_proj_error_per_dim**2).mean( axis=0) loss_components['norm_level_wnmse_per_dim'] = norm_level_wnmse_per_dim if is_computing_signed_siamese_pairs: loss_components[ 'siam_reflection_wnmse_torch'] = siam_reflection_wnmse_torch loss_components[ 'siam_same_levelvec_wnmse_torch'] = siam_same_levelvec_wnmse_torch loss_components[ 'siam_frac_aug_wnmse_torch'] = siam_frac_aug_wnmse_torch loss_components[ 'J_nspace_proj_loss_per_dim'] = J_nspace_proj_loss_per_dim loss_components[ 'cov_nspace_proj_loss_per_dim'] = cov_nspace_proj_loss_per_dim loss_components[ 'J_rspace_proj_loss_per_dim'] = J_rspace_proj_loss_per_dim loss_components[ 'cov_rspace_proj_loss_per_dim'] = cov_rspace_proj_loss_per_dim return loss_components
x_torch = utils.convert_into_at_least_2d_pytorch_tensor( x).requires_grad_().to(self.device) J_torch = self.J_torch(x_torch) return J_torch.cpu().detach().numpy() if __name__ == "__main__": rand_seed = 47 np.random.seed(rand_seed) torch.random.manual_seed(rand_seed) dfeat = DummyDifferentiableFeature() x = np.random.rand(2, 5) print("x = ") print(x) J = dfeat.J(x) print("J = ") print(J) J_torch = utils.convert_into_at_least_2d_pytorch_tensor( J).requires_grad_().to('cpu') [J_svd_U_torch, J_svd_s_torch, J_svd_V_torch] = torch.svd(J_torch, some=True) print("J_svd_U_torch = ", J_svd_U_torch) print("J_svd_s_torch = ", J_svd_s_torch) print("J_svd_V_torch = ", J_svd_V_torch) # print("J_svd_U_torch @ J_svd_U_torch.transpose() = ", J_svd_U_torch @ J_svd_U_torch.transpose(-2, -1)) # print("J_svd_U_torch.transpose() @ J_svd_U_torch = ", J_svd_U_torch.transpose(-2, -1) @ J_svd_U_torch) # print("J_svd_V_torch @ J_svd_V_torch.transpose() = ", J_svd_V_torch @ J_svd_V_torch.transpose(-2, -1)) # print("J_svd_V_torch.transpose() @ J_svd_V_torch = ", J_svd_V_torch.transpose(-2, -1) @ J_svd_V_torch) # print("det(J_svd_U_torch) = ", torch.det(J_svd_U_torch))