class IPCA: def __init__(self, num_components: int = 150): self.num_components = num_components self.inc_pca = IncrementalPCA(n_components=self.num_components) def fit(self, x, batch_size: int = 32): input_len = len(x) iterator = range(0, input_len, batch_size) iterator = tqdm(iterator, desc="Processing PCA batches") for batch_index in iterator: batch_start = batch_index batch_end = min(batch_start + batch_size, input_len) #print('batch size : ', batch_end-batch_start) try: self.inc_pca.partial_fit(x[batch_start:batch_end]) except: # batch_size < n_components break #batches = np.array_split(x, batch_size) #for batch in batches: # print('batch size : ', len(batch)) # self.inc_pca.partial_fit(batch) def fit_transform(self, x, batch_size: int = 32, with_loss: bool = False): self.fit(x, batch_size) x_pca = self.transform(x) if with_loss: x_reconstructed = self.reconstruct(x_pca) loss = self.compute_loss(x, x_reconstructed) return x_pca, x_reconstructed, loss return x_pca # return "loadings" for vector x def transform(self, x): x_inc_pca = self.inc_pca.transform(x) return x_inc_pca def reconstruct(self, x_pca): x = self.inc_pca.inverse_transform(x_pca) return x # return the projected vector with reducted dimensionality def project(x): # first transform, then go back to the initial space x_projected = self.inc_pca.inverse_transform(self.transform(x)) return x_projected def compute_loss(self, x, x_reconstructed): return ((x - x_reconstructed)**2).mean()
def visualize_PCA_recon(dataloader, dataloader_bs1, savedir, net=None, n_components=128): '''saves example reconstruction to save directory''' # setup PCA model model = IncrementalPCA(n_components=n_components) # fit PCA model for sample in dataloader: inputs = sample['image'] batch_size = len(inputs) inputs = inputs.view(batch_size, -1) # flatten images if batch_size >= n_components: model.partial_fit(inputs.numpy()) else: print("batch size %s is smaller than n_components %s" % (batch_size, n_components)) # get projections for idx, sample in enumerate(dataloader_bs1): inputs = sample['image'] inputs = inputs.view(1, -1) # flatten image outputs = model.inverse_transform(model.transform(inputs.numpy())) save_img(inputs, os.path.join(savedir, "input_%s.png" % idx)) save_img(torch.from_numpy(outputs), os.path.join(savedir, "recon_%s.png" % idx)) if idx >= 5: break
class PCA(Model): """Given a set of input vectors, find their principle components""" def __init__(self, fn=None, n_comp=None, batch_size=None): self.model = IncrementalPCA() self.fn = fn self.params = {"n_components": n_comp, "batch_size": batch_size} self.set_params() def load(self, fn): """Set parameters after loading from filename""" super().load(fn) self.params = self.model.get_params() return def fit(self, reps): """Fit a list of representations""" X = [r.to_vector() for r in reps] self.model.fit(X) def err(self, to_transform, to_check_against): """Mesh error between reconstructed to_transform representation and mesh conversion of to_check_against """ vec = to_transform.to_vector() vec_trans = self.model.transform(vec) vec_recon = self.model.inverse_transform(vec_trans) transformed = to_transform.from_vector(vec_recon) mesh1 = transformed.mesh() mesh2 = to_check_against.mesh() error = representation.mesh_error(mesh1, mesh2) return error
class IncrementalEigenfaces: """This implememtation is wrong. the mask should be fixed formo the beginning like in the above.""" def __init__(self, n_components, **pca_kwargs): self.n_samples = 0 self.pca = IncrementalPCA( n_components=n_components, batch_size=None, whiten=False, **pca_kwargs) def add_probes(self, X): """NB: batch_size >= n_components""" self.pca.partial_fit(X) self.n_samples += len(X) def _transform(self, X, mask): """Override pca.transform to project over the unmasked region""" X = X - self.pca.mean_ X_transformed = X[mask] @ self.pca.components_.T[mask, :] return X_transformed def best_probe(self, atoms, mask): shape = atoms.shape atoms = atoms.ravel() mask = mask.ravel() coeffs = self._transform(atoms, mask) best = self.pca.inverse_transform(coeffs) return best.reshape(shape)
def mnist_compression_incremental_pca_scikit(): from sklearn.datasets import fetch_mldata from sklearn.model_selection import train_test_split from sklearn.decomposition import IncrementalPCA mnist = fetch_mldata("MNIST original", data_home="./") X, y = mnist["data"], mnist["target"] train_x, test_x, train_y, test_y = train_test_split(X, y, random_state=42) batch_size = 100 inc_pca = IncrementalPCA(n_components=154) for batch in np.array_split(train_x, batch_size): print(".", end="") inc_pca.partial_fit(batch) reduced_x = inc_pca.transform(train_x) recovered_x = inc_pca.inverse_transform(reduced_x) plt.figure(figsize=(7, 4)) plt.subplot(121) plot_digits(train_x[::2100]) plt.title("Original") plt.subplot(122) plot_digits(recovered_x[::2100]) plt.title("Compression -> Reconstruction") save_fig("Compression_of_MNIST_incremental_pca") plt.show()
def ipca(mov, components = 50, batch =1000): # vectorize the images num_frames, h, w = mov.shape frame_size = h * w frame_samples = np.reshape(mov, (num_frames, frame_size)).T # run IPCA to approxiate the SVD ipca_f = IncrementalPCA(n_components=components, batch_size=batch) ipca_f.fit(frame_samples) # construct the reduced version of the movie vectors using only the # principal component projection proj_frame_vectors = ipca_f.inverse_transform(ipca_f.transform(frame_samples)) # get the temporal principal components (pixel time series) and # associated singular values eigenseries = ipca_f.components_.T # the rows of eigenseries are approximately orthogonal # so we can approximately obtain eigenframes by multiplying the # projected frame matrix by this transpose on the right eigenframes = np.dot(proj_frame_vectors, eigenseries) return eigenseries, eigenframes, proj_frame_vectors
def ipca(image, components, _batch_size=25): """Reconstruct an image from IPCA compression using specific number of components to use and batch size Args: image: PIL Image, Numpy array or path of 3D image components: Number of components used for reconstruction batch_size: Batch size used for learn (default 25) Returns: Reconstructed image Example: >>> from PIL import Image >>> import numpy as np >>> from ipfml.processing import reconstruction >>> image_values = Image.open('./images/test_img.png') >>> reconstructed_image = reconstruction.ipca(image_values, 20) >>> reconstructed_image.shape (200, 200) """ lab_img = transform.get_LAB_L(image) lab_img = np.array(lab_img, 'uint8') transformer = IncrementalPCA(n_components=components, batch_size=_batch_size) transformed_image = transformer.fit_transform(lab_img) restored_image = transformer.inverse_transform(transformed_image) return restored_image
def ipca(mov, components=50, batch=1000): # vectorize the images num_frames, h, w = np.shape(mov) frame_size = h * w frame_samples = np.reshape(mov, (num_frames, frame_size)).T # run IPCA to approxiate the SVD ipca_f = IncrementalPCA(n_components=components, batch_size=batch) ipca_f.fit(frame_samples) # construct the reduced version of the movie vectors using only the # principal component projection proj_frame_vectors = ipca_f.inverse_transform(ipca_f.transform(frame_samples)) # get the temporal principal components (pixel time series) and # associated singular values eigenseries = ipca_f.components_.T # the rows of eigenseries are approximately orthogonal # so we can approximately obtain eigenframes by multiplying the # projected frame matrix by this transpose on the right eigenframes = np.dot(proj_frame_vectors, eigenseries) return eigenseries, eigenframes, proj_frame_vectors
class IPCA(object): def __init__(self, n_components=None, whiten=False, copy=True, batch_size=None): """ :param n_components: default为None ,int 或None, 想要保留的分量数,None 时, min(n_samples, n_features) :param whiten: bool型,可选项, 默认为False, 当true(默认情况下为false)时,components_ 向量除以 n_samples*components_以确保具有单位组件级方差的不相关输出。 :param copy: 默认为True, False时,x 将被覆盖,将节约能存,但存在不安全 :param batch_size: default None, 批量样本数, 只在fit 中使用,设为None,系统自动设成5*n_features, 以保持经度与内存开销的平衡 """ self.model = IncrementalPCA(n_components=n_components, whiten=whiten, copy=copy, batch_size=batch_size) def fit(self, x, y=None): self.model.fit(X=x, y=y) def transform(self, x): return self.model.transform(X=x) def fit_transform(self, x, y=None): return self.model.fit_transform(X=x, y=y) def get_params(self, deep=True): # 获取评估器的参数 return self.model.get_params(deep=deep) def set_params(self, **params): # 设置评估器的参数 self.model.set_params(**params) def inverse_transform(self, x): # 与 fit_tansform 刚好相反的两个操作 return self.model.inverse_transform(X=x) def get_precision(self): # 根据生成模型计算精度矩阵 return self.model.get_precision() def get_covariance(self): # 根据生成模型获取协方差 return self.model.get_covariance() def partial_fit(self, x, y=None, check_input=True): # 增量训练 self.model.partial_fit(X=x, y=y, check_input=check_input) def get_attributes(self): component = self.model.components_ explained_variance = self.model.explained_variance_ explained_variance_ratio = self.model.explained_variance_ratio_ singular_values = self.model.singular_values_ means = self.model.mean_ # 每个特征的均值 var = self.model.var_ # 每个特征的方差 noise_variance = self.model.noise_variance_ # 评估的噪声协方差 n_component = self.model.n_components_ n_samples_seen = self.model.n_samples_seen_ return component, explained_variance, explained_variance_ratio, singular_values, means, var, noise_variance, \ n_component, n_samples_seen
def extract_img(result_compressed): k = result_compressed[0] ipca = IncrementalPCA(n_components=k) img_compressed = result_compressed[1] dict_att = result_compressed[2] for key in dict_att.keys(): ipca.__setattr__(key, dict_att[key]) # print((dict_att['mean_'])) img_extracted = ipca.inverse_transform(img_compressed) return img_extracted
def extract_img(k, result_compressed): ipca = IncrementalPCA(n_components=k) img_compressed = result_compressed[0] dict_att = result_compressed[1] for key in dict_att.keys(): ipca.__setattr__(key, dict_att[key]) img_extracted = ipca.inverse_transform(img_compressed) return img_extracted
class KDEPCAGen(GenBase): def __init__(self, kernel="gaussian", bandwidth=0.1, n_components=None, kde_params={}): super().__init__() self.pca = IncrementalPCA(n_components=n_components) self.bandwidth = bandwidth self.kernel = kernel self.kde_params = kde_params self.manifold = None self.len_data = None def fit(self, x): x_pca = self.pca.fit_transform(x) self.manifold = KDEGen(kernel=self.kernel, bandwidth=self.bandwidth, **self.kde_params).fit(x_pca) return self def sample_radius(self, x_exp, n_min_kernels=20, r=None, n_samples=1, random_state=None): x_exp_pca = self.pca.transform(x_exp) x_sample_pca = self.manifold.sample_radius(x_exp_pca, n_min_kernels=n_min_kernels, r=r, n_samples=n_samples, random_state=random_state) x_sample = self.pca.inverse_transform(x_sample_pca) return x_sample def sample(self, n_samples=1, random_state=None): x_sample_pca = self.manifold.sample(n_samples=n_samples, random_state=random_state) x_sample = self.pca.inverse_transform(x_sample_pca) return x_sample
def test_incremental_pca_inverse(): """Test that the projection of data can be inverted.""" rng = np.random.RandomState(1999) n, p = 50, 3 X = rng.randn(n, p) # spherical data X[:, 1] *= .00001 # make middle component relatively small X += [5, 4, 3] # make a large mean # same check that we can find the original data from the transformed # signal (since the data is almost of rank n_components) ipca = IncrementalPCA(n_components=2, batch_size=10).fit(X) Y = ipca.transform(X) Y_inverse = ipca.inverse_transform(Y) assert_almost_equal(X, Y_inverse, decimal=3)
def test_incremental_pca_inverse(): # Test that the projection of data can be inverted. rng = np.random.RandomState(1999) n, p = 50, 3 X = rng.randn(n, p) # spherical data X[:, 1] *= .00001 # make middle component relatively small X += [5, 4, 3] # make a large mean # same check that we can find the original data from the transformed # signal (since the data is almost of rank n_components) ipca = IncrementalPCA(n_components=2, batch_size=10).fit(X) Y = ipca.transform(X) Y_inverse = ipca.inverse_transform(Y) assert_almost_equal(X, Y_inverse, decimal=3)
def find_factors(self, values: np.ndarray) -> (np.ndarray, np.ndarray): algorithm = IncrementalPCA(**self.params) algorithm.fit(values) if self.verbose: print('Sorted eigen values') print(algorithm.singular_values_) print('Sorted eigen vectors') print(algorithm.components_) print('Explained variance') print(algorithm.explained_variance_) print('Explained variance ratio') print(algorithm.explained_variance_ratio_) transformed_values = algorithm.transform(values) reconstructed_values = algorithm.inverse_transform(transformed_values) return transformed_values, reconstructed_values
def test_whitening(): # Test that PCA and IncrementalPCA transforms match to sign flip. X = datasets.make_low_rank_matrix( 1000, 10, tail_strength=0.0, effective_rank=2, random_state=1999 ) prec = 3 n_samples, n_features = X.shape for nc in [None, 9]: pca = PCA(whiten=True, n_components=nc).fit(X) ipca = IncrementalPCA(whiten=True, n_components=nc, batch_size=250).fit(X) Xt_pca = pca.transform(X) Xt_ipca = ipca.transform(X) assert_almost_equal(np.abs(Xt_pca), np.abs(Xt_ipca), decimal=prec) Xinv_ipca = ipca.inverse_transform(Xt_ipca) Xinv_pca = pca.inverse_transform(Xt_pca) assert_almost_equal(X, Xinv_ipca, decimal=prec) assert_almost_equal(X, Xinv_pca, decimal=prec) assert_almost_equal(Xinv_pca, Xinv_ipca, decimal=prec)
def test_whitening(): """Test that PCA and IncrementalPCA transforms match to sign flip.""" X = datasets.make_low_rank_matrix(1000, 10, tail_strength=0., effective_rank=2, random_state=1999) prec = 3 n_samples, n_features = X.shape for nc in [None, 9]: pca = PCA(whiten=True, n_components=nc).fit(X) ipca = IncrementalPCA(whiten=True, n_components=nc, batch_size=250).fit(X) Xt_pca = pca.transform(X) Xt_ipca = ipca.transform(X) assert_almost_equal(np.abs(Xt_pca), np.abs(Xt_ipca), decimal=prec) Xinv_ipca = ipca.inverse_transform(Xt_ipca) Xinv_pca = pca.inverse_transform(Xt_pca) assert_almost_equal(X, Xinv_ipca, decimal=prec) assert_almost_equal(X, Xinv_pca, decimal=prec) assert_almost_equal(Xinv_pca, Xinv_ipca, decimal=prec)
class DimReductorPCA(DimReductor): ''' Principal Component Analysis dimensionality reduction method. Uses scikit-learn's IncrementalPCA. ''' def __init__(self, X, n_features=10): super().__init__(X, n_features) self.X = np.array([X[i].flatten() for i in range(len(X))], dtype=np.float16) self.pca = IncrementalPCA(n_components=n_features, batch_size=self.batch_size) self._fit() def _fit(self): ''' Break data into mini-batches and incrementally fit the model with data. ''' for i in range(len(self.X) // self.batch_size): minibatch = self.X[self.batch_size * i:self.batch_size * (i + 1)] self.pca.partial_fit(minibatch) def transform(self, data): ''' Apply dimensionality reduction to the data. ''' data = np.array([data[i].flatten() for i in range(len(data))], dtype=np.float16) dim_reduced = self.pca.transform(data) return dim_reduced def reverse(self, dim_reduced_data): ''' Reverse dimensionality reduction on data, reshape result into 28x28 images. DOES NOT reverse any input preprocessing. ''' reconstr_data = self.pca.inverse_transform(dim_reduced_data) reconstr_images = [ reconstr_data[i].reshape((28, 28, 1)) for i in range(len(reconstr_data)) ] reconstr_images = np.array(reconstr_images) return reconstr_images
def incremental_PCA(): n_batches = 100 mnist = fetch_mldata('MNIST original') X = mnist["data"] y = mnist["target"] X_train, X_test, y_train, y_test = train_test_split(X, y) # Doesnt hold all of the data at once, loads and fits it incrementally inc_pca = IncrementalPCA(n_components=154) for X_batch in np.array_split(X_train, n_batches): print(".", end="") # not shown in the book inc_pca.partial_fit(X_batch) # PCA and incremental PCA results are not exactly identical. Incremental PCA gives a very good approximate solution X_reduced = inc_pca.transform(X_train) X_recovered_inc_pca = inc_pca.inverse_transform(X_reduced) plt.figure(figsize=(7, 4)) plt.subplot(121) plot_digits(X_train[::2100]) plt.subplot(122) plot_digits(X_recovered_inc_pca[::2100]) plt.tight_layout() plt.savefig(PNG_PATH + "inc_PCA_mnist_compression_plot", dpi=300) plt.close() # Using memmap to save the data to a file and load incrementally from there filename = "my_mnist.data" m, n = X_train.shape X_mm = np.memmap(filename, dtype='float32', mode='write', shape=(m, n)) X_mm[:] = X_train del X_mm X_mm = np.memmap(filename, dtype="float32", mode="readonly", shape=(m, n)) batch_size = m // n_batches inc_pca = IncrementalPCA(n_components=154, batch_size=batch_size) inc_pca.fit(X_mm) rnd_pca = PCA(n_components=154, svd_solver="randomized", random_state=42) X_reduced = rnd_pca.fit_transform(X_train)
def IPCA(self, components=50, batch=1000): ''' Iterative Principal Component analysis, see sklearn.decomposition.incremental_pca Parameters: ------------ components (default 50) = number of independent components to return batch (default 1000) = number of pixels to load into memory simultaneously in IPCA. More requires more memory but leads to better fit Returns ------- eigenseries: principal components (pixel time series) and associated singular values eigenframes: eigenframes are obtained by multiplying the projected frame matrix by the projected movie (whitened frames?) proj_frame_vectors:the reduced version of the movie vectors using only the principal component projection ''' # vectorize the images num_frames, h, w = np.shape(self) frame_size = h * w frame_samples = np.reshape(self, (num_frames, frame_size)).T # run IPCA to approxiate the SVD ipca_f = IncrementalPCA(n_components=components, batch_size=batch) ipca_f.fit(frame_samples) # construct the reduced version of the movie vectors using only the # principal component projection proj_frame_vectors = ipca_f.inverse_transform( ipca_f.transform(frame_samples)) # get the temporal principal components (pixel time series) and # associated singular values eigenseries = ipca_f.components_.T # the rows of eigenseries are approximately orthogonal # so we can approximately obtain eigenframes by multiplying the # projected frame matrix by this transpose on the right eigenframes = np.dot(proj_frame_vectors, eigenseries) return eigenseries, eigenframes, proj_frame_vectors
def incremental(): mnist = datasets.fetch_mldata('MNIST original') X = mnist['data'] n_batches = 100 partial_fit = False if partial_fit: inc_pca = IncrementalPCA(n_components=154) for X_batch in np.array_split(X, n_batches): inc_pca.partial_fit(X_batch) X_reduced = inc_pca.transform(X) X_restored = inc_pca.inverse_transform(X_reduced) # X_mm = np.memmap('08_mnist_memmap', dtype='float32', mode='readonly', shape=X.shape) # X_mm[:] = X # inc_pca = IncrementalPCA(n_components=154, batch_size=X.shape[0] // n_batches) # inc_pca.fit(X_mm) else: rnd_pca = PCA(n_components=X.shape[1], svd_solver='randomized') rnd_pca.fit(X) cumsum = np.cumsum(rnd_pca.explained_variance_ratio_) min_d = np.nonzero(cumsum > 0.95)[0][0] + 1 print(min_d)
def IPCA(self, components = 50, batch =1000): ''' Iterative Principal Component analysis, see sklearn.decomposition.incremental_pca Parameters: ------------ components (default 50) = number of independent components to return batch (default 1000) = number of pixels to load into memory simultaneously in IPCA. More requires more memory but leads to better fit Returns ------- eigenseries: principal components (pixel time series) and associated singular values eigenframes: eigenframes are obtained by multiplying the projected frame matrix by the projected movie (whitened frames?) proj_frame_vectors:the reduced version of the movie vectors using only the principal component projection ''' # vectorize the images num_frames, h, w = np.shape(self); frame_size = h * w; frame_samples = np.reshape(self, (num_frames, frame_size)).T # run IPCA to approxiate the SVD ipca_f = IncrementalPCA(n_components=components, batch_size=batch) ipca_f.fit(frame_samples) # construct the reduced version of the movie vectors using only the # principal component projection proj_frame_vectors = ipca_f.inverse_transform(ipca_f.transform(frame_samples)) # get the temporal principal components (pixel time series) and # associated singular values eigenseries = ipca_f.components_.T # the rows of eigenseries are approximately orthogonal # so we can approximately obtain eigenframes by multiplying the # projected frame matrix by this transpose on the right eigenframes = np.dot(proj_frame_vectors, eigenseries) return eigenseries, eigenframes, proj_frame_vectors
def run(self): self.logger.name = type(self).__name__ self.logger.info(f"filtering {self.args['video_path']}") with h5py.File(self.args["video_path"], "r") as f: data = f["data"][()] dshape = data.shape # flatten the data # n_samples <-> n_time # n_features <-> n_pixels is the dimension that is reduced # i.e. the pixels' time series are each approximated to be # a linear combination of n_components time series. data = data.reshape(dshape[0], -1) # split the data split_data = np.array_split(data, self.args["n_chunks"], axis=0) # incrementally fit the data in time chunks ipca = IncrementalPCA(n_components=self.args["n_components"]) for chunk in split_data: ipca.partial_fit(chunk) # reconstruct from the fitted components frame_counter = 0 with h5py.File(self.args["video_output"], "w") as f: output = f.create_dataset("data", shape=dshape, dtype=data.dtype, chunks=self.args["h5_chunk_shape"]) for chunk in split_data: nframes = chunk.shape[0] frame_end = frame_counter + nframes output[frame_counter: frame_end] = ipca.inverse_transform( ipca.transform(chunk)).reshape(nframes, *dshape[1:]) frame_counter += nframes self.logger.info(f"wrote {self.args['video_output']}")
class SpectrogramCompressor(object): def __init__(self, n_components, batch_size): super().__init__() self._meanstd = StandardScaler() self._pca = IncrementalPCA(n_components, batch_size=batch_size) def partial_fit(self, data): # self._meanstd.partial_fit(data) # data = self._meanstd.transform(data) self._pca.partial_fit(data) # data = self._pca.transform(data) # self._kmeans.partial_fit(data) def transform(self, data): # data = self._meanstd.transform(data) data = self._pca.transform(data) # data = self._kmeans.predict(data) return data def inverse_transform(self, data): # data = self._kmeans.cluster_centers_[data] data = self._pca.inverse_transform(data) # data = self._meanstd.inverse_transform(data) return data
class model (): def __init__(self, config, data, test=False, test_imb_ratio=1.0, test_reverse=False): self.config = config self.training_opt = self.config['training_opt'] self.data = {key: item[1] for key, item in data.items()} self.test_mode = test self.num_gpus = torch.cuda.device_count() self.do_shuffle = config['shuffle'] if 'shuffle' in config else False # Setup prior distribution self.prior_distribution = {key: item[0] for key, item in data.items()} # Setup logger self.logger = Logger(self.training_opt['log_dir']) # init moving average self.embed_mean = torch.zeros(int(self.training_opt['feature_dim'])).numpy() self.mu = 0.9 self.sumexp_logits = torch.zeros(1) if not test: self.tensorboard = SummaryWriter(log_dir=f"{self.training_opt['log_dir']}/tensorboard") self.current_step = 0 self.current_epoch = 0 # Initialize model self.init_models() # apply incremental pca self.apply_pca = ('apply_ipca' in self.config) and self.config['apply_ipca'] if self.apply_pca: print('==========> Apply Incremental PCA <=======') self.pca = IncrementalPCA(n_components=self.config['num_components'], batch_size=self.training_opt['batch_size']) # Load pre-trained model parameters if 'model_dir' in self.config and self.config['model_dir'] is not None: self.load_model(self.config['model_dir']) # Under training mode, initialize training steps, optimizers, schedulers, criterions if not self.test_mode: # If using steps for training, we need to calculate training steps # for each epoch based on actual number of training data instead of # oversampled data number print('Using steps for training.') self.training_data_num = len(self.data['train'].dataset) self.epoch_steps = int(self.training_data_num / self.training_opt['batch_size']) # Initialize model optimizer and scheduler print('Initializing model optimizer.') self.init_optimizers(self.model_optim_params_dict) self.init_criterions() # Set up log file self.log_file = os.path.join(self.training_opt['log_dir'], 'log.txt') self.logger.log_cfg(self.config) else: if test_reverse: self.log_file = os.path.join(self.training_opt['log_dir'], f'test-reverse_imb{test_imb_ratio}.txt') else: self.log_file = os.path.join(self.training_opt['log_dir'], f'test_imb{test_imb_ratio}.txt') self.logger.log_cfg(self.config) def write_summary(self, split, step, **kargs): for n, v in kargs.items(): if hasattr(self, 'tensorboard'): self.tensorboard.add_scalar( tag=f"{split}/{n}", scalar_value=v, global_step=step ) def init_models(self, optimizer=True): networks_defs = self.config['networks'] self.networks = {} self.model_optim_params_dict = {} self.model_optim_named_params = {} print("Using", torch.cuda.device_count(), "GPUs.") for key, val in networks_defs.items(): # Networks def_file = val['def_file'] model_args = val['params'] model_args.update({'test': self.test_mode}) if "Prior" in def_file: model_args["prior"] = self.prior_distribution["train"] self.networks[key] = source_import(def_file).create_model(**model_args) self.networks[key] = nn.DataParallel(self.networks[key]).cuda() if 'fix' in val and val['fix']: print('Freezing weights of module {}'.format(key)) for param_name, param in self.networks[key].named_parameters(): # Freeze all parameters except final fc layer if 'fc' not in param_name: param.requires_grad = False print('=====> Freezing: {} | False'.format(key)) if 'fix_set' in val: for fix_layer in val['fix_set']: for param_name, param in self.networks[key].named_parameters(): if fix_layer == param_name: param.requires_grad = False print('=====> Freezing: {} | {}'.format(param_name, param.requires_grad)) continue # Optimizer list optim_params = val['optim_params'] self.model_optim_named_params.update(dict(self.networks[key].named_parameters())) self.model_optim_params_dict[key] = {'params': self.networks[key].parameters(), 'lr': optim_params['lr'], 'momentum': optim_params['momentum'], 'weight_decay': optim_params['weight_decay']} def init_criterions(self): criterion_defs = self.config['criterions'] self.criterions = {} self.criterion_weights = {} for key, val in criterion_defs.items(): def_file = val['def_file'] loss_args = val['loss_params'] self.criterions[key] = source_import(def_file).create_loss(**loss_args).cuda() self.criterion_weights[key] = val['weight'] if val['optim_params']: print('Initializing criterion optimizer.') optim_params = val['optim_params'] optim_params = [{'params': self.criterions[key].parameters(), 'lr': optim_params['lr'], 'momentum': optim_params['momentum'], 'weight_decay': optim_params['weight_decay']}] # Initialize criterion optimizer and scheduler self.criterion_optimizer, \ self.criterion_optimizer_scheduler = self.init_optimizers(optim_params) else: self.criterion_optimizer = None def init_optimizers(self, optim_params_dict): ''' seperate backbone optimizer and classifier optimizer by Kaihua ''' networks_defs = self.config['networks'] self.model_optimizer_dict = {} self.model_scheduler_dict = {} for key, val in networks_defs.items(): # optimizer if 'optimizer' in self.training_opt and self.training_opt['optimizer'] == 'adam': print('=====> Using Adam optimizer') optimizer = optim.Adam([optim_params_dict[key],]) else: print('=====> Using SGD optimizer') optimizer = optim.SGD([optim_params_dict[key],]) self.model_optimizer_dict[key] = optimizer # scheduler scheduler_params = val['scheduler_params'] if scheduler_params['coslr']: print("===> Module {} : Using coslr eta_min={}".format(key, scheduler_params['endlr'])) self.model_scheduler_dict[key] = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, self.training_opt['num_epochs'], eta_min=scheduler_params['endlr']) elif scheduler_params['warmup']: print("===> Module {} : Using warmup".format(key)) self.model_scheduler_dict[key] = WarmupMultiStepLR(optimizer, scheduler_params['lr_step'], gamma=scheduler_params['lr_factor'], warmup_epochs=scheduler_params['warm_epoch']) else: self.model_scheduler_dict[key] = optim.lr_scheduler.StepLR(optimizer, step_size=scheduler_params['step_size'], gamma=scheduler_params['gamma']) return def show_current_lr(self): max_lr = 0.0 for key, val in self.model_optimizer_dict.items(): lr_set = list(set([para['lr'] for para in val.param_groups])) if max(lr_set) > max_lr: max_lr = max(lr_set) lr_set = ','.join([str(i) for i in lr_set]) print_str = ['=====> Current Learning Rate of model {} : {}'.format(key, str(lr_set))] print_write(print_str, self.log_file) return max_lr def batch_forward(self, inputs, labels=None, feature_ext=False, phase='train'): ''' This is a general single batch running function. ''' # Calculate Features self.features = self.networks['feat_model'](inputs) if self.apply_pca: if phase=='train' and self.features.shape[0] > 0: self.pca.partial_fit(self.features.cpu().numpy()) else: pca_feat = self.pca.transform(self.features.cpu().numpy()) pca_feat[:, 0] = 0.0 new_feat = self.pca.inverse_transform(pca_feat) self.features = torch.from_numpy(new_feat).float().to(self.features.device) # update moving average if phase == 'train': self.embed_mean = self.mu * self.embed_mean + self.features.detach().mean(0).view(-1).cpu().numpy() # If not just extracting features, calculate logits if not feature_ext: # cont_eval = 'continue_eval' in self.training_opt and self.training_opt['continue_eval'] and phase != 'train' self.logits, self.route_logits = self.networks['classifier'](self.features, labels, self.embed_mean) def batch_backward(self, print_grad=False): # Zero out optimizer gradients for key, optimizer in self.model_optimizer_dict.items(): optimizer.zero_grad() if self.criterion_optimizer: self.criterion_optimizer.zero_grad() # Back-propagation from loss outputs self.loss.backward() # display gradient if self.training_opt['display_grad']: print_grad_norm(self.model_optim_named_params, print_write, self.log_file, verbose=print_grad) # Step optimizers for key, optimizer in self.model_optimizer_dict.items(): optimizer.step() if self.criterion_optimizer: self.criterion_optimizer.step() def batch_loss(self, labels): self.loss = 0 # First, apply performance loss if 'PerformanceLoss' in self.criterions.keys(): self.loss_perf = self.criterions['PerformanceLoss'](self.logits, labels) self.loss_perf *= self.criterion_weights['PerformanceLoss'] self.loss += self.loss_perf # Apply loss on Route Weights if set up if 'RouteWeightLoss' in self.criterions.keys(): self.loss_route = self.criterions['RouteWeightLoss'](self.route_logits, labels) self.loss_route = self.loss_route * self.criterion_weights['RouteWeightLoss'] # Add Route Weights loss to total loss self.loss += self.loss_route # hard-coded self.sumexp_logits = torch.sum(torch.exp(self.logits), dim=-1) def shuffle_batch(self, x, y): index = torch.randperm(x.size(0)) x = x[index] y = y[index] return x, y def train(self): # When training the network print_str = ['Phase: train'] print_write(print_str, self.log_file) time.sleep(0.25) print_write(['Force shuffle in training??? --- ', self.do_shuffle], self.log_file) # Initialize best model best_model_weights = {} best_model_weights['feat_model'] = copy.deepcopy(self.networks['feat_model'].state_dict()) best_model_weights['classifier'] = copy.deepcopy(self.networks['classifier'].state_dict()) best_acc = 0.0 best_epoch = 0 end_epoch = self.training_opt['num_epochs'] # Loop over epochs for epoch in range(1, end_epoch + 1): self.current_epoch = epoch for key, model in self.networks.items(): # only train the module with lr > 0 if self.config['networks'][key]['optim_params']['lr'] == 0.0: print_write(['=====> module {} is set to eval due to 0.0 learning rate.'.format(key)], self.log_file) model.eval() else: model.train() torch.cuda.empty_cache() # Set model modes and set scheduler # In training, step optimizer scheduler and set model to train() for key, scheduler in self.model_scheduler_dict.items(): scheduler.step() if self.criterion_optimizer: self.criterion_optimizer_scheduler.step() # Iterate over dataset total_preds = [] total_labels = [] # indicate current path print_write([self.training_opt['log_dir']], self.log_file) # print learning rate current_lr = self.show_current_lr() current_lr = min(current_lr * 50, 1.0) # scale the original mu according to the lr if 'CIFAR' not in self.training_opt['dataset']: self.mu = 1.0 - (1 - 0.9) * current_lr for step, (inputs, labels, indexes) in enumerate(self.data['train']): # Break when step equal to epoch step if step == self.epoch_steps: break self.current_step += 1 if self.do_shuffle: inputs, labels = self.shuffle_batch(inputs, labels) inputs, labels = inputs.cuda(), labels.cuda() # If on training phase, enable gradients with torch.set_grad_enabled(True): # If training, forward with loss, and no top 5 accuracy calculation self.batch_forward(inputs, labels, phase='train') self.batch_loss(labels) self.batch_backward(print_grad=(step % self.training_opt['display_grad_step'] == 0)) # Tracking predictions _, preds = torch.max(self.logits, 1) total_preds.append(torch2numpy(preds)) total_labels.append(torch2numpy(labels)) # Output minibatch training results if step % self.training_opt['display_step'] == 0: records = dict() if 'RouteWeightLoss' in self.criterions: records['loss_route'] = self.loss_route.item() else: records["loss_route"] = 0. if 'PerformanceLoss' in self.criterions: records['loss_perf'] = self.loss_perf.item() records['loss'] = self.loss.item() minibatch_acc = mic_acc_cal(preds, labels) records['acc'] = minibatch_acc records['sum_exp_logits'] = torch.mean(self.sumexp_logits).item() print_str = ['Epoch: [%d/%d]' % (epoch, self.training_opt['num_epochs']), 'Step: %5d' % (step)] print_str.extend([ f'{key}: {val:.3f}' for key, val in records.items() ]) self.write_summary( split='train', step=self.current_step, **records ) print_write(print_str, self.log_file) loss_info = { 'Epoch': epoch, 'Step': step, 'Total': records['loss'], 'CE': records['loss_perf'], 'route': records['loss_route'], } self.logger.log_loss(loss_info) # batch-level: sampler update if hasattr(self.data['train'].sampler, 'update_weights'): if hasattr(self.data['train'].sampler, 'ptype'): ptype = self.data['train'].sampler.ptype else: ptype = 'score' ws = get_priority(ptype, self.logits.detach(), labels) inlist = [indexes.cpu().numpy(), ws] if self.training_opt['sampler']['type'] == 'ClassPrioritySampler': inlist.append(labels.cpu().numpy()) self.data['train'].sampler.update_weights(*inlist) # epoch-level: reset sampler weight if hasattr(self.data['train'].sampler, 'get_weights'): self.logger.log_ws(epoch, self.data['train'].sampler.get_weights()) if hasattr(self.data['train'].sampler, 'reset_weights'): self.data['train'].sampler.reset_weights(epoch) # After every epoch, validation rsls = {'epoch': epoch} rsls_train = self.eval_with_preds(total_preds, total_labels) rsls_eval = self.eval(phase='val') rsls.update(rsls_train) rsls.update(rsls_eval) # Reset class weights for sampling if pri_mode is valid if hasattr(self.data['train'].sampler, 'reset_priority'): ws = get_priority(self.data['train'].sampler.ptype, self.total_logits.detach(), self.total_labels) self.data['train'].sampler.reset_priority(ws, self.total_labels.cpu().numpy()) # Log results self.logger.log_acc(rsls) # Under validation, the best model need to be updated if self.eval_acc_mic_top1 > best_acc: best_epoch = epoch best_acc = self.eval_acc_mic_top1 best_model_weights['feat_model'] = copy.deepcopy(self.networks['feat_model'].state_dict()) best_model_weights['classifier'] = copy.deepcopy(self.networks['classifier'].state_dict()) print('===> Saving checkpoint') self.save_latest(epoch) print() print('Training Complete.') print_str = ['Best validation accuracy is %.3f at epoch %d' % (best_acc, best_epoch)] print_write(print_str, self.log_file) # Save the best model self.save_model(epoch, best_epoch, best_model_weights, best_acc) # Test on the test set if 'CIFAR' not in self.training_opt["dataset"]: self.reset_model(best_model_weights) self.eval('test' if 'test' in self.data else 'val') print('Done') def eval_with_preds(self, preds, labels): # Count the number of examples n_total = sum([len(p) for p in preds]) # Split the examples into normal and mixup normal_preds, normal_labels = [], [] mixup_preds, mixup_labels1, mixup_labels2, mixup_ws = [], [], [], [] for p, l in zip(preds, labels): if isinstance(l, tuple): mixup_preds.append(p) mixup_labels1.append(l[0]) mixup_labels2.append(l[1]) mixup_ws.append(l[2] * np.ones_like(l[0])) else: normal_preds.append(p) normal_labels.append(l) # Calculate normal prediction accuracy rsl = {'train_all':0., 'train_many':0., 'train_median':0., 'train_low': 0.} if len(normal_preds) > 0: normal_preds, normal_labels = list(map(np.concatenate, [normal_preds, normal_labels])) n_top1 = mic_acc_cal(normal_preds, normal_labels) n_top1_many, \ n_top1_median, \ n_top1_low, = shot_acc(normal_preds, normal_labels, self.data['train']) rsl['train_all'] += len(normal_preds) / n_total * n_top1 rsl['train_many'] += len(normal_preds) / n_total * n_top1_many rsl['train_median'] += len(normal_preds) / n_total * n_top1_median rsl['train_low'] += len(normal_preds) / n_total * n_top1_low # Calculate mixup prediction accuracy if len(mixup_preds) > 0: mixup_preds, mixup_labels, mixup_ws = \ list(map(np.concatenate, [mixup_preds*2, mixup_labels1+mixup_labels2, mixup_ws])) mixup_ws = np.concatenate([mixup_ws, 1-mixup_ws]) n_top1 = weighted_mic_acc_cal(mixup_preds, mixup_labels, mixup_ws) n_top1_many, \ n_top1_median, \ n_top1_low, = weighted_shot_acc(mixup_preds, mixup_labels, mixup_ws, self.data['train']) rsl['train_all'] += len(mixup_preds) / 2 / n_total * n_top1 rsl['train_many'] += len(mixup_preds) / 2 / n_total * n_top1_many rsl['train_median'] += len(mixup_preds) / 2 / n_total * n_top1_median rsl['train_low'] += len(mixup_preds) / 2 / n_total * n_top1_low # Top-1 accuracy and additional string print_str = ['\n Training acc Top1: %.3f \n' % (rsl['train_all']), 'Many_top1: %.3f' % (rsl['train_many']), 'Median_top1: %.3f' % (rsl['train_median']), 'Low_top1: %.3f' % (rsl['train_low']), '\n'] print_write(print_str, self.log_file) return rsl def store_logits(self, phase): if phase not in self.data: print(f'No phase {phase}. Not storing logits.') return self.total_logits = torch.empty((0, self.training_opt['num_classes'])).cuda() self.total_labels = torch.empty(0, dtype=torch.long).cuda() # Iterate over dataset for model in self.networks.values(): model.eval() for inputs, labels, paths in tqdm(self.data[phase]): inputs, labels = inputs.cuda(), labels.cuda() # If on training phase, enable gradients with torch.set_grad_enabled(False): # In validation or testing self.batch_forward(inputs, labels, phase="val") if hasattr(self.networks["classifier"].module, "thresholds"): self.logits = self.route_logits - self.networks["classifier"].module.thresholds self.total_logits = torch.cat((self.total_logits, self.logits)) self.total_labels = torch.cat((self.total_labels, labels)) np.save(os.path.join(self.training_opt['log_dir'], f"{phase}_total_logits"), self.total_logits.cpu().data.numpy()) np.save(os.path.join(self.training_opt['log_dir'], f"{phase}_total_labels"), self.total_labels.cpu().data.numpy()) def eval(self, phase='val', save_feat=False): print_str = ['Phase: %s' % (phase)] print_write(print_str, self.log_file) if phase == "test": self.store_logits(phase="train") self.store_logits(phase="val") self.store_logits(phase="test") time.sleep(0.25) torch.cuda.empty_cache() # In validation or testing mode, set model to eval() and initialize running loss/correct for model in self.networks.values(): model.eval() self.total_logits = torch.empty((0, self.training_opt['num_classes'])).cuda() self.total_labels = torch.empty(0, dtype=torch.long).cuda() self.total_paths = np.empty(0) feats_all, labels_all, idxs_all, logits_all = [], [], [], [] featmaps_all = [] # feature saving initialization if save_feat: self.saving_feature_with_label_init() # Iterate over dataset for inputs, labels, paths in tqdm(self.data[phase]): inputs, labels = inputs.cuda(), labels.cuda() # If on training phase, enable gradients with torch.set_grad_enabled(False): # In validation or testing self.batch_forward(inputs, labels, phase=phase) # feature saving update if save_feat: self.saving_feature_with_label_update(self.features, self.logits, labels) if "Softmax" in self.config["criterions"]["PerformanceLoss"]["def_file"] and \ "RouteWeightLoss" not in self.config["criterions"] and \ "DotProductClassifier" in self.config["networks"]["classifier"]["def_file"]: self.logits -= torch.log(self.prior_distribution["train"]).cuda() else: self.logits += torch.log(self.prior_distribution[phase]).cuda() self.total_logits = torch.cat((self.total_logits, self.logits)) self.total_labels = torch.cat((self.total_labels, labels)) self.total_paths = np.concatenate((self.total_paths, paths)) # feature saving export if save_feat: self.saving_feature_with_label_export() probs, preds = F.softmax(self.total_logits.detach(), dim=1).max(dim=1) # Calculate the overall accuracy and F measurement self.eval_acc_mic_top1= mic_acc_cal(preds[self.total_labels != -1], self.total_labels[self.total_labels != -1]) self.eval_f_measure = F_measure(preds, self.total_labels, theta=self.training_opt['open_threshold']) self.many_acc_top1, \ self.median_acc_top1, \ self.low_acc_top1, \ self.cls_accs = shot_acc(preds[self.total_labels != -1], self.total_labels[self.total_labels != -1], self.data['train'], acc_per_cls=True) # Top-1 accuracy and additional string print_str = ['\n\n', 'Phase: %s' % (phase), '\n\n', 'Evaluation_accuracy_micro_top1: %.3f' % (self.eval_acc_mic_top1), '\n', 'Averaged F-measure: %.3f' % (self.eval_f_measure), '\n', 'Many_shot_accuracy_top1: %.3f' % (self.many_acc_top1), 'Median_shot_accuracy_top1: %.3f' % (self.median_acc_top1), 'Low_shot_accuracy_top1: %.3f' % (self.low_acc_top1), '\n'] rsl = {phase + '_all': self.eval_acc_mic_top1, phase + '_many': self.many_acc_top1, phase + '_median': self.median_acc_top1, phase + '_low': self.low_acc_top1, phase + '_fscore': self.eval_f_measure} if phase == 'val': print_write(print_str, self.log_file) self.write_summary( split='val', step=self.current_epoch, eval_acc_mic_top1=self.eval_acc_mic_top1, many_acc_top1=self.many_acc_top1, median_acc_top1=self.median_acc_top1, low_acc_top1=self.low_acc_top1, eval_f_measure=self.eval_f_measure, ) else: acc_str = ["{:.1f} \t {:.1f} \t {:.1f} \t {:.1f}".format( self.many_acc_top1 * 100, self.median_acc_top1 * 100, self.low_acc_top1 * 100, self.eval_acc_mic_top1 * 100)] if self.log_file is not None and os.path.exists(self.log_file): print_write(print_str, self.log_file) print_write(acc_str, self.log_file) else: print(*print_str) print(*acc_str) if phase == 'test': with open(os.path.join(self.training_opt['log_dir'], 'cls_accs.pkl'), 'wb') as f: pickle.dump(self.cls_accs, f) return rsl def reset_model(self, model_state): for key, model in self.networks.items(): weights = model_state[key] weights = {k: weights[k] for k in weights if k in model.state_dict()} model.load_state_dict(weights) def load_model(self, model_dir=None): model_dir = self.training_opt['log_dir'] if model_dir is None else model_dir if 'CIFAR' in self.training_opt['dataset']: # CIFARs don't have val set, so use the latest model print('Validation on the latest model.') if not model_dir.endswith('.pth'): model_dir = os.path.join(model_dir, 'latest_model_checkpoint.pth') else: print('Validation on the best model.') if not model_dir.endswith('.pth'): model_dir = os.path.join(model_dir, 'final_model_checkpoint.pth') print('Loading model from %s' % (model_dir)) checkpoint = torch.load(model_dir) if 'latest' in model_dir: model_state = checkpoint['state_dict'] else: model_state = checkpoint['state_dict_best'] for key, model in self.networks.items(): ########################################## # if loading classifier in training: # 1. only tuning memory embedding # 2. retrain the entire classifier ########################################## if 'embed' in checkpoint: print('============> Load Moving Average <===========') self.embed_mean = checkpoint['embed'] if not self.test_mode and 'Classifier' in self.config['networks'][key]['def_file']: if 'tuning_memory' in self.config and self.config['tuning_memory']: print('=============== WARNING! WARNING! ===============') print('========> Only Tuning Memory Embedding <========') for param_name, param in self.networks[key].named_parameters(): # frezing all params only tuning memory_embeding if 'embed' in param_name: param.requires_grad = True print('=====> Abandon Weight {} in {} from the checkpoints.'.format(param_name, key)) if param_name in model_state[key]: del model_state[key][param_name] else: param.requires_grad = False print('=====> Tuning: {} | {}'.format(str(param.requires_grad).ljust(5, ' '), param_name)) print('=================================================') else: # Skip classifier initialization #print('================ WARNING! WARNING! ================') print('=======> Load classifier from checkpoint <=======') #print('===================================================') #continue weights = model_state[key] weights = {k: weights[k] for k in weights if k in model.state_dict()} x = model.state_dict() x.update(weights) if all([weights[k].sum().item() == x[k].sum().item() for k in weights if k in x]): print('=====> All keys in weights have been loaded to the module {}'.format(key)) else: print('=====> Error! Error! Error! Error! Loading failure in module {}'.format(key)) model.load_state_dict(x) def save_latest(self, epoch): model_weights = {} model_weights['feat_model'] = copy.deepcopy(self.networks['feat_model'].state_dict()) model_weights['classifier'] = copy.deepcopy(self.networks['classifier'].state_dict()) model_states = { 'epoch': epoch, 'state_dict': model_weights, 'embed': self.embed_mean, } model_dir = os.path.join(self.training_opt['log_dir'], 'latest_model_checkpoint.pth') torch.save(model_states, model_dir) def save_model(self, epoch, best_epoch, best_model_weights, best_acc): model_states = {'epoch': epoch, 'best_epoch': best_epoch, 'state_dict_best': best_model_weights, 'best_acc': best_acc, 'embed': self.embed_mean,} model_dir = os.path.join(self.training_opt['log_dir'], 'final_model_checkpoint.pth') torch.save(model_states, model_dir) def output_logits(self): filename = os.path.join(self.training_opt['log_dir'], 'logits') print("Saving total logits to: %s.npz" % filename) np.savez(filename, logits=self.total_logits.detach().cpu().numpy(), labels=self.total_labels.detach().cpu().numpy(), paths=self.total_paths) def saving_feature_with_label_init(self): self.saving_feature_container = [] self.saving_logit_container = [] self.saving_label_container = [] def saving_feature_with_label_update(self, features, logits, labels): self.saving_feature_container.append(features.detach().cpu()) self.saving_logit_container.append(logits.detach().cpu()) self.saving_label_container.append(labels.detach().cpu()) def saving_feature_with_label_export(self): eval_features = {'features': torch.cat(self.saving_feature_container, dim=0).numpy(), 'labels': torch.cat(self.saving_label_container, dim=0).numpy(), 'logits': torch.cat(self.saving_logit_container, dim=0).numpy(), } eval_features_dir = os.path.join(self.training_opt['log_dir'], 'eval_features_with_labels.pth') torch.save(eval_features, eval_features_dir) print_write(['=====> Features with labels are saved as {}'.format(eval_features_dir)], self.log_file) def calculate_thresholds(self, phase): for model in self.networks.values(): model.eval() store_logits = [] if phase in self.data: for inputs, labels, paths in tqdm(self.data[phase]): inputs, labels = inputs.cuda(), labels.cuda() # If on training phase, enable gradients with torch.set_grad_enabled(False): # In validation or testing self.batch_forward(inputs, labels, phase="test") store_logits.append(self.route_logits) # route_logits: (B, C) store_logits = torch.cat(store_logits, dim=0) # (number_of_samples, C) thresholds = torch.logsumexp(store_logits, dim=0) - np.log(store_logits.size(0)) return thresholds.data.cpu().numpy() else: return None
train_test_split(X, y, test_size=0.2, random_state=1) print(X_train.shape, X_test.shape, y_train.shape, y_test.shape) # Incremental PCA 수행 n_batches = 100 # 미니 배치 횟수 - PCA를 나눠서 실행하는 횟수 n_pc = 154 # 전체 PC(principal component)들 중에서 선택할 PC 개수 inc_pca = IncrementalPCA(n_components=n_pc) for X_batch in np.split(X_train, n_batches): print('=', end='') inc_pca.partial_fit(X_batch) # fit 아니라 partial_fit 호출 print() # 훈련 셋의 차원을 784 -> 154로 축소 X_train_reduced = inc_pca.transform(X_train) print(X_train_reduced.shape) # (56000, 154) # 차원이 축소된 훈련 셋을 다시 784차원으로 복원 X_train_recovered = inc_pca.inverse_transform(X_train_reduced) print(X_train_recovered.shape) # 원본 데이터와 얼마나 차이가 나는지 그래프로 확인 fig, ax = plt.subplots(nrows=1, ncols=2) idx = 1000 image_original = X_train[idx].reshape((28, 28)) image_recovered = X_train_recovered[idx].reshape((28, 28)) ax[0].imshow(image_original, cmap=plt.cm.binary) ax[1].imshow(image_recovered, cmap=plt.cm.binary) plt.show()
def get_image_features(data_type, block): """ Method which returns the data type expected """ if data_type == 'lab': block_file_path = '/tmp/lab_img.png' block.save(block_file_path) data = transform.get_LAB_L_SVD_s(Image.open(block_file_path)) if data_type == 'mscn': img_mscn_revisited = transform.rgb_to_mscn(block) # save tmp as img img_output = Image.fromarray(img_mscn_revisited.astype('uint8'), 'L') mscn_revisited_file_path = '/tmp/mscn_revisited_img.png' img_output.save(mscn_revisited_file_path) img_block = Image.open(mscn_revisited_file_path) # extract from temp image data = compression.get_SVD_s(img_block) """if data_type == 'mscn': img_gray = np.array(color.rgb2gray(np.asarray(block))*255, 'uint8') img_mscn = transform.calculate_mscn_coefficients(img_gray, 7) img_mscn_norm = transform.normalize_2D_arr(img_mscn) img_mscn_gray = np.array(img_mscn_norm*255, 'uint8') data = compression.get_SVD_s(img_mscn_gray) """ if data_type == 'low_bits_6': low_bits_6 = transform.rgb_to_LAB_L_low_bits(block, 6) data = compression.get_SVD_s(low_bits_6) if data_type == 'low_bits_5': low_bits_5 = transform.rgb_to_LAB_L_low_bits(block, 5) data = compression.get_SVD_s(low_bits_5) if data_type == 'low_bits_4': low_bits_4 = transform.rgb_to_LAB_L_low_bits(block, 4) data = compression.get_SVD_s(low_bits_4) if data_type == 'low_bits_3': low_bits_3 = transform.rgb_to_LAB_L_low_bits(block, 3) data = compression.get_SVD_s(low_bits_3) if data_type == 'low_bits_2': low_bits_2 = transform.rgb_to_LAB_L_low_bits(block, 2) data = compression.get_SVD_s(low_bits_2) if data_type == 'low_bits_4_shifted_2': data = compression.get_SVD_s(transform.rgb_to_LAB_L_bits( block, (3, 6))) if data_type == 'sub_blocks_stats': block = np.asarray(block) width, height, _ = block.shape sub_width, sub_height = int(width / 4), int(height / 4) sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height)) data = [] for sub_b in sub_blocks: # by default use the whole lab L canal l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b)) # get information we want from svd data.append(np.mean(l_svd_data)) data.append(np.median(l_svd_data)) data.append(np.percentile(l_svd_data, 25)) data.append(np.percentile(l_svd_data, 75)) data.append(np.var(l_svd_data)) area_under_curve = utils.integral_area_trapz(l_svd_data, dx=100) data.append(area_under_curve) # convert into numpy array after computing all stats data = np.asarray(data) if data_type == 'sub_blocks_stats_reduced': block = np.asarray(block) width, height, _ = block.shape sub_width, sub_height = int(width / 4), int(height / 4) sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height)) data = [] for sub_b in sub_blocks: # by default use the whole lab L canal l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b)) # get information we want from svd data.append(np.mean(l_svd_data)) data.append(np.median(l_svd_data)) data.append(np.percentile(l_svd_data, 25)) data.append(np.percentile(l_svd_data, 75)) data.append(np.var(l_svd_data)) # convert into numpy array after computing all stats data = np.asarray(data) if data_type == 'sub_blocks_area': block = np.asarray(block) width, height, _ = block.shape sub_width, sub_height = int(width / 8), int(height / 8) sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height)) data = [] for sub_b in sub_blocks: # by default use the whole lab L canal l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b)) area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50) data.append(area_under_curve) # convert into numpy array after computing all stats data = np.asarray(data) if data_type == 'sub_blocks_area_normed': block = np.asarray(block) width, height, _ = block.shape sub_width, sub_height = int(width / 8), int(height / 8) sub_blocks = segmentation.divide_in_blocks(block, (sub_width, sub_height)) data = [] for sub_b in sub_blocks: # by default use the whole lab L canal l_svd_data = np.array(transform.get_LAB_L_SVD_s(sub_b)) l_svd_data = utils.normalize_arr(l_svd_data) area_under_curve = utils.integral_area_trapz(l_svd_data, dx=50) data.append(area_under_curve) # convert into numpy array after computing all stats data = np.asarray(data) if data_type == 'mscn_var_4': data = _get_mscn_variance(block, (100, 100)) if data_type == 'mscn_var_16': data = _get_mscn_variance(block, (50, 50)) if data_type == 'mscn_var_64': data = _get_mscn_variance(block, (25, 25)) if data_type == 'mscn_var_16_max': data = _get_mscn_variance(block, (50, 50)) data = np.asarray(data) size = int(len(data) / 4) indices = data.argsort()[-size:][::-1] data = data[indices] if data_type == 'mscn_var_64_max': data = _get_mscn_variance(block, (25, 25)) data = np.asarray(data) size = int(len(data) / 4) indices = data.argsort()[-size:][::-1] data = data[indices] if data_type == 'ica_diff': current_image = transform.get_LAB_L(block) ica = FastICA(n_components=50) ica.fit(current_image) image_ica = ica.fit_transform(current_image) image_restored = ica.inverse_transform(image_ica) final_image = utils.normalize_2D_arr(image_restored) final_image = np.array(final_image * 255, 'uint8') sv_values = utils.normalize_arr(compression.get_SVD_s(current_image)) ica_sv_values = utils.normalize_arr(compression.get_SVD_s(final_image)) data = abs(np.array(sv_values) - np.array(ica_sv_values)) if data_type == 'svd_trunc_diff': current_image = transform.get_LAB_L(block) svd = TruncatedSVD(n_components=30, n_iter=100, random_state=42) transformed_image = svd.fit_transform(current_image) restored_image = svd.inverse_transform(transformed_image) reduced_image = (current_image - restored_image) U, s, V = compression.get_SVD(reduced_image) data = s if data_type == 'ipca_diff': current_image = transform.get_LAB_L(block) transformer = IncrementalPCA(n_components=20, batch_size=25) transformed_image = transformer.fit_transform(current_image) restored_image = transformer.inverse_transform(transformed_image) reduced_image = (current_image - restored_image) U, s, V = compression.get_SVD(reduced_image) data = s if data_type == 'svd_reconstruct': reconstructed_interval = (90, 200) begin, end = reconstructed_interval lab_img = transform.get_LAB_L(block) lab_img = np.array(lab_img, 'uint8') U, s, V = lin_svd(lab_img, full_matrices=True) smat = np.zeros((end - begin, end - begin), dtype=complex) smat[:, :] = np.diag(s[begin:end]) output_img = np.dot(U[:, begin:end], np.dot(smat, V[begin:end, :])) output_img = np.array(output_img, 'uint8') data = compression.get_SVD_s(output_img) if 'sv_std_filters' in data_type: # convert into lab by default to apply filters lab_img = transform.get_LAB_L(block) arr = np.array(lab_img) images = [] # Apply list of filter on arr images.append(medfilt2d(arr, [3, 3])) images.append(medfilt2d(arr, [5, 5])) images.append(wiener(arr, [3, 3])) images.append(wiener(arr, [5, 5])) # By default computation of current block image s_arr = compression.get_SVD_s(arr) sv_vector = [s_arr] # for each new image apply SVD and get SV for img in images: s = compression.get_SVD_s(img) sv_vector.append(s) sv_array = np.array(sv_vector) _, length = sv_array.shape sv_std = [] # normalize each SV vectors and compute standard deviation for each sub vectors for i in range(length): sv_array[:, i] = utils.normalize_arr(sv_array[:, i]) sv_std.append(np.std(sv_array[:, i])) indices = [] if 'lowest' in data_type: indices = utils.get_indices_of_lowest_values(sv_std, 200) if 'highest' in data_type: indices = utils.get_indices_of_highest_values(sv_std, 200) # data are arranged following std trend computed data = s_arr[indices] # with the use of wavelet if 'wave_sv_std_filters' in data_type: # convert into lab by default to apply filters lab_img = transform.get_LAB_L(block) arr = np.array(lab_img) images = [] # Apply list of filter on arr images.append(medfilt2d(arr, [3, 3])) # By default computation of current block image s_arr = compression.get_SVD_s(arr) sv_vector = [s_arr] # for each new image apply SVD and get SV for img in images: s = compression.get_SVD_s(img) sv_vector.append(s) sv_array = np.array(sv_vector) _, length = sv_array.shape sv_std = [] # normalize each SV vectors and compute standard deviation for each sub vectors for i in range(length): sv_array[:, i] = utils.normalize_arr(sv_array[:, i]) sv_std.append(np.std(sv_array[:, i])) indices = [] if 'lowest' in data_type: indices = utils.get_indices_of_lowest_values(sv_std, 200) if 'highest' in data_type: indices = utils.get_indices_of_highest_values(sv_std, 200) # data are arranged following std trend computed data = s_arr[indices] # with the use of wavelet if 'sv_std_filters_full' in data_type: # convert into lab by default to apply filters lab_img = transform.get_LAB_L(block) arr = np.array(lab_img) images = [] # Apply list of filter on arr kernel = np.ones((3, 3), np.float32) / 9 images.append(cv2.filter2D(arr, -1, kernel)) kernel = np.ones((5, 5), np.float32) / 25 images.append(cv2.filter2D(arr, -1, kernel)) images.append(cv2.GaussianBlur(arr, (3, 3), 0.5)) images.append(cv2.GaussianBlur(arr, (3, 3), 1)) images.append(cv2.GaussianBlur(arr, (3, 3), 1.5)) images.append(cv2.GaussianBlur(arr, (5, 5), 0.5)) images.append(cv2.GaussianBlur(arr, (5, 5), 1)) images.append(cv2.GaussianBlur(arr, (5, 5), 1.5)) images.append(medfilt2d(arr, [3, 3])) images.append(medfilt2d(arr, [5, 5])) images.append(wiener(arr, [3, 3])) images.append(wiener(arr, [5, 5])) wave = w2d(arr, 'db1', 2) images.append(np.array(wave, 'float64')) # By default computation of current block image s_arr = compression.get_SVD_s(arr) sv_vector = [s_arr] # for each new image apply SVD and get SV for img in images: s = compression.get_SVD_s(img) sv_vector.append(s) sv_array = np.array(sv_vector) _, length = sv_array.shape sv_std = [] # normalize each SV vectors and compute standard deviation for each sub vectors for i in range(length): sv_array[:, i] = utils.normalize_arr(sv_array[:, i]) sv_std.append(np.std(sv_array[:, i])) indices = [] if 'lowest' in data_type: indices = utils.get_indices_of_lowest_values(sv_std, 200) if 'highest' in data_type: indices = utils.get_indices_of_highest_values(sv_std, 200) # data are arranged following std trend computed data = s_arr[indices] if 'sv_entropy_std_filters' in data_type: lab_img = transform.get_LAB_L(block) arr = np.array(lab_img) images = [] kernel = np.ones((3, 3), np.float32) / 9 images.append(cv2.filter2D(arr, -1, kernel)) kernel = np.ones((5, 5), np.float32) / 25 images.append(cv2.filter2D(arr, -1, kernel)) images.append(cv2.GaussianBlur(arr, (3, 3), 0.5)) images.append(cv2.GaussianBlur(arr, (3, 3), 1)) images.append(cv2.GaussianBlur(arr, (3, 3), 1.5)) images.append(cv2.GaussianBlur(arr, (5, 5), 0.5)) images.append(cv2.GaussianBlur(arr, (5, 5), 1)) images.append(cv2.GaussianBlur(arr, (5, 5), 1.5)) images.append(medfilt2d(arr, [3, 3])) images.append(medfilt2d(arr, [5, 5])) images.append(wiener(arr, [3, 3])) images.append(wiener(arr, [5, 5])) wave = w2d(arr, 'db1', 2) images.append(np.array(wave, 'float64')) sv_vector = [] sv_entropy_list = [] # for each new image apply SVD and get SV for img in images: s = compression.get_SVD_s(img) sv_vector.append(s) sv_entropy = [ utils.get_entropy_contribution_of_i(s, id_sv) for id_sv, sv in enumerate(s) ] sv_entropy_list.append(sv_entropy) sv_std = [] sv_array = np.array(sv_vector) _, length = sv_array.shape # normalize each SV vectors and compute standard deviation for each sub vectors for i in range(length): sv_array[:, i] = utils.normalize_arr(sv_array[:, i]) sv_std.append(np.std(sv_array[:, i])) indices = [] if 'lowest' in data_type: indices = utils.get_indices_of_lowest_values(sv_std, 200) if 'highest' in data_type: indices = utils.get_indices_of_highest_values(sv_std, 200) # data are arranged following std trend computed s_arr = compression.get_SVD_s(arr) data = s_arr[indices] if 'convolutional_kernels' in data_type: sub_zones = segmentation.divide_in_blocks(block, (20, 20)) data = [] diff_std_list_3 = [] diff_std_list_5 = [] diff_mean_list_3 = [] diff_mean_list_5 = [] plane_std_list_3 = [] plane_std_list_5 = [] plane_mean_list_3 = [] plane_mean_list_5 = [] plane_max_std_list_3 = [] plane_max_std_list_5 = [] plane_max_mean_list_3 = [] plane_max_mean_list_5 = [] for sub_zone in sub_zones: l_img = transform.get_LAB_L(sub_zone) normed_l_img = utils.normalize_2D_arr(l_img) # bilateral with window of size (3, 3) normed_diff = convolution.convolution2D(normed_l_img, kernels.min_bilateral_diff, (3, 3)) std_diff = np.std(normed_diff) mean_diff = np.mean(normed_diff) diff_std_list_3.append(std_diff) diff_mean_list_3.append(mean_diff) # bilateral with window of size (5, 5) normed_diff = convolution.convolution2D(normed_l_img, kernels.min_bilateral_diff, (5, 5)) std_diff = np.std(normed_diff) mean_diff = np.mean(normed_diff) diff_std_list_5.append(std_diff) diff_mean_list_5.append(mean_diff) # plane mean with window of size (3, 3) normed_plane_mean = convolution.convolution2D( normed_l_img, kernels.plane_mean, (3, 3)) std_plane_mean = np.std(normed_plane_mean) mean_plane_mean = np.mean(normed_plane_mean) plane_std_list_3.append(std_plane_mean) plane_mean_list_3.append(mean_plane_mean) # plane mean with window of size (5, 5) normed_plane_mean = convolution.convolution2D( normed_l_img, kernels.plane_mean, (5, 5)) std_plane_mean = np.std(normed_plane_mean) mean_plane_mean = np.mean(normed_plane_mean) plane_std_list_5.append(std_plane_mean) plane_mean_list_5.append(mean_plane_mean) # plane max error with window of size (3, 3) normed_plane_max = convolution.convolution2D( normed_l_img, kernels.plane_max_error, (3, 3)) std_plane_max = np.std(normed_plane_max) mean_plane_max = np.mean(normed_plane_max) plane_max_std_list_3.append(std_plane_max) plane_max_mean_list_3.append(mean_plane_max) # plane max error with window of size (5, 5) normed_plane_max = convolution.convolution2D( normed_l_img, kernels.plane_max_error, (5, 5)) std_plane_max = np.std(normed_plane_max) mean_plane_max = np.mean(normed_plane_max) plane_max_std_list_5.append(std_plane_max) plane_max_mean_list_5.append(mean_plane_max) diff_std_list_3 = np.array(diff_std_list_3) diff_std_list_5 = np.array(diff_std_list_5) diff_mean_list_3 = np.array(diff_mean_list_3) diff_mean_list_5 = np.array(diff_mean_list_5) plane_std_list_3 = np.array(plane_std_list_3) plane_std_list_5 = np.array(plane_std_list_5) plane_mean_list_3 = np.array(plane_mean_list_3) plane_mean_list_5 = np.array(plane_mean_list_5) plane_max_std_list_3 = np.array(plane_max_std_list_3) plane_max_std_list_5 = np.array(plane_max_std_list_5) plane_max_mean_list_3 = np.array(plane_max_mean_list_3) plane_max_mean_list_5 = np.array(plane_max_mean_list_5) if 'std_max_blocks' in data_type: data.append(np.std(diff_std_list_3[0:int(len(sub_zones) / 5)])) data.append(np.std(diff_mean_list_3[0:int(len(sub_zones) / 5)])) data.append(np.std(diff_std_list_5[0:int(len(sub_zones) / 5)])) data.append(np.std(diff_mean_list_5[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_std_list_3[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_mean_list_3[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_std_list_5[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_mean_list_5[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_max_std_list_3[0:int(len(sub_zones) / 5)])) data.append( np.std(plane_max_mean_list_3[0:int(len(sub_zones) / 5)])) data.append(np.std(plane_max_std_list_5[0:int(len(sub_zones) / 5)])) data.append( np.std(plane_max_mean_list_5[0:int(len(sub_zones) / 5)])) if 'mean_max_blocks' in data_type: data.append(np.mean(diff_std_list_3[0:int(len(sub_zones) / 5)])) data.append(np.mean(diff_mean_list_3[0:int(len(sub_zones) / 5)])) data.append(np.mean(diff_std_list_5[0:int(len(sub_zones) / 5)])) data.append(np.mean(diff_mean_list_5[0:int(len(sub_zones) / 5)])) data.append(np.mean(plane_std_list_3[0:int(len(sub_zones) / 5)])) data.append(np.mean(plane_mean_list_3[0:int(len(sub_zones) / 5)])) data.append(np.mean(plane_std_list_5[0:int(len(sub_zones) / 5)])) data.append(np.mean(plane_mean_list_5[0:int(len(sub_zones) / 5)])) data.append( np.mean(plane_max_std_list_3[0:int(len(sub_zones) / 5)])) data.append( np.mean(plane_max_mean_list_3[0:int(len(sub_zones) / 5)])) data.append( np.mean(plane_max_std_list_5[0:int(len(sub_zones) / 5)])) data.append( np.mean(plane_max_mean_list_5[0:int(len(sub_zones) / 5)])) if 'std_normed' in data_type: data.append(np.std(diff_std_list_3)) data.append(np.std(diff_mean_list_3)) data.append(np.std(diff_std_list_5)) data.append(np.std(diff_mean_list_5)) data.append(np.std(plane_std_list_3)) data.append(np.std(plane_mean_list_3)) data.append(np.std(plane_std_list_5)) data.append(np.std(plane_mean_list_5)) data.append(np.std(plane_max_std_list_3)) data.append(np.std(plane_max_mean_list_3)) data.append(np.std(plane_max_std_list_5)) data.append(np.std(plane_max_mean_list_5)) if 'mean_normed' in data_type: data.append(np.mean(diff_std_list_3)) data.append(np.mean(diff_mean_list_3)) data.append(np.mean(diff_std_list_5)) data.append(np.mean(diff_mean_list_5)) data.append(np.mean(plane_std_list_3)) data.append(np.mean(plane_mean_list_3)) data.append(np.mean(plane_std_list_5)) data.append(np.mean(plane_mean_list_5)) data.append(np.mean(plane_max_std_list_3)) data.append(np.mean(plane_max_mean_list_3)) data.append(np.mean(plane_max_std_list_5)) data.append(np.mean(plane_max_mean_list_5)) data = np.array(data) if data_type == 'convolutional_kernel_stats_svd': l_img = transform.get_LAB_L(block) normed_l_img = utils.normalize_2D_arr(l_img) # bilateral with window of size (5, 5) normed_diff = convolution.convolution2D(normed_l_img, kernels.min_bilateral_diff, (5, 5)) # getting sigma vector from SVD compression s = compression.get_SVD_s(normed_diff) data = s if data_type == 'svd_entropy': l_img = transform.get_LAB_L(block) blocks = segmentation.divide_in_blocks(l_img, (20, 20)) values = [] for b in blocks: sv = compression.get_SVD_s(b) values.append(utils.get_entropy(sv)) data = np.array(values) if data_type == 'svd_entropy_20': l_img = transform.get_LAB_L(block) blocks = segmentation.divide_in_blocks(l_img, (20, 20)) values = [] for b in blocks: sv = compression.get_SVD_s(b) values.append(utils.get_entropy(sv)) data = np.array(values) if data_type == 'svd_entropy_noise_20': l_img = transform.get_LAB_L(block) blocks = segmentation.divide_in_blocks(l_img, (20, 20)) values = [] for b in blocks: sv = compression.get_SVD_s(b) sv_size = len(sv) values.append(utils.get_entropy(sv[int(sv_size / 4):])) data = np.array(values) return data
Xts = dataset ['Xts'] Yts = dataset ['Yts'].ravel() scaler = StandardScaler() Xts = scaler.fit_transform(Xts.T).T Xtr = scaler.fit_transform(Xtr.T).T if dset==2: #Xtr=Xtr.reshape(10000,dim_image,size_image,size_image).transpose([0,2, 3, 1]).mean(3).reshape(10000,size_image*size_image) #Xts=Xts.reshape(2000,dim_image,size_image,size_image).transpose([0,2, 3, 1]).mean(3).reshape(2000,size_image*size_image) pca = PCA(n_components=100) pca.fit(Xtr) Xtr=pca.transform(Xtr) Xts=pca.transform(Xts) print(sum(pca.explained_variance_ratio_)) if plot: xplot=scaler.fit_transform(pca.inverse_transform(Xts).T).T #Xts = scaler.fit_transform(Xts.T).T #Xtr = scaler.fit_transform(Xtr.T).T ##1plt.jet() if plot: plt.figure() for i in range(0,30): image=xplot[i,].reshape(dim_image,size_image,size_image).transpose([1, 2, 0]) plt.subplot(5, 6, i+1) plt.imshow(image[:,:,:],interpolation='bicubic') plt.title(Yts[i]) indices = np.random.choice(Xts.shape[0],
n_comp = 80 n_chunks = 100 ipca = IncrementalPCA(n_components=n_comp) print('Training IPCA') for chunk in D.chunked(n_chunks): ipca.partial_fit(chunk) OutIPCA = np.array([]) print('Fitting IPCA') for chunk in D.chunked(n_chunks): Tr = ipca.transform(chunk) OutIPCA = np.vstack([OutIPCA, Tr]) if OutIPCA.size else Tr print('Training and fitting PCA') pca = PCA(n_components=n_comp) OutPCA = pca.fit_transform(D.X) print('Squared Errors:') print('Squared Error for PCA ', np.linalg.norm(D.X - pca.inverse_transform(OutPCA))) print('Squared Error for IPCA ', np.linalg.norm(D.X - ipca.inverse_transform(OutIPCA)))
#----------------------------------------------------------------- # Incremental PCA # page 283 #----------------------------------------------------------------- from sklearn.decomposition import IncrementalPCA n_batches = 100 inc_pca = IncrementalPCA(n_components=154) for X_batch in np.array_split(X_mnist, n_batches): inc_pca.partial_fit(X_batch) X_mnist_reduced = inc_pca.transform(X_mnist) X_mnist_recovered_inc = inc_pca.inverse_transform(X_mnist_reduced) plt.figure(figsize=(7, 4)) plt.subplot(121) plot_digits(X_mnist[::2100]) plt.subplot(122) plot_digits(X_mnist_recovered_inc[::2100]) plt.tight_layout() plt.show() #--------------------------------------------------------------- # numpy의 memmap파이썬 클래스를 사용하기 #--------------------------------------------------------------- np.allclose(pca.mean_, inc_pca.mean_) np.allclose(X_mnist_reduced, X_mnist_reduced)
from sklearn.decomposition import IncrementalPCA one_digit = X[0].copy() n_batches = 100 inc_pca = IncrementalPCA(n_components=154) for X_batch in np.array_split(X_train, n_batches): inc_pca.partial_fit(X_batch) X_reduced = inc_pca.transform(X_train) # In[230]: digit_reduced = inc_pca.transform([one_digit]) digit_recovered = inc_pca.inverse_transform([digit_reduced]) # How much information did we lose here print(np.linalg.norm(one_digit - digit_recovered)) # And let's plot them both showImage(one_digit) showImage(digit_recovered) # In[231]: digit_reduced.shape # # Swiss Roll database and Kernel PCA # # Till now we used the digits database, and it is not a manifold. So let's use the [Swiss roll database](https://scikit-learn.org/stable/auto_examples/cluster/plot_ward_structured_vs_unstructured.html#sphx-glr-auto-examples-cluster-plot-ward-structured-vs-unstructured-py) which is a 2d manifold in a 3d space. And let's try different dimensionality reduction methods on it.
plt.title("Original", fontsize=16) plt.subplot(122) plot_digits(X_recovered[::2100]) plt.title("Compressed", fontsize=16) plt.show() # save_fig("mnist_compression_plot") # 增量PCA(Incrementtal PCA) from sklearn.decomposition import IncrementalPCA n_batches = 100 inc_pca = IncrementalPCA(n_components=154) for X_batch in np.array_split(X_train, n_batches): print(".", end="") # not shown in the book inc_pca.partial_fit(X_batch) X_reduced = inc_pca.transform(X_train) X_recovered_inc_pca = inc_pca.inverse_transform(X_reduced) plt.figure(figsize=(7, 4)) plt.subplot(121) plot_digits(X_train[::2100]) plt.subplot(122) plot_digits(X_recovered_inc_pca[::2100]) plt.tight_layout() plt.show() # Using memmap() filename = 'mnist-original.mat' m, n = X_train.shape X_mm = np.memmap(filename, dtype="float32", mode="write", shape=(m, n))
def plot_at_k(k): ipca = IncrementalPCA(n_components=k) image_recon = ipca.inverse_transform(ipca.fit_transform(image_bw)) plt.imshow(image_recon, cmap=plt.cm.gray)
# ** La cantidad necesaria es 74 componentes, en vez de 1280, pueden explicar el 95% de la varianza de la imagen!** # 74 en vez de 1280! # # Procedamos ahora a construir la imagen utilizando sólo las 74 componentes y veamos si la reconstrucción de la imagen es visualmente distinta a la original. # # ### Reconstruyendo la imagen en b&n con 74 componentes # # 1. Primero, utilizamos al función `fit_transform` de la libreria IncrementalPCA para identificar las 74 componentes y representar la matriz en esas 74 dimensiones # 2. Luego, reconstruiremos la matriz original utilizando sólo esas 74 dimensiones mediante la función `inverse_transform` # # Finalmente dibujamos la imagen para estimar la calidad de la misma # + ipca = IncrementalPCA(n_components=k) image_recon = ipca.inverse_transform(ipca.fit_transform(image_bw)) # Plotting the reconstructed image plt.figure(figsize=[12, 8]) plt.imshow(image_recon, cmap=plt.cm.gray) # - # Bueno, para un 95% de la imagen esperábamos una buena calidad, no? notemos que la definición sigue siendo bastante buena. # # Lo que se pierde es un poco de claridad, puede observarse como un efecto de blur en algunas partes de la foto. # # # #### Probemos usar un mayor número de componentes, 150 # +