def encode(self, x, verbose=False): '''Generate the Fisher vector representation of an image or its feature vectors. Parameters ---------- x : array, shape (H,W,C) or (M,D) An input image, or a set of local feature vectors for the image. If (H,W,D), the (M,D) feature vectors are extracted using the CNN. verbose : bool, optional Whether to print FV information from VLFeat call, default=False. Returns ------- encoding : array, shape (2*k*D,) FV encoding of img, where k=num_clusters in GMM and D is CNN output depth. ''' assert hasattr(self, 'means'), 'Must call `fit` before generating encodings' if x.ndim == 2 and x.shape[-1] == self.D: feat = x elif x.ndim == 3: feat = self._localfeatures(x) else: raise ValueError( '`img` to be encoded must be 3D image or 2D feature vectors') return fisher.fisher(feat.T, self.means.T, self.covars.T, self.priors.T, improved=True, fast=False, verbose=verbose)
def get_fv(image_size, means, covariance, priors, image=None, sift_dict=None, image_p=None): if sift_dict is None or image_p is None or image_p not in sift_dict.keys(): kps, descs = face_descriptors.get_sift_desc(image, start=0) else: kps_data, descs = sift_dict[image_p] kps = face_descriptors.desirialize_keypoints(kps_data) # descs = StandardScaler().fit_transform(descs) pca = PCA(n_components=64) descs = pca.fit_transform(descs) descs = embed_spatial_info(kps, descs, image_size) # print(descs.transpose().shape,priors.shape, means.shape, covariance.shape) # D: dimension(SIFT 128-64 dim),n: number of descriptors per image, N: number of GMM, F: Features (SIFT 128-64 dim) # fisher([D,n],[F,N],[F,N],[N]) accepts dim fv_encoding = fisher(descs.transpose(), means, covariance, priors, normalized=True, square_root=True, fast=True, improved=True) return fv_encoding
def _featuresExtraction(self): testFeaMat = np.empty( ((self.compIndex) * 2 * self.dictSize * self.nRegion, 0), dtype='float32') temp = [] normImage = np.empty([2, 2]) image = pydicom.filereader.dcmread(self.filePath).pixel_array mask = self.tumorMaskPath normImage = self._norm(image) se = disk(2) dilMask = binary_dilation(mask, se) feas = self._extractLocFeas(normImage, dilMask, self.patSize) feas = np.matmul(self.principalComp, (feas - self.cluFeasAvg[:, None])) regInd = self._regionInd(normImage, dilMask, self.nRegion) for j in range(self.nRegion): temp = np.append( temp, fisher(feas[::, regInd == j], self.means.transpose(), self.covariances.transpose(), self.priors.transpose(), improved=True)) testFeaMat = np.append(testFeaMat, temp[:, np.newaxis], axis=1) return testFeaMat
def test_fisher_encoding(): N = 21 K = 3 D = 5 x = np.zeros(D * N, dtype=np.float32) for i in range(D * N): x[i] = i x = x.reshape(D, N) mu = np.zeros(D * K, dtype=np.float32) for i in range(D * K): mu[i] = i mu = mu.reshape(D, K) sigma2 = np.ones((D, K), dtype=np.float32) prior = (1.0 / K) * np.ones(K, dtype=np.float32) observed_enc = fisher(x, mu, sigma2, prior, verbose=True) # expected result obtained from running vl_fisher_encode from a C program expected_enc = np.array([0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 0.000000000, 70.519210815, 70.519210815, 70.519210815, 70.519210815, 70.519210815, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, -0.058321182, 3073.876220703, 3073.876220703, 3073.876220703, 3073.876220703, 3073.876220703], dtype=np.float32) assert_allclose(expected_enc, observed_enc)
def fv_encoder(datas, N, K, D): datas = features_norm(datas, N) # [N, D] datas = np.array(datas).transpose() # [D, N] mu, sigma2, prior = gain_mu_sigma_prior(K, D) # [D, K] observed_enc = fisher(datas.astype('float32'), mu, sigma2, prior, verbose=False) return observed_enc
def fv_encoding(gmm, sample): means = np.float32(np.transpose(gmm.means_)) covars = np.float32(np.transpose(gmm.covars_)) priors = np.float32(np.transpose(gmm.weights_)) sample = np.float32(sample) encoded_sample = fisher(sample, means, covars, priors, verbose=False, improved=True) return encoded_sample
def generate_fisher_vector(x, _means, _covariances, _priors, _improved=True, _normalized=False): return fisher(np.transpose(x.astype(dtype='float32')), means=np.transpose(_means.astype(dtype='float32')), covariances=np.transpose( _covariances.astype(dtype='float32')), priors=np.transpose(_priors.astype(dtype='float32')), improved=_improved, normalized=_normalized)
def test_fisher_dimension(): N = 1000 K = 512 D = 128 x = np.random.uniform(size=(D, N)).astype(np.float32) means = np.random.uniform(size=(D, K)).astype(np.float32) covariances = np.random.uniform(size=(D, K)).astype(np.float32) priors = np.random.uniform(size=(K,)).astype(np.float32) priors /= np.linalg.norm(priors) enc = fisher(x, means, covariances, priors, verbose=True) expected = 2 * K * D observed = len(enc) assert(expected == observed)
def __fisher_vector(self, x): """Compute Fisher vector from feature vector x.""" means, covars, priors = self.gmm_ x = x.transpose() return fisher(x, means, covars, priors, improved=True)
### generate fisher vector observed_t0 = FisherEncode(x0.T, mu.T, sigma2.T, prior, improved=False, fast=False) observed_t = FisherEncode(x.T, mu.T, sigma2.T, prior, improved=False, fast=False) observed_vl0 = fisher(x0, mu, sigma2, prior, False, improved=False, fast=False, verbose=True) observed_vl = fisher(x, mu, sigma2, prior, False, improved=False, fast=False, verbose=True)
def _fisher_vector(self, x): params = self.gmm.means_, self.gmm.covariances_, self.gmm.weights_ # convert to float32 and transpose params = [p.astype(np.float32).T for p in params] return fisher(x.astype(np.float32).T, *params, improved=True)