def encode(self, x, verbose=False):
'''Generate the Fisher vector representation of an image or its feature vectors.
Parameters
----------
x : array, shape (H,W,C) or (M,D)
An input image, or a set of local feature vectors for the image.
If (H,W,D), the (M,D) feature vectors are extracted using the CNN.
verbose : bool, optional
Whether to print FV information from VLFeat call, default=False.
Returns
-------
encoding : array, shape (2*k*D,)
FV encoding of img, where k=num_clusters in GMM and D is CNN output depth.
'''
assert hasattr(self,
'means'), 'Must call `fit` before generating encodings'
if x.ndim == 2 and x.shape[-1] == self.D:
feat = x
elif x.ndim == 3:
feat = self._localfeatures(x)
else:
raise ValueError(
'`img` to be encoded must be 3D image or 2D feature vectors')
return fisher.fisher(feat.T,
self.means.T,
self.covars.T,
self.priors.T,
improved=True,
fast=False,
verbose=verbose)
def get_fv(image_size,
means,
covariance,
priors,
image=None,
sift_dict=None,
image_p=None):
if sift_dict is None or image_p is None or image_p not in sift_dict.keys():
kps, descs = face_descriptors.get_sift_desc(image, start=0)
else:
kps_data, descs = sift_dict[image_p]
kps = face_descriptors.desirialize_keypoints(kps_data)
# descs = StandardScaler().fit_transform(descs)
pca = PCA(n_components=64)
descs = pca.fit_transform(descs)
descs = embed_spatial_info(kps, descs, image_size)
# print(descs.transpose().shape,priors.shape, means.shape, covariance.shape)
# D: dimension(SIFT 128-64 dim),n: number of descriptors per image, N: number of GMM, F: Features (SIFT 128-64 dim)
# fisher([D,n],[F,N],[F,N],[N]) accepts dim
fv_encoding = fisher(descs.transpose(),
means,
covariance,
priors,
normalized=True,
square_root=True,
fast=True,
improved=True)
return fv_encoding
def _featuresExtraction(self):
testFeaMat = np.empty(
((self.compIndex) * 2 * self.dictSize * self.nRegion, 0),
dtype='float32')
temp = []
normImage = np.empty([2, 2])
image = pydicom.filereader.dcmread(self.filePath).pixel_array
mask = self.tumorMaskPath
normImage = self._norm(image)
se = disk(2)
dilMask = binary_dilation(mask, se)
feas = self._extractLocFeas(normImage, dilMask, self.patSize)
feas = np.matmul(self.principalComp, (feas - self.cluFeasAvg[:, None]))
regInd = self._regionInd(normImage, dilMask, self.nRegion)
for j in range(self.nRegion):
temp = np.append(
temp,
fisher(feas[::, regInd == j],
self.means.transpose(),
self.covariances.transpose(),
self.priors.transpose(),
improved=True))
testFeaMat = np.append(testFeaMat, temp[:, np.newaxis], axis=1)
return testFeaMat
def test_fisher_encoding():
N = 21
K = 3
D = 5
x = np.zeros(D * N, dtype=np.float32)
for i in range(D * N):
x[i] = i
x = x.reshape(D, N)
mu = np.zeros(D * K, dtype=np.float32)
for i in range(D * K):
mu[i] = i
mu = mu.reshape(D, K)
sigma2 = np.ones((D, K), dtype=np.float32)
prior = (1.0 / K) * np.ones(K, dtype=np.float32)
observed_enc = fisher(x, mu, sigma2, prior, verbose=True)
# expected result obtained from running vl_fisher_encode from a C program
expected_enc = np.array([0.000000000, 0.000000000, 0.000000000,
0.000000000, 0.000000000, 0.000000000,
0.000000000, 0.000000000, 0.000000000,
0.000000000, 70.519210815, 70.519210815,
70.519210815, 70.519210815, 70.519210815,
-0.058321182, -0.058321182, -0.058321182,
-0.058321182, -0.058321182, -0.058321182,
-0.058321182, -0.058321182, -0.058321182,
-0.058321182, 3073.876220703, 3073.876220703,
3073.876220703, 3073.876220703, 3073.876220703],
dtype=np.float32)
assert_allclose(expected_enc, observed_enc)
def fv_encoder(datas, N, K, D):
datas = features_norm(datas, N) # [N, D]
datas = np.array(datas).transpose() # [D, N]
mu, sigma2, prior = gain_mu_sigma_prior(K, D) # [D, K]
observed_enc = fisher(datas.astype('float32'),
mu,
sigma2,
prior,
verbose=False)
return observed_enc
def fv_encoding(gmm, sample):
means = np.float32(np.transpose(gmm.means_))
covars = np.float32(np.transpose(gmm.covars_))
priors = np.float32(np.transpose(gmm.weights_))
sample = np.float32(sample)
encoded_sample = fisher(sample,
means,
covars,
priors,
verbose=False,
improved=True)
return encoded_sample
def generate_fisher_vector(x,
_means,
_covariances,
_priors,
_improved=True,
_normalized=False):
return fisher(np.transpose(x.astype(dtype='float32')),
means=np.transpose(_means.astype(dtype='float32')),
covariances=np.transpose(
_covariances.astype(dtype='float32')),
priors=np.transpose(_priors.astype(dtype='float32')),
improved=_improved,
normalized=_normalized)
def test_fisher_dimension():
N = 1000
K = 512
D = 128
x = np.random.uniform(size=(D, N)).astype(np.float32)
means = np.random.uniform(size=(D, K)).astype(np.float32)
covariances = np.random.uniform(size=(D, K)).astype(np.float32)
priors = np.random.uniform(size=(K,)).astype(np.float32)
priors /= np.linalg.norm(priors)
enc = fisher(x, means, covariances, priors, verbose=True)
expected = 2 * K * D
observed = len(enc)
assert(expected == observed)
def __fisher_vector(self, x):
"""Compute Fisher vector from feature vector x."""
means, covars, priors = self.gmm_
x = x.transpose()
return fisher(x, means, covars, priors, improved=True)
### generate fisher vector
observed_t0 = FisherEncode(x0.T,
mu.T,
sigma2.T,
prior,
improved=False,
fast=False)
observed_t = FisherEncode(x.T,
mu.T,
sigma2.T,
prior,
improved=False,
fast=False)
observed_vl0 = fisher(x0,
mu,
sigma2,
prior,
False,
improved=False,
fast=False,
verbose=True)
observed_vl = fisher(x,
mu,
sigma2,
prior,
False,
improved=False,
fast=False,
verbose=True)
def _fisher_vector(self, x):
params = self.gmm.means_, self.gmm.covariances_, self.gmm.weights_
# convert to float32 and transpose
params = [p.astype(np.float32).T for p in params]
return fisher(x.astype(np.float32).T, *params, improved=True)
