def sstats_to_sqrt_features(ss, gmm):
""" Converts sufficient statistics into an approximated form of sqrt
Fisher vectors.
Inputs
------
ss: array [N * (K + 2 * D * K), ]
Sufficient statistics.
gmm: instance of the yael object, gmm
Gaussian mixture model.
Output
------
fv: array [N, K + 2 * D * K]
Fisher vectors for each of the N sufficient statistics.
"""
K = gmm.k
D = gmm.d
ss = ss.reshape(-1, K + 2 * K * D)
N = ss.shape[0]
# Get parameters and reshape them.
pi = yael.fvec_to_numpy(gmm.w, K) # 1xK
mu = (yael.fvec_to_numpy(gmm.mu,
K * D).reshape(D, K,
order='F')[np.newaxis]) # 1xDxK
sigma = (yael.fvec_to_numpy(gmm.sigma,
K * D).reshape(D, K,
order='F')[np.newaxis]) # 1xDxK
# Get each part of the sufficient statistics.
Q_sum = ss[:, :K] # NxK (then Nx1xK)
Q_xx = ss[:, K:K + D * K] # NxKD
Q_xx_2 = ss[:, K + D * K:K + 2 * D * K] # NxKD
# Sqrt the soft counts.
sqrtQ = np.sqrt(Q_sum)
# Compute the gradients.
d_pi = sqrtQ
d_mu = Q_xx - (Q_sum[:, np.newaxis] * mu).reshape(N, D * K, order='F')
d_mu = d_mu / np.repeat(sqrtQ, D)
d_sigma = (-Q_xx_2 -
(Q_sum[:, np.newaxis] * mu**2).reshape(N, D * K, order='F') +
(Q_sum[:, np.newaxis] * sigma).reshape(N, D * K, order='F') +
2 * Q_xx * mu.reshape(1, K * D, order='F')) / np.repeat(
sqrtQ, D)
# Glue everything together.
fv = np.hstack((d_pi, d_mu, d_sigma))
return fv
def descriptors_to_fisher_vector(xx, gmm, **kwargs):
"""Computes the Fisher vector on a set of descriptors.
Parameters
----------
xx: array_like, shape (N, D) or (D, )
The set of descriptors
gmm: instance of yael object gmm
Gauassian mixture model of the descriptors.
Returns
-------
fv: array_like, shape (K + 2 * D * K, )
Fisher vector (derivatives with respect to the mixing weights, means
and variances) of the given descriptors.
Reference
---------
J. Krapac, J. Verbeek, F. Jurie. Modeling Spatial Layout with Fisher
Vectors for Image Categorization. In ICCV, 2011.
http://hal.inria.fr/docs/00/61/94/03/PDF/final.r1.pdf
"""
# yael assumes that the data is in C-contiguous format.
xx = np.ascontiguousarray(np.atleast_2d(xx))
N = xx.shape[0]
K = gmm.k
D = gmm.d
# Compute posterior probabilities using yael.
Q = gmm_predict_proba(xx, gmm) # NxK
# Get parameters and reshape them.
pi = yael.fvec_to_numpy(gmm.w, K) # 1xK
mu = yael.fvec_to_numpy(gmm.mu, K * D).reshape(K, D) # DxK
sigma = yael.fvec_to_numpy(gmm.sigma, K * D).reshape(K, D) # DxK
# Compute the sufficient statistics of descriptors.
Q_sum = np.sum(Q, 0)[:, np.newaxis] / N # Kx1
Q_xx = np.dot(Q.T, xx) / N # KxD
Q_xx_2 = np.dot(Q.T, xx ** 2) / N # KxD
# Compute derivatives with respect to mixing weights, means and variances.
d_pi = Q_sum.squeeze() - pi
d_mu = Q_xx - Q_sum * mu
d_sigma = - Q_xx_2 - Q_sum * mu ** 2 + Q_sum * sigma + 2 * Q_xx * mu
# Merge derivatives into a vector.
return np.hstack((d_pi, d_mu.flatten(), d_sigma.flatten()))
def sstats_to_sqrt_features(ss, gmm):
""" Converts sufficient statistics into an approximated form of sqrt
Fisher vectors.
Inputs
------
ss: array [N * (K + 2 * D * K), ]
Sufficient statistics.
gmm: instance of the yael object, gmm
Gaussian mixture model.
Output
------
fv: array [N, K + 2 * D * K]
Fisher vectors for each of the N sufficient statistics.
"""
K = gmm.k
D = gmm.d
ss = ss.reshape(-1, K + 2 * K * D)
N = ss.shape[0]
# Get parameters and reshape them.
pi = yael.fvec_to_numpy(gmm.w, K) # 1xK
mu = (yael.fvec_to_numpy(gmm.mu, K * D)
.reshape(D, K, order='F')[np.newaxis]) # 1xDxK
sigma = (yael.fvec_to_numpy(gmm.sigma, K * D).
reshape(D, K, order='F')[np.newaxis]) # 1xDxK
# Get each part of the sufficient statistics.
Q_sum = ss[:, :K] # NxK (then Nx1xK)
Q_xx = ss[:, K:K + D * K] # NxKD
Q_xx_2 = ss[:, K + D * K:K + 2 * D * K] # NxKD
# Sqrt the soft counts.
sqrtQ = np.sqrt(Q_sum)
# Compute the gradients.
d_pi = sqrtQ
d_mu = Q_xx - (Q_sum[:, np.newaxis] * mu).reshape(N, D * K, order='F')
d_mu = d_mu / np.repeat(sqrtQ, D)
d_sigma = (
- Q_xx_2
- (Q_sum[:, np.newaxis] * mu ** 2).reshape(N, D * K, order='F')
+ (Q_sum[:, np.newaxis] * sigma).reshape(N, D * K, order='F')
+ 2 * Q_xx * mu.reshape(1, K * D, order='F')) / np.repeat(sqrtQ, D)
# Glue everything together.
fv = np.hstack((d_pi, d_mu, d_sigma))
return fv
def readXVecsFromOpenedFile(file, dim, count, extension):
if extension == 'bvecs':
points = yael.fvec_new(dim * count)
yael.b2fvecs_fread(file, points, count)
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
elif extension == 'fvecs':
points = yael.fvec_new(dim * count)
yael.fvecs_fread(file, points, count, dim)
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
raise Exception('Bad file extension!')
def readXVecsFromOpenedFile(file, dim, count, extension):
if extension == 'bvecs':
points = yael.fvec_new(dim * count)
yael.b2fvecs_fread(file, points, count)
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
elif extension == 'fvecs':
points = yael.fvec_new(dim * count)
yael.fvecs_fread(file, points, count, dim)
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
raise Exception('Bad file extension!')
def descs_to_sstats(xx, gmm):
""" Converts the descriptors to sufficient statistics.
Inputs
------
xx: array [nr_descs, nr_dimensions]
Data matrix containing the descriptors.
gmm: yael.gmm instance
Mixture of Gaussian object.
Output
------
sstats: array [nr_clusters + 2 * nr_clusters * nr_dimensions, ]
Concatenation of the averaged posterior probabilities `Q_sum`, the
first moment `Q_xx` and second-order moment `Q_xx_2`.
"""
xx = np.atleast_2d(xx)
N = xx.shape[0]
K = gmm.k
D = gmm.d
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K) # NxK
yael.free(Q_yael)
# Compute statistics.
sstats = np.zeros(K + 2 * K * D, dtype=np.float32)
sstats[:K] = np.sum(Q, 0) / N # 1xK
sstats[K:K + K * D] = dot(Q.T, xx).flatten() / N # 1xKD
sstats[K + K * D:K + 2 * K * D] = dot(Q.T, xx**2).flatten() / N # 1xKD
return sstats
def descs_to_sstats(xx, gmm):
""" Converts the descriptors to sufficient statistics.
Inputs
------
xx: array [nr_descs, nr_dimensions]
Data matrix containing the descriptors.
gmm: yael.gmm instance
Mixture of Gaussian object.
Output
------
Q_sum: array [nr_clusters, ]
Averaged posterior probabilities.
"""
K = gmm.k
N = xx.shape[0]
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K)
yael.free(Q_yael)
# Compute statistics.
Q_sum = sum(Q, 0) / N # 1xK
return np.array(Q_sum, dtype=np.float32)
def descs_to_sstats(xx, gmm):
""" Converts the descriptors to sufficient statistics.
Inputs
------
xx: array [nr_descs, nr_dimensions]
Data matrix containing the descriptors.
gmm: yael.gmm instance
Mixture of Gaussian object.
Output
------
sstats: array [nr_clusters + 2 * nr_clusters * nr_dimensions, ]
Concatenation of the averaged posterior probabilities `Q_sum`, the
first moment `Q_xx` and second-order moment `Q_xx_2`.
"""
xx = np.atleast_2d(xx)
N = xx.shape[0]
K = gmm.k
D = gmm.d
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K) # NxK
yael.free(Q_yael)
# Compute statistics.
sstats = np.zeros(K + 2 * K * D, dtype=np.float32)
sstats[: K] = np.sum(Q, 0) / N # 1xK
sstats[K: K + K * D] = dot(Q.T, xx).flatten() / N # 1xKD
sstats[K + K * D: K + 2 * K * D] = dot(
Q.T, xx ** 2).flatten() / N # 1xKD
return sstats
def descs_to_sstats(xx, gmm):
""" Converts the descriptors to sufficient statistics.
Inputs
------
xx: array [nr_descs, nr_dimensions]
Data matrix containing the descriptors.
gmm: yael.gmm instance
Mixture of Gaussian object.
Output
------
Q_sum: array [nr_clusters, ]
Averaged posterior probabilities.
"""
K = gmm.k
N = xx.shape[0]
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K)
yael.free(Q_yael)
# Compute statistics.
Q_sum = sum(Q, 0) / N # 1xK
return np.array(Q_sum, dtype=np.float32)
def fit(self, X):
n_samples, self.n_features = X.shape
yael_X = yael.numpy_to_fvec_ref(X)
yael_gmm = yael.gmm_learn(
self.n_features, n_samples, self.n_components, self.n_iter, yael_X,
self.n_threads, 0, self.n_init,
yael.GMM_FLAGS_W | yael.GMM_FLAGS_SIGMA | yael.GMM_FLAGS_MU)
self.means_ = yael.fvec_to_numpy(
yael_gmm.mu, self.n_components * self.n_features).reshape(
(self.n_components, self.n_features))
self.covars_ = yael.fvec_to_numpy(
yael_gmm.sigma, self.n_components * self.n_features).reshape(
(self.n_components, self.n_features))
self.weights_ = yael.fvec_to_numpy(yael_gmm.w, self.n_components)
yael.gmm_delete(yael_gmm)
def gmm_predict_proba(xx, gmm):
"""Computes posterior probabilities using yael."""
N = xx.shape[0]
K = gmm.k
Q_yael = yael.fvec_new(N * K)
yael.gmm_compute_p(
N, yael.numpy_to_fvec_ref(xx), gmm, Q_yael, yael.GMM_FLAGS_W)
Q = yael.fvec_to_numpy(Q_yael, N * K).reshape(N, K)
yael.free(Q_yael)
return Q
def readXvecs(filename, dim, count, tonumpy=True):
extension = filename.strip().split('.')[-1]
if extension == 'bvecs':
points = yael.fvec_new(dim * count)
yael.b2fvecs_read(filename, dim, count, points)
if tonumpy:
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
elif extension == 'fvecs':
points = yael.fvec_new(dim * count)
yael.fvecs_read(filename, dim, count, points)
if tonumpy:
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
elif extension == 'i8vecs':
file = open(filename, 'r')
points = np.zeros((count, dim), dtype='float32')
for i in xrange(count):
file.read(4)
points[i,:] = np.fromfile(file, np.int8, dim).astype('float32')
return points
elif extension == 'ivecs':
points = yael.ivec_new(dim * count)
yael.ivecs_fread(open(filename, 'r'), points, count, dim)
if tonumpy:
a = yael.ivec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
else:
raise Exception('Bad file extension!')
def readXvecs(filename, dim, count, tonumpy=True):
extension = filename.strip().split('.')[-1]
if extension == 'bvecs':
points = yael.fvec_new(dim * count)
yael.b2fvecs_read(filename, dim, count, points)
if tonumpy:
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
elif extension == 'fvecs':
points = yael.fvec_new(dim * count)
yael.fvecs_read(filename, dim, count, points)
if tonumpy:
a = yael.fvec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
elif extension == 'i8vecs':
file = open(filename, 'r')
points = np.zeros((count, dim), dtype='float32')
for i in xrange(count):
file.read(4)
points[i, :] = np.fromfile(file, np.int8, dim).astype('float32')
return points
elif extension == 'ivecs':
points = yael.ivec_new(dim * count)
yael.ivecs_fread(open(filename, 'r'), points, count, dim)
if tonumpy:
a = yael.ivec_to_numpy(points, (count, dim))
yael.free(points)
return a
else:
return points
else:
raise Exception('Bad file extension!')
def descs_to_spatial_sstats(xx, ll, gmm):
""" Computes spatial statistics from descriptors and their position.
Inputs
------
xx: array [N, D], required
N D-dimensional descriptors from an video (usually, after they are
processed with PCA).
ll: array [N, 3], required
Descriptor locations in an image; on each row, we have the triplet
(x, y, t).
gmm: instance of yael object gmm
Gauassian mixture object.
Output
------
ss: array [1, K + 2 * 3 * k]
Sufficient statistics in the form of a vector that concatenates
(i) the sum of posteriors, (ii) an expected value of the locations
ll under the posterior distribution Q and (iii) the second-order
moment of the locations ll under the posterior distribution Q.
"""
N = ll.shape[0]
K = gmm.k
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K)
yael.free(Q_yael)
# Compute statistics.
Q_sum = sum(Q, 0) / N # 1xK
Q_ll = dot(Q.T, ll).flatten() / N # 1x3K
Q_ll_2 = dot(Q.T, ll ** 2).flatten() / N # 1x3K
return np.array(hstack((Q_sum, Q_ll, Q_ll_2)), dtype=np.float32)
def descs_to_spatial_sstats(xx, ll, gmm):
""" Computes spatial statistics from descriptors and their position.
Inputs
------
xx: array [N, D], required
N D-dimensional descriptors from an video (usually, after they are
processed with PCA).
ll: array [N, 3], required
Descriptor locations in an image; on each row, we have the triplet
(x, y, t).
gmm: instance of yael object gmm
Gauassian mixture object.
Output
------
ss: array [1, K + 2 * 3 * k]
Sufficient statistics in the form of a vector that concatenates
(i) the sum of posteriors, (ii) an expected value of the locations
ll under the posterior distribution Q and (iii) the second-order
moment of the locations ll under the posterior distribution Q.
"""
N = ll.shape[0]
K = gmm.k
# Compute posterior probabilities using yael.
Q_yael = fvec_new(N * K)
gmm_compute_p(N, numpy_to_fvec_ref(xx), gmm, Q_yael, GMM_FLAGS_W)
Q = fvec_to_numpy(Q_yael, N * K).reshape(N, K)
yael.free(Q_yael)
# Compute statistics.
Q_sum = sum(Q, 0) / N # 1xK
Q_ll = dot(Q.T, ll).flatten() / N # 1x3K
Q_ll_2 = dot(Q.T, ll**2).flatten() / N # 1x3K
return np.array(hstack((Q_sum, Q_ll, Q_ll_2)), dtype=np.float32)
if 'fvec_to_numpy' not in dir(yael):
#globals():
print('Numpy interface not compiled in')
sys.exit(1)
print('float array')
numpy_a = np.array(list(range(5)), dtype='float32')
print(numpy_a)
print('-> Yael A')
yael_a = yael.FloatArray.acquirepointer(yael.numpy_to_fvec(numpy_a))
n = numpy_a.size
yael.fvec_print(yael_a, n)
print('-> Numpy A')
print(yael.fvec_to_numpy(yael_a, n))
print('int array')
numpy_a = np.array(list(range(5)), dtype='int32')
print(numpy_a)
print('-> Yael A2')
yael_a = yael.IntArray.acquirepointer(yael.numpy_to_ivec(numpy_a))
n = numpy_a.size
yael.ivec_print(yael_a, n)
print('-> Numpy A2')
print(yael.ivec_to_numpy(yael_a, n))
print('float array, pass by reference')
numpy_a = np.array(list(range(5)), dtype='float32')
print(numpy_a)
