def __generate_gmm(self, train_frames_features):
"""
Train Gaussian Mixture for process fisher vectors.
Return the fitted GMM
"""
if self.fit_by_bic:
if self.verbose:
print("---- Generate GMM with fitting by BIC... ----")
gmm = FisherVectorGMM(
covariance_type=self.covariance_type).fit_by_bic(
X=train_frames_features,
choices_n_kernels=self.n_kernels,
verbose=self.verbose)
n_kernels_current_GMM = len(gmm.means)
if self.fit_by_bic and isinstance(self.threshold_neutral, list):
self.threshold_neutral = self.threshold_neutral[
self.n_kernels.index(n_kernels_current_GMM)]
self.n_kernels = n_kernels_current_GMM
return gmm
else:
if self.verbose:
print("---- Generate GMM with " + str(self.n_kernels) +
" kernels... ----")
return FisherVectorGMM(n_kernels=self.n_kernels,
covariance_type=self.covariance_type).fit(
X=train_frames_features, verbose=False)
def test_with_gaussian_samples_image(data,n_kernels):
test_data = np.array(data)
test_data = test_data.reshape([test_data.shape[0],-1,1])
print(test_data.shape)
fv_gmm = FisherVectorGMM(n_kernels=n_kernels).fit(test_data)
n_test_videos = len(data)
fv = fv_gmm.predict(test_data[:n_test_videos])
print(fv.shape)
return fv
def test_with_gaussian_samples_video(self): np.random.seed(22) shape = [200, 15, 10, 30] test_data = np.concatenate([np.random.normal(np.zeros(30), size=shape), np.random.normal(np.ones(30), size=shape)], axis=0) n_kernels = 2 fv_gmm = FisherVectorGMM(n_kernels=n_kernels).fit(test_data) n_test_videos = 20 fv = fv_gmm.predict(test_data[:n_test_videos]) self.assertEquals(fv.shape, (n_test_videos, 15, 2 * n_kernels, 30))
def test_with_gaussian_samples_image_bic(self): np.random.seed(23) shape = [200, 10, 30] test_data = np.concatenate([np.random.normal(-np.ones(30), size=shape), np.random.normal(np.ones(30), size=shape)], axis=0) n_kernels = 2 fv_gmm = FisherVectorGMM().fit_by_bic(test_data, choices_n_kernels=[2,7]) n_test_videos = 20 fv = fv_gmm.predict(test_data[:n_test_videos]) self.assertEquals(fv.shape, (n_test_videos, 2 * n_kernels, 30)) self.assertEquals(fv_gmm.n_kernels, 2)
def test_dump_and_load(self):
np.random.seed(22)
shape = [200, 10, 30]
test_data = np.concatenate([np.random.normal(np.zeros(30), size=shape), np.random.normal(np.ones(30), size=shape)],
axis=0)
n_kernels = 2
pickle_path = ".test_fv_gmm.pickle"
fv_gmm = FisherVectorGMM(n_kernels=n_kernels).fit(test_data, model_dump_path=pickle_path)
fv_gmm_loaded = FisherVectorGMM.load_from_pickle(pickle_path)
os.remove(pickle_path)
covars_match = np.all(fv_gmm.covars == fv_gmm_loaded.covars)
means_match = np.all(fv_gmm.means == fv_gmm_loaded.means)
self.assertTrue(covars_match and means_match)
def trainFisherVectorGMM(self,
features,
by_bic=True,
model_dump_path=None,
n_kernels=50):
"""
Trains a GMM for Fisher Vectors on the given features.
:param features: features as ndarray of shape (n_videos, n_frames, n_descriptors_per_image, n_dim_descriptor) and labels (list) of videos
:param by_bic: denotes whether the gmm fit is chosen based on the lowest BIC
:param n_kernels: provide if not fitted by bic (bic-method chooses from a fixed set of n_kernels)
:return: the fitted object of type FisherVectorGMM
"""
if model_dump_path is None:
model_dump_path = self.getDumpFileName('model')
self.logger.info('Started training FisherVector GMM')
fv_gmm = FisherVectorGMM(n_kernels=n_kernels)
if by_bic:
fv_gmm.fit_by_bic(features, model_dump_path=model_dump_path)
else:
fv_gmm.fit(features, model_dump_path=model_dump_path)
assert fv_gmm.fitted
self.logger.info(
'Finished training FisherVector GMM. Dumped model to {}'.format(
model_dump_path))
return fv_gmm
def fisher_vector(sample_path, file_path, save_path):
gmm = np.load(sample_path)
gmm_reshape = np.reshape(gmm, (1, gmm.shape[0], gmm.shape[1]))
print(gmm_reshape.shape)
fv_gmm = FisherVectorGMM(n_kernels=64).fit(gmm_reshape)
# fv_gmm = FisherVectorGMM().fit_by_bic(gmm_reshape, choices_n_kernels=[2,4,8,16,32,64])
for dirpath, dirnames, filenames in os.walk(file_path):
for filename in filenames:
fullpath = os.path.join(dirpath, filename)
image_data = np.load(fullpath)
print(filename)
image_data_reshape = np.reshape(
image_data, (1, image_data.shape[0], image_data.shape[1]))
print(image_data_reshape.shape)
fv = fv_gmm.predict(image_data_reshape)
fv_reshape = np.reshape(fv, (1, fv.shape[1] * fv.shape[2]))
print(fv_reshape.shape)
data1 = pd.DataFrame(fv_reshape)
data1.to_csv(save_path + '/' + filename + '.csv',
mode='a',
header=False,
index=False)
def generate_gmm(input_folder, N):
words = np.concatenate([
folder_descriptors(folder) for folder in glob.glob(input_folder + '/*')
])
print("Training GMM of size", N)
fv_gmm = FisherVectorGMM(n_kernels=N).fit(words)
#Throw away gaussians with weights that are too small:
#th = 1.0 / N
#Post process the GMM
# th = 0.1/N
# means = np.float32([m for k,m in zip(range(0, len(weights)), means) if weights[k] > th])
# covs = np.float32([m for k,m in zip(range(0, len(weights)), covs) if weights[k] > th])
# weights = np.float32([m for k,m in zip(range(0, len(weights)), weights) if weights[k] > th])
# np.save("means.gmm", means)
# np.save("covs.gmm", covs)
# np.save("weights.gmm", weights)
# return means, covs, weights
return fv_gmm
pattern = '*.wav'
for entry in list_of_files:
if fnmatch.fnmatch(entry, pattern):
fs, signal = wav.read('audio/' + entry)
signal_preemphasized = speechpy.processing.preemphasis(signal,
cof=0.97)
mfcc = speechpy.feature.mfcc(signal_preemphasized,
sampling_frequency=fs,
frame_length=0.020,
frame_stride=0.01,
num_filters=40,
fft_length=512,
low_frequency=0,
high_frequency=None)
mfcc_cmvn = speechpy.processing.cmvnw(mfcc,
win_size=301,
variance_normalization=True)
deriv = speechpy.processing.derivative_extraction(mfcc_cmvn, 1)
deriv_2 = speechpy.processing.derivative_extraction(deriv, 1)
mfcc_feature_cube = speechpy.feature.extract_derivative_feature(mfcc)
new = np.concatenate((mfcc_cmvn, deriv, deriv_2), axis=1)
shape = [3687, 13, 0] #[3687, 13, 3] # e.g. SIFT image features
image_data = mfcc_feature_cube
fv_gmm = FisherVectorGMM(n_kernels=20).fit(image_data)
#fv_gmm = FisherVectorGMM().fit_by_bic(image_data, choices_n_kernels=[2,5,10,20])
image_data_test = image_data[:
20] # use a fraction of the data to compute the fisher vectors
fv = fv_gmm.predict(image_data_test)
np.savetxt(entry + "_fishers.txt", fv_gmm)
#print(fv)