def get_per_video_mbh_data(split, suffix=''):
""" Returns the 'train' or 'test' video descriptors. The `suffix` argument
can be '_morenull' to load the data with 5296 null samples.
"""
base_path = ('/home/clear/oneata/data/trecvid12/features/'
'dense5.track15mbh.small.skip_1/')
sstats_fn = os.path.join(
base_path, 'statistics_k_256', '%s%s.dat' % (split, suffix))
labels_fn = os.path.join(
base_path, 'statistics_k_256', 'labels_%s%s.info' % (split, suffix))
info_fn = os.path.join(
base_path, 'statistics_k_256', 'info_%s%s.info' % (split, suffix))
gmm_fn = os.path.join(base_path, 'gmm', 'gmm_256')
sstats = np.fromfile(sstats_fn, dtype=np.float32)
labels = np.array([tuple_label[0]
for tuple_label in cPickle.load(open(labels_fn, 'r'))])
video_names = cPickle.load(open(info_fn, 'r'))['video_names']
# Convert sufficient statistics to Fisher vectors.
gmm = gmm_read(open(gmm_fn, 'r'))
data = FVModel.sstats_to_features(sstats, gmm)
return data, labels, video_names
def load_gmm(filename):
""" Loads GMM object from file using yael. """
with open(filename, 'r') as _file:
gmm = gmm_read(_file)
return gmm
def load_gmm(filename):
""" Loads GMM object from file using yael. """
with open(filename, 'r') as _file:
gmm = gmm_read(_file)
return gmm
def get_audio_data_dan(split, K=512, ssl = None):
feature_path = ('/home/lear/oneata/data/trecvid12/features/'
'mfcc/statistics_k_%d/' % K)
gmm_path = '/home/lear/oneata/data/trecvid12/features/mfcc/gmm/gmm_%d' % K
filelist = '/home/lear/oneata/src/med12lear/data/%s.list' % split
filename = os.path.join(feature_path, split + '.raw')
if os.path.exists(filename):
video_data, video_labels, video_names = load_cache_file(filename, ssl)
return video_data, video_labels.squeeze(), video_names
# from fisher_vectors.model.fv_model import FVModel
from fv_model import sstats_to_features
video_names = []
video_labels = []
video_data = []
sstats_path = os.path.join(feature_path, 'stats.tmp', '%s.dat')
gmm = gmm_read(open(gmm_path, 'r'))
dim = gmm.k + 2 * gmm.d * gmm.k
empty = 0
with open(os.path.join(filelist), 'r') as ff:
for line in ff.readlines():
try:
sample_name, class_name = line.split()
except ValueError:
sample_name = line.strip()
class_name = 'unknown'
classno = rev_label_map[class_name]
vidname = sample_name.split('-')[0]
video_labels.append([classno])
video_names.append(vidname)
sstats_filename = sstats_path % sample_name
if os.access(sstats_filename, os.R_OK):
sstats = np.fromfile(sstats_filename, dtype=np.float32)
else:
print "Missing sstats", sstats_filename
sstats = None
if (sstats == None or np.isnan(np.max(sstats))
or len(sstats) != dim or np.sum(sstats ** 2) == 0):
warnings.warn("Empty video %s." % vidname)
fv = np.zeros((1, dim))
empty += 1
else:
sstats = np.atleast_2d(sstats)
fv = sstats_to_features(sstats, gmm)
print "%-30s" % vidname,
print "%dx%d" % sstats.shape
video_data.append(fv)
print "Empty videos: %d (out of %d)." % (empty, len(video_names))
video_data = np.array(np.vstack(video_data), dtype=np.float32)
video_labels = np.array(video_labels)
with open(filename, "w") as ff:
np.save(ff, video_data)
np.save(ff, video_labels)
cPickle.dump(video_names, ff)
return video_data, video_labels.squeeze(), video_names
def main():
def all_files_exist(feature_path, video_name, what_to_save):
return np.all([
os.path.exists(
feature_path % {
'descriptor': desc_type,
'model': MODELS[model]['full_name'],
'K': kk,
'slice_or_video': slice_or_video,
'video_name': video_name})
for kk in Ks
for desc_type in DESCRIPTOR_TYPES
for slice_or_video in what_to_save
for model in MODELS.keys()])
"""
def get_nr_frames(filelist_path, video_name):
with open(filelist_path, 'r') as ff:
for line in ff.readlines():
if line.startswith(video_name):
sample, _ = line.split()
return int(sample.split('-')[-1])
assert False, "Could not find the number of frames for video %s" % video_name
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--video', help="video name")
parser.add_argument('--videodir', default=['videos'], help="video directory (default: 'videos')")
parser.add_argument('-k', '--nr_clusters', default=['64'], nargs='+', help="number of GMM clusters.")
parser.add_argument('-s', '--split', choices=SPLITS, default='train', help="which split of the data to use.")
parser.add_argument('--redo', default=False, action='store_true')
parser.add_argument('--h3spm', default=False, action='store_true', help="use H3 spatial pyramid.")
parser.add_argument('--save', default=['video'], nargs='+', choices=('video', 'slice'), help="what to store: video descriptor or slice descriptors.")
parser.add_argument('--slice', default=None, type=int, help="slice size (in number of frames).")
parser.add_argument('--scenecut', default=None, help="Full path to scenecut file, contains shot boundaries.")
parser.add_argument('--featurepath', default=None, help="Full path to where features are saved.")
parser.add_argument('-v', '--verbose', action='count', help="verbosity level.")
args = parser.parse_args()
if args.scenecut and (not os.path.exists(args.scenecut)):
print "Scenecut file '%s' does not exist" % args.scenecut
sys.exit(1)
#BASEPATH = "/home/lear/oneata/data/thumos14"
PCA = MED_BASEDIR + "compute_descriptors/denseTrack/vocab/MED15/pca/%s/features_pca_k256.npz"
GMM = MED_BASEDIR + "compute_descriptors/denseTrack/vocab/MED15/gmm/%s/features_k256.gmm"
#video_path = os.path.join(BASEPATH, 'videos', args.split, args.video)
video_path = os.path.join(
args.videodir,
args.video)
#filelist_path = os.path.join(BASEPATH, 'filelists', '%s.list' % args.split)
"""
feature_path = os.path.join(
BASEPATH,
'features',
'dense5.track15%(descriptor)s.stab.320px',
'%(model)s_k_%(K)d',
'%(slice_or_video)s_descriptors' + ('_h3spm' if args.h3spm else ''),
'%(video_name)s.fvecs')
"""
if args.featurepath == None:
print "Feature path not specified"
sys.exit(1)
feature_path = os.path.join(
args.featurepath,
'%(descriptor)s_%(model)s_k_%(K)d_descriptor.fvecs'
)
"""
feature_path = os.path.join(
'processing',
'videos_workdir',
'%(video_name)s',
'%(descriptor)s_%(model)s_k_%(K)d_descriptor.fvecs'
)
"""
print "Video path:", video_path
print "Feature path:", feature_path
assert (args.slice is None) == ('slice' not in args.save), "Use either both the `--slice` and `--save slice` or none of them."
Ks = map(int, args.nr_clusters)
spm = (1, 3, 1) if args.h3spm else (1, 1, 1)
slice_size = args.slice or 1e5
what_to_save = [
('slice_%d' % slice_size if sv == 'slice' else sv)
for sv in args.save]
# Skip the work if all the feature files already exist.
if not args.redo and all_files_exist(feature_path, args.video, what_to_save):
if args.verbose > 0:
print "All feature file exist for video", args.video
sys.exit(0)
# Load PCA files.
pcas = {
desc_type: np.load(PCA % desc_type)
for desc_type in DESCRIPTOR_TYPES}
# Load GMM files.
gmms = {
kk: {
desc_type: yael.gmm_read(open(GMM % (desc_type), 'r'))
for desc_type in DESCRIPTOR_TYPES}
for kk in Ks}
#nr_frames = get_nr_frames(filelist_path, args.video)
#if args.verbose > 2:
# print "Video has %d frames." % nr_frames
if args.scenecut:
slices = parse_scenecutfile(args.scenecut)
else:
slices = []
#slices = np.array([150])
features, nr_descriptors = compute_features(
video_path, DESCRIPTOR_TYPES, pcas, gmms, MODELS.keys(),
spm, slices, verbose=args.verbose)
save_features(
features, nr_descriptors, args.video, feature_path, DESCRIPTOR_TYPES,
MODELS.keys(), what_to_save)
def load_sample_data(
dataset, sample, analytical_fim=False, pi_derivatives=False,
sqrt_nr_descs=False, return_info=False, encoding='fv'):
ENC_PARAMS = {
'fv': {
'suffix_enc': '',
'get_dim': lambda gmm: gmm.k * (2 * gmm.d + 1),
'sstats_to_features': FVModel.sstats_to_features,
'sstats_to_normalized_features': FVModel.sstats_to_normalized_features,
},
'sfv': {
'suffix_enc': '_sfv',
'get_dim': lambda gmm: gmm.k * (2 * 3 + 1),
'sstats_to_features': SFVModel.spatial_sstats_to_spatial_features,
},
}
if str(sample) in ('train', 'test'):
stats_file = "%s.dat" % sample
labels_file = "labels_%s.info" % sample
info_file = "info_%s.info" % sample
else:
stats_tmp = "stats.tmp%s%s/%s" % (
dataset.SUFFIX_STATS, ENC_PARAMS[encoding]['suffix_enc'], sample)
stats_file = "%s.dat" % stats_tmp
labels_file = "%s.info" % stats_tmp
info_file = "%s.info" % stats_tmp
stats_path = os.path.join(dataset.SSTATS_DIR, stats_file)
labels_path = os.path.join(dataset.SSTATS_DIR, labels_file)
info_path = os.path.join(dataset.SSTATS_DIR, info_file)
with open(dataset.GMM, 'r') as ff:
gmm = yael.gmm_read(ff)
K = gmm.k
D = ENC_PARAMS[encoding]['get_dim'](gmm)
data = np.fromfile(stats_path, dtype=np.float32)
data = data.reshape(-1, D)
counts = data[:, : K]
if analytical_fim:
data = ENC_PARAMS[encoding]['sstats_to_normalized_features'](data, gmm)
else:
data = ENC_PARAMS[encoding]['sstats_to_features'](data, gmm)
with open(labels_path, 'r') as ff:
labels = cPickle.load(ff)
with open(info_path, 'r') as ff:
info = cPickle.load(ff)
if sqrt_nr_descs:
T = info['nr_descs']
T = np.sqrt(T)[:, np.newaxis]
else:
T = 1.
if pi_derivatives or encoding == 'sfv':
# For the spatial Fisher vector encoding I drop the `pi_derivatives`,
# so I need the full vector, no matter the value of the `pi_derivates`
# parameter.
idxs = slice(D)
else:
idxs = slice(K, D)
if return_info:
return T * data[:, idxs], labels, counts, info
else:
return T * data[:, idxs], labels, counts
def load_sample_data(dataset,
sample,
analytical_fim=False,
pi_derivatives=False,
sqrt_nr_descs=False,
return_info=False,
encoding='fv'):
ENC_PARAMS = {
'fv': {
'suffix_enc': '',
'get_dim': lambda gmm: gmm.k * (2 * gmm.d + 1),
'sstats_to_features': FVModel.sstats_to_features,
'sstats_to_normalized_features':
FVModel.sstats_to_normalized_features,
},
'sfv': {
'suffix_enc': '_sfv',
'get_dim': lambda gmm: gmm.k * (2 * 3 + 1),
'sstats_to_features': SFVModel.spatial_sstats_to_spatial_features,
},
}
if str(sample) in ('train', 'test'):
stats_file = "%s.dat" % sample
labels_file = "labels_%s.info" % sample
info_file = "info_%s.info" % sample
else:
stats_tmp = "stats.tmp%s%s/%s" % (
dataset.SUFFIX_STATS, ENC_PARAMS[encoding]['suffix_enc'], sample)
stats_file = "%s.dat" % stats_tmp
labels_file = "%s.info" % stats_tmp
info_file = "%s.info" % stats_tmp
stats_path = os.path.join(dataset.SSTATS_DIR, stats_file)
labels_path = os.path.join(dataset.SSTATS_DIR, labels_file)
info_path = os.path.join(dataset.SSTATS_DIR, info_file)
with open(dataset.GMM, 'r') as ff:
gmm = yael.gmm_read(ff)
K = gmm.k
D = ENC_PARAMS[encoding]['get_dim'](gmm)
data = np.fromfile(stats_path, dtype=np.float32)
data = data.reshape(-1, D)
counts = data[:, :K]
if analytical_fim:
data = ENC_PARAMS[encoding]['sstats_to_normalized_features'](data, gmm)
else:
data = ENC_PARAMS[encoding]['sstats_to_features'](data, gmm)
with open(labels_path, 'r') as ff:
labels = cPickle.load(ff)
with open(info_path, 'r') as ff:
info = cPickle.load(ff)
if sqrt_nr_descs:
T = info['nr_descs']
T = np.sqrt(T)[:, np.newaxis]
else:
T = 1.
if pi_derivatives or encoding == 'sfv':
# For the spatial Fisher vector encoding I drop the `pi_derivatives`,
# so I need the full vector, no matter the value of the `pi_derivates`
# parameter.
idxs = slice(D)
else:
idxs = slice(K, D)
if return_info:
return T * data[:, idxs], labels, counts, info
else:
return T * data[:, idxs], labels, counts
def get_per_video_mbh_data_given_list(list_name, **kwargs):
""" Loads the Fisher vectors corresponding to the samples found in the
list specified by `list_name`.
"""
K = kwargs.get('K', 256)
sstats_path = ('/home/clear/oneata/data/trecvid12/features'
'/dense5.track15mbh.small.skip_1/statistics_k_%d' % K)
# Default base directories.
list_base_path = kwargs.get(
'list_base_path', '/home/lear/douze/src/experiments/trecvid12/data')
cache_base_path = kwargs.get(
'cache_base_path', sstats_path)
double_norm = kwargs.get('double_norm', False)
suffix = '.double_norm' if double_norm else ''
# If this file exists load them directly.
cache_filename = os.path.join(cache_base_path, 'mbh_' + list_name + suffix + '.raw')
if os.path.exists(cache_filename):
print "Loading Fisher vectors from MBH descriptors for list %s..." % (
list_name)
with open(cache_filename, 'r') as ff:
data = np.load(ff)
labels = np.load(ff)
names = cPickle.load(ff)
return data, labels, names
D = 64
data, labels, names = [], [], []
sstats_generic_name = os.path.join(sstats_path, 'stats.tmp' + suffix, '%s.dat')
list_path = os.path.join(list_base_path, list_name + '.list')
gmm_path = os.path.join(sstats_path, 'gmm_%d' % K)
gmm = gmm_read(open(gmm_path, 'r'))
# Get descriptors for the files in list.
print "Creating cache file from list %s..." % list_name
for line in open(list_path, 'r'):
fields = line.split()
sstats_filename = fields[0]
video_name = sstats_filename.split('-')[0]
sys.stdout.write("%s\t\r" % video_name)
try:
class_name = fields[1]
except IndexError:
class_name = 'unknown'
try:
sstats = np.fromfile(sstats_generic_name % sstats_filename,
dtype=np.float32)
if double_norm:
fv = sstats
else:
fv = FVModel.sstats_to_features(sstats, gmm)
except IOError:
print ('Sufficient statistics for video %s are missing;'
'replacing with zeros.') % video_name
fv = np.zeros(K + 2 * K * D)
data.append(fv)
names.append(video_name)
labels.append(rev_label_map[class_name])
data = np.vstack(data)
labels = np.array(labels)
assert data.shape[0] == len(labels), "Data size doesn't match nr of labels"
# Cache results to file.
with open(cache_filename, 'w') as ff:
np.save(ff, data)
np.save(ff, labels)
cPickle.dump(names, ff)
return data, labels, names
