def get_per_video_mbh_data(split, suffix=''): """ Returns the 'train' or 'test' video descriptors. The `suffix` argument can be '_morenull' to load the data with 5296 null samples. """ base_path = ('/home/clear/oneata/data/trecvid12/features/' 'dense5.track15mbh.small.skip_1/') sstats_fn = os.path.join( base_path, 'statistics_k_256', '%s%s.dat' % (split, suffix)) labels_fn = os.path.join( base_path, 'statistics_k_256', 'labels_%s%s.info' % (split, suffix)) info_fn = os.path.join( base_path, 'statistics_k_256', 'info_%s%s.info' % (split, suffix)) gmm_fn = os.path.join(base_path, 'gmm', 'gmm_256') sstats = np.fromfile(sstats_fn, dtype=np.float32) labels = np.array([tuple_label[0] for tuple_label in cPickle.load(open(labels_fn, 'r'))]) video_names = cPickle.load(open(info_fn, 'r'))['video_names'] # Convert sufficient statistics to Fisher vectors. gmm = gmm_read(open(gmm_fn, 'r')) data = FVModel.sstats_to_features(sstats, gmm) return data, labels, video_names
def load_gmm(filename): """ Loads GMM object from file using yael. """ with open(filename, 'r') as _file: gmm = gmm_read(_file) return gmm
def get_audio_data_dan(split, K=512, ssl = None): feature_path = ('/home/lear/oneata/data/trecvid12/features/' 'mfcc/statistics_k_%d/' % K) gmm_path = '/home/lear/oneata/data/trecvid12/features/mfcc/gmm/gmm_%d' % K filelist = '/home/lear/oneata/src/med12lear/data/%s.list' % split filename = os.path.join(feature_path, split + '.raw') if os.path.exists(filename): video_data, video_labels, video_names = load_cache_file(filename, ssl) return video_data, video_labels.squeeze(), video_names # from fisher_vectors.model.fv_model import FVModel from fv_model import sstats_to_features video_names = [] video_labels = [] video_data = [] sstats_path = os.path.join(feature_path, 'stats.tmp', '%s.dat') gmm = gmm_read(open(gmm_path, 'r')) dim = gmm.k + 2 * gmm.d * gmm.k empty = 0 with open(os.path.join(filelist), 'r') as ff: for line in ff.readlines(): try: sample_name, class_name = line.split() except ValueError: sample_name = line.strip() class_name = 'unknown' classno = rev_label_map[class_name] vidname = sample_name.split('-')[0] video_labels.append([classno]) video_names.append(vidname) sstats_filename = sstats_path % sample_name if os.access(sstats_filename, os.R_OK): sstats = np.fromfile(sstats_filename, dtype=np.float32) else: print "Missing sstats", sstats_filename sstats = None if (sstats == None or np.isnan(np.max(sstats)) or len(sstats) != dim or np.sum(sstats ** 2) == 0): warnings.warn("Empty video %s." % vidname) fv = np.zeros((1, dim)) empty += 1 else: sstats = np.atleast_2d(sstats) fv = sstats_to_features(sstats, gmm) print "%-30s" % vidname, print "%dx%d" % sstats.shape video_data.append(fv) print "Empty videos: %d (out of %d)." % (empty, len(video_names)) video_data = np.array(np.vstack(video_data), dtype=np.float32) video_labels = np.array(video_labels) with open(filename, "w") as ff: np.save(ff, video_data) np.save(ff, video_labels) cPickle.dump(video_names, ff) return video_data, video_labels.squeeze(), video_names
def main(): def all_files_exist(feature_path, video_name, what_to_save): return np.all([ os.path.exists( feature_path % { 'descriptor': desc_type, 'model': MODELS[model]['full_name'], 'K': kk, 'slice_or_video': slice_or_video, 'video_name': video_name}) for kk in Ks for desc_type in DESCRIPTOR_TYPES for slice_or_video in what_to_save for model in MODELS.keys()]) """ def get_nr_frames(filelist_path, video_name): with open(filelist_path, 'r') as ff: for line in ff.readlines(): if line.startswith(video_name): sample, _ = line.split() return int(sample.split('-')[-1]) assert False, "Could not find the number of frames for video %s" % video_name """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--video', help="video name") parser.add_argument('--videodir', default=['videos'], help="video directory (default: 'videos')") parser.add_argument('-k', '--nr_clusters', default=['64'], nargs='+', help="number of GMM clusters.") parser.add_argument('-s', '--split', choices=SPLITS, default='train', help="which split of the data to use.") parser.add_argument('--redo', default=False, action='store_true') parser.add_argument('--h3spm', default=False, action='store_true', help="use H3 spatial pyramid.") parser.add_argument('--save', default=['video'], nargs='+', choices=('video', 'slice'), help="what to store: video descriptor or slice descriptors.") parser.add_argument('--slice', default=None, type=int, help="slice size (in number of frames).") parser.add_argument('--scenecut', default=None, help="Full path to scenecut file, contains shot boundaries.") parser.add_argument('--featurepath', default=None, help="Full path to where features are saved.") parser.add_argument('-v', '--verbose', action='count', help="verbosity level.") args = parser.parse_args() if args.scenecut and (not os.path.exists(args.scenecut)): print "Scenecut file '%s' does not exist" % args.scenecut sys.exit(1) #BASEPATH = "/home/lear/oneata/data/thumos14" PCA = MED_BASEDIR + "compute_descriptors/denseTrack/vocab/MED15/pca/%s/features_pca_k256.npz" GMM = MED_BASEDIR + "compute_descriptors/denseTrack/vocab/MED15/gmm/%s/features_k256.gmm" #video_path = os.path.join(BASEPATH, 'videos', args.split, args.video) video_path = os.path.join( args.videodir, args.video) #filelist_path = os.path.join(BASEPATH, 'filelists', '%s.list' % args.split) """ feature_path = os.path.join( BASEPATH, 'features', 'dense5.track15%(descriptor)s.stab.320px', '%(model)s_k_%(K)d', '%(slice_or_video)s_descriptors' + ('_h3spm' if args.h3spm else ''), '%(video_name)s.fvecs') """ if args.featurepath == None: print "Feature path not specified" sys.exit(1) feature_path = os.path.join( args.featurepath, '%(descriptor)s_%(model)s_k_%(K)d_descriptor.fvecs' ) """ feature_path = os.path.join( 'processing', 'videos_workdir', '%(video_name)s', '%(descriptor)s_%(model)s_k_%(K)d_descriptor.fvecs' ) """ print "Video path:", video_path print "Feature path:", feature_path assert (args.slice is None) == ('slice' not in args.save), "Use either both the `--slice` and `--save slice` or none of them." Ks = map(int, args.nr_clusters) spm = (1, 3, 1) if args.h3spm else (1, 1, 1) slice_size = args.slice or 1e5 what_to_save = [ ('slice_%d' % slice_size if sv == 'slice' else sv) for sv in args.save] # Skip the work if all the feature files already exist. if not args.redo and all_files_exist(feature_path, args.video, what_to_save): if args.verbose > 0: print "All feature file exist for video", args.video sys.exit(0) # Load PCA files. pcas = { desc_type: np.load(PCA % desc_type) for desc_type in DESCRIPTOR_TYPES} # Load GMM files. gmms = { kk: { desc_type: yael.gmm_read(open(GMM % (desc_type), 'r')) for desc_type in DESCRIPTOR_TYPES} for kk in Ks} #nr_frames = get_nr_frames(filelist_path, args.video) #if args.verbose > 2: # print "Video has %d frames." % nr_frames if args.scenecut: slices = parse_scenecutfile(args.scenecut) else: slices = [] #slices = np.array([150]) features, nr_descriptors = compute_features( video_path, DESCRIPTOR_TYPES, pcas, gmms, MODELS.keys(), spm, slices, verbose=args.verbose) save_features( features, nr_descriptors, args.video, feature_path, DESCRIPTOR_TYPES, MODELS.keys(), what_to_save)
def load_sample_data( dataset, sample, analytical_fim=False, pi_derivatives=False, sqrt_nr_descs=False, return_info=False, encoding='fv'): ENC_PARAMS = { 'fv': { 'suffix_enc': '', 'get_dim': lambda gmm: gmm.k * (2 * gmm.d + 1), 'sstats_to_features': FVModel.sstats_to_features, 'sstats_to_normalized_features': FVModel.sstats_to_normalized_features, }, 'sfv': { 'suffix_enc': '_sfv', 'get_dim': lambda gmm: gmm.k * (2 * 3 + 1), 'sstats_to_features': SFVModel.spatial_sstats_to_spatial_features, }, } if str(sample) in ('train', 'test'): stats_file = "%s.dat" % sample labels_file = "labels_%s.info" % sample info_file = "info_%s.info" % sample else: stats_tmp = "stats.tmp%s%s/%s" % ( dataset.SUFFIX_STATS, ENC_PARAMS[encoding]['suffix_enc'], sample) stats_file = "%s.dat" % stats_tmp labels_file = "%s.info" % stats_tmp info_file = "%s.info" % stats_tmp stats_path = os.path.join(dataset.SSTATS_DIR, stats_file) labels_path = os.path.join(dataset.SSTATS_DIR, labels_file) info_path = os.path.join(dataset.SSTATS_DIR, info_file) with open(dataset.GMM, 'r') as ff: gmm = yael.gmm_read(ff) K = gmm.k D = ENC_PARAMS[encoding]['get_dim'](gmm) data = np.fromfile(stats_path, dtype=np.float32) data = data.reshape(-1, D) counts = data[:, : K] if analytical_fim: data = ENC_PARAMS[encoding]['sstats_to_normalized_features'](data, gmm) else: data = ENC_PARAMS[encoding]['sstats_to_features'](data, gmm) with open(labels_path, 'r') as ff: labels = cPickle.load(ff) with open(info_path, 'r') as ff: info = cPickle.load(ff) if sqrt_nr_descs: T = info['nr_descs'] T = np.sqrt(T)[:, np.newaxis] else: T = 1. if pi_derivatives or encoding == 'sfv': # For the spatial Fisher vector encoding I drop the `pi_derivatives`, # so I need the full vector, no matter the value of the `pi_derivates` # parameter. idxs = slice(D) else: idxs = slice(K, D) if return_info: return T * data[:, idxs], labels, counts, info else: return T * data[:, idxs], labels, counts
def load_sample_data(dataset, sample, analytical_fim=False, pi_derivatives=False, sqrt_nr_descs=False, return_info=False, encoding='fv'): ENC_PARAMS = { 'fv': { 'suffix_enc': '', 'get_dim': lambda gmm: gmm.k * (2 * gmm.d + 1), 'sstats_to_features': FVModel.sstats_to_features, 'sstats_to_normalized_features': FVModel.sstats_to_normalized_features, }, 'sfv': { 'suffix_enc': '_sfv', 'get_dim': lambda gmm: gmm.k * (2 * 3 + 1), 'sstats_to_features': SFVModel.spatial_sstats_to_spatial_features, }, } if str(sample) in ('train', 'test'): stats_file = "%s.dat" % sample labels_file = "labels_%s.info" % sample info_file = "info_%s.info" % sample else: stats_tmp = "stats.tmp%s%s/%s" % ( dataset.SUFFIX_STATS, ENC_PARAMS[encoding]['suffix_enc'], sample) stats_file = "%s.dat" % stats_tmp labels_file = "%s.info" % stats_tmp info_file = "%s.info" % stats_tmp stats_path = os.path.join(dataset.SSTATS_DIR, stats_file) labels_path = os.path.join(dataset.SSTATS_DIR, labels_file) info_path = os.path.join(dataset.SSTATS_DIR, info_file) with open(dataset.GMM, 'r') as ff: gmm = yael.gmm_read(ff) K = gmm.k D = ENC_PARAMS[encoding]['get_dim'](gmm) data = np.fromfile(stats_path, dtype=np.float32) data = data.reshape(-1, D) counts = data[:, :K] if analytical_fim: data = ENC_PARAMS[encoding]['sstats_to_normalized_features'](data, gmm) else: data = ENC_PARAMS[encoding]['sstats_to_features'](data, gmm) with open(labels_path, 'r') as ff: labels = cPickle.load(ff) with open(info_path, 'r') as ff: info = cPickle.load(ff) if sqrt_nr_descs: T = info['nr_descs'] T = np.sqrt(T)[:, np.newaxis] else: T = 1. if pi_derivatives or encoding == 'sfv': # For the spatial Fisher vector encoding I drop the `pi_derivatives`, # so I need the full vector, no matter the value of the `pi_derivates` # parameter. idxs = slice(D) else: idxs = slice(K, D) if return_info: return T * data[:, idxs], labels, counts, info else: return T * data[:, idxs], labels, counts
def get_per_video_mbh_data_given_list(list_name, **kwargs): """ Loads the Fisher vectors corresponding to the samples found in the list specified by `list_name`. """ K = kwargs.get('K', 256) sstats_path = ('/home/clear/oneata/data/trecvid12/features' '/dense5.track15mbh.small.skip_1/statistics_k_%d' % K) # Default base directories. list_base_path = kwargs.get( 'list_base_path', '/home/lear/douze/src/experiments/trecvid12/data') cache_base_path = kwargs.get( 'cache_base_path', sstats_path) double_norm = kwargs.get('double_norm', False) suffix = '.double_norm' if double_norm else '' # If this file exists load them directly. cache_filename = os.path.join(cache_base_path, 'mbh_' + list_name + suffix + '.raw') if os.path.exists(cache_filename): print "Loading Fisher vectors from MBH descriptors for list %s..." % ( list_name) with open(cache_filename, 'r') as ff: data = np.load(ff) labels = np.load(ff) names = cPickle.load(ff) return data, labels, names D = 64 data, labels, names = [], [], [] sstats_generic_name = os.path.join(sstats_path, 'stats.tmp' + suffix, '%s.dat') list_path = os.path.join(list_base_path, list_name + '.list') gmm_path = os.path.join(sstats_path, 'gmm_%d' % K) gmm = gmm_read(open(gmm_path, 'r')) # Get descriptors for the files in list. print "Creating cache file from list %s..." % list_name for line in open(list_path, 'r'): fields = line.split() sstats_filename = fields[0] video_name = sstats_filename.split('-')[0] sys.stdout.write("%s\t\r" % video_name) try: class_name = fields[1] except IndexError: class_name = 'unknown' try: sstats = np.fromfile(sstats_generic_name % sstats_filename, dtype=np.float32) if double_norm: fv = sstats else: fv = FVModel.sstats_to_features(sstats, gmm) except IOError: print ('Sufficient statistics for video %s are missing;' 'replacing with zeros.') % video_name fv = np.zeros(K + 2 * K * D) data.append(fv) names.append(video_name) labels.append(rev_label_map[class_name]) data = np.vstack(data) labels = np.array(labels) assert data.shape[0] == len(labels), "Data size doesn't match nr of labels" # Cache results to file. with open(cache_filename, 'w') as ff: np.save(ff, data) np.save(ff, labels) cPickle.dump(names, ff) return data, labels, names