def main():
parser = argparse.ArgumentParser(
description="Extracts a subset of low-level descriptors for "
"PCA and GMM learning.")
# Arguments for the `config` command.
parser.add_argument(
'-t', '--task', choices=['compute', 'merge'],
help="what to do: extract low-level descriptors `compute` or merge "
"existing low-level descriptors `merge`.")
parser.add_argument(
'-d', '--dataset', required=True,
help="name of the dataset configuration file; it should have the "
"format <file>.<class_instance> (e.g., 'hollywood2.small'.")
parser.add_argument(
'-b', '--begin_idx', type=int, default=None,
help="index of the fisrt sample to process.")
parser.add_argument(
'-e', '--end_idx', type=int, default=None,
help="index of the last sample to process.")
parser.add_argument(
'-p', '--proportion_of_descriptors', type=float, default=0.001,
help="proportion of kept descriptors.")
parser.add_argument(
'-ss', '--subsample', type=int, default=10,
help="selects one out of `ss` videos.")
parser.add_argument(
'-np', '--nr_processes', type=int, default=yael.count_cpu(),
help="number of processes to launch.")
parser.add_argument(
'--redo', default=False, action='store_true',
help="recompute Fisher vectors even if they already exist.")
parser.add_argument(
'-v', '--verbose', action='count', help="verbosity level.")
args = parser.parse_args()
# Dynamically import the configuration instance.
dataset = import_dataset(*args.dataset.split('.'))
if not 0 < args.proportion_of_descriptors < 1:
print "The proportion of kept descriptors should be between 0 and 1."
sys.exit(1)
if args.task == 'compute':
samples = dataset.get_samples(['train'])
bb, ee = correct_idxs(args.begin_idx, args.end_idx, 0, len(samples))
def worker(sample):
video_path = dataset.get_video_path(sample)
out_path = dataset.get_descriptor_path(sample)
return extract_subset_of_descriptors(
video_path, out_path, dataset.descriptor_type,
args.proportion_of_descriptors, redo=args.redo,
verbose=args.verbose)
RunOnSet(args.nr_processes, samples[bb: ee: args.subsample], worker)
elif args.task == 'merge':
merge(
dataset.subset_path, dataset.descriptor_base_path,
dataset.subset_size, dataset.descriptor_type, verbose=args.verbose)
def main():
VALID_DESCRIPTORS = ['HOG', 'HOF', 'MBH']
VALID_MODELS = ['FV', 'SFV']
parser = argparse.ArgumentParser(
description="Extract Fisher vector on dense trajectory descriptors.")
# Arguments for the `config` command.
parser.add_argument(
'-d', '--dataset', required=True,
help="name of the dataset configuration file; it should have the "
"format <file>.<class_instance> (e.g., 'hollywood2.small'.")
parser.add_argument(
'-b', '--begin_idx', type=int, default=None,
help="index of the fisrt sample to process.")
parser.add_argument(
'-e', '--end_idx', type=int, default=None,
help="index of the last sample to process.")
parser.add_argument(
'-np', '--nr_processes', type=int, default=yael.count_cpu(),
help="number of processes to launch.")
parser.add_argument(
'--redo', default=False, action='store_true',
help="recompute Fisher vectors even if they already exist.")
parser.add_argument(
'-v', '--verbose', action='count', help="verbosity level.")
args = parser.parse_args()
# Dynamically import the configuration instance.
dataset = import_dataset(*args.dataset.split('.'))
# Check dataset parameters.
if dataset.descriptor_type not in VALID_DESCRIPTORS:
print "Descriptor type %s is unknown." % dataset.descriptor_type
sys.exit(1)
if any([model not in VALID_MODELS for model in dataset.models]):
print "The models are unknown."
sys.exit(1)
samples = dataset.get_samples()
bb, ee = correct_idxs(args.begin_idx, args.end_idx, 0, len(samples))
pca = load_pca(dataset.pca_path)
gmm = load_gmm(dataset.gmm_path)
def worker(sample):
video_path = dataset.get_video_path(sample)
def get_feature_path(**kwargs):
return dataset.get_feature_path(sample, **kwargs)
return compute_features(
video_path, get_feature_path, dataset.descriptor_type, pca,
gmm, redo=args.redo, models=dataset.models, spms=dataset.spms,
verbose=args.verbose)
RunOnSet(args.nr_processes, samples[bb: ee], worker)
def main():
parser = argparse.ArgumentParser(
description="Computes GMM on a subset of descriptors.")
parser.add_argument(
'-d', '--dataset', required=True,
help="name of the dataset configuration file.")
parser.add_argument(
'-np', '--nr_processes', type=int, default=yael.count_cpu(),
help="number of GMM clusters.")
args = parser.parse_args()
compute_gmm_given_dataset(args.dataset, args.nr_processes)
def get_per_video_densetracks(split, K=256, descriptor='hog', pyramid=True, ssl = None):
"""Loads Heng's features."""
DESC_DIM = {'mbhx': 48, 'mbhy': 48, 'hog': 48, 'hof': 54}
FV_DIM = 2 * DESC_DIM[descriptor] * K
if pyramid == True:
dims = slice(FV_DIM * 4)
else:
dims = slice(FV_DIM)
outfile = os.path.join("data", "per_video_cache", "densetracks_%s_%s_k%d.raw" % (descriptor, split, K))
if os.path.exists(outfile):
video_data, video_labels, video_names = load_cache_file(outfile, ssl = ssl)
return video_data[:, dims], video_labels, video_names
import gzip
from itertools import izip
if "trecvid13" in split or split.startswith("trecvid12_test"):
infile = "/scratch/clear/hewang/trecvid2013/trecvid13/result_DenseTrack_MED_stab_480/fisher/%s_%s_k%d.gz"
else:
infile = "/scratch/clear/hewang/trecvid2013/trecvid11_full/result_DenseTrack_MED_stab_480/fisher/%s_%s_k%d.gz"
video_names, video_labels = zip(*parse_listfile(split))
N = len(video_names)
missing = [1] * N
video_data = np.zeros((N, FV_DIM * 4), dtype=np.float32)
nthread = yael.count_cpu()
# nthread = 1
def get_video((ii, (video_name, video_label))):
try:
with gzip.open(infile % (video_name, descriptor, K) ,"r") as ff:
video_name_, fisher_vector, fisher_vector_h3, _ = ff.read().split('\n')
assert video_name_.split()[1] == video_name
# fisher_vector_1 = map(np.float32, fisher_vector.split())
fisher_vector = np.fromstring(fisher_vector, dtype = np.float32, sep = ' ')
# assert np.all(fisher_vector_1 == fisher_vector_2)
# fisher_vector_h3 = map(np.float32, fisher_vector_h3.split())
fisher_vector_h3 = np.fromstring(fisher_vector_h3, dtype = np.float32, sep = ' ')
result = "%d" % len(fisher_vector)
missing[ii] = 0
except IOError:
result = "missing"
print "\t#%6d/%6d. %s: %s" % (ii, N, video_name, result)
video_data[ii] = np.hstack((fisher_vector, fisher_vector_h3))
def __init__(self, dataset, model, **kwargs):
self.dataset = dataset
self.model = model
self.K = self.model.K
self.grid = self.model.grids[0]
self.NR_CPUS = count_cpu()
self.sw_profile = kwargs.get('sw_profile', False)
# The length of a descriptor.
desc_type = dataset.FTYPE.split('.')[-1]
if 'mbh' in desc_type:
self.LEN_DESC = 192
elif 'hoghof' in desc_type:
self.LEN_DESC = 96 + 108
elif 'all' in desc_type:
self.LEN_DESC = 96 + 108 + 192
else:
raise Exception('Unknown descriptor type.')
# Set filenames that will be used later on.
self.nr_pca_comps = 64
self.nr_samples = 2e5
self.fn_subsample = os.path.join(self.dataset.FEAT_DIR,
'subset.siftgeo')
self.fn_subsample_dat = os.path.join(self.dataset.FEAT_DIR,
'subset.dat')
self.fn_pca = os.path.join(self.dataset.FEAT_DIR, 'pca',
'pca_%d.pkl' % self.nr_pca_comps)
self.fn_gmm = os.path.join(self.dataset.FEAT_DIR, 'gmm',
'gmm_%d' % self.K)
self.bare_fn = '%s_%d_%d_%d_%d.dat'
self.spatial_bare_fn = 'spatial_%s_%d_%d_%d_%d.dat'
self.root_path = os.path.join(self.dataset.FEAT_DIR,
'statistics_k_%d' % self.K)
# Create path for temporary files.
self.temp_path = os.path.join(self.root_path, 'stats.tmp')
self.descriptor_filename = os.path.join(
self.dataset.FEAT_DIR, 'features', '%s.siftgeo')
def __init__(self, dataset, model, **kwargs):
self.dataset = dataset
self.model = model
self.K = self.model.K
self.grid = self.model.grids[0]
self.NR_CPUS = count_cpu()
self.sw_profile = kwargs.get('sw_profile', False)
# The length of a descriptor.
desc_type = dataset.FTYPE.split('.')[-1]
if 'mbh' in desc_type:
self.LEN_DESC = 192
elif 'hoghof' in desc_type:
self.LEN_DESC = 96 + 108
elif 'all' in desc_type:
self.LEN_DESC = 96 + 108 + 192
else:
raise Exception('Unknown descriptor type.')
# Set filenames that will be used later on.
self.nr_pca_comps = 64
self.nr_samples = 2e5
self.fn_subsample = os.path.join(self.dataset.FEAT_DIR,
'subset.siftgeo')
self.fn_subsample_dat = os.path.join(self.dataset.FEAT_DIR,
'subset.dat')
self.fn_pca = os.path.join(self.dataset.FEAT_DIR, 'pca',
'pca_%d.pkl' % self.nr_pca_comps)
self.fn_gmm = os.path.join(self.dataset.FEAT_DIR, 'gmm',
'gmm_%d' % self.K)
self.bare_fn = '%s_%d_%d_%d_%d.dat'
self.spatial_bare_fn = 'spatial_%s_%d_%d_%d_%d.dat'
self.root_path = os.path.join(self.dataset.FEAT_DIR,
'statistics_k_%d' % self.K)
# Create path for temporary files.
self.temp_path = os.path.join(self.root_path, 'stats.tmp')
self.descriptor_filename = os.path.join(self.dataset.FEAT_DIR,
'features', '%s.siftgeo')
event_class = '',
selection = [],
versions = [],
exconfig = "",
nearmiss_status = 'allpos',
needs_sorting = True)
settings = Struct(
crossval_mixweights = True,
criterion = 'ap',
print_format = 'stdout',
power = 0.5,
fixedparams = {},
fold_test_fraction = 0.25,
ssl = 0,
n_thread = yael.count_cpu())
args = sys.argv[1:]
while args:
a = args.pop(0)
if a in ('-h', '--help'): usage()
elif a == '--key': key = args.pop(0)
elif a == '--train': todo = 'train'
elif a == '--test': todo = 'test'
elif a == '--event-class':
event_class = args.pop(0)
indata.event_class = event_class
def get_per_video_color_data(split, sfv=False, K=256, D=64):
"""
The descriptor key is a directory name. The dimensions can be used to select mu/w/sigma
"""
fisher_dim = K - 1 + 2 * D * K
spatial_fisher_dim = 2 * 3 * K
if sfv:
dim = fisher_dim + spatial_fisher_dim
else:
dim = fisher_dim
# Cache file.
sfv_suffix = '' if sfv else '_nosfv'
dim_suffix = ('_d%d' % D) if D != 64 else ''
filename = os.path.join(
"data", "per_video_cache/%s_color_k%d%s%s.raw" % (
split, K, dim_suffix, sfv_suffix))
if os.path.exists(filename):
# Load data from cache file.
print " load per video data", filename
with open(filename, "r") as ff:
video_data = np.load(ff)
video_labels = np.load(ff)
video_names = cPickle.load(ff)
return video_data[:, :dim], video_labels, video_names
video_names = []
video_labels = []
video_data = []
sstats_path = os.path.expanduser(
"~oneata/data/trecvid13/features/dense.color/"
"statistics_k_%d/stats.tmp%s_sfv" % (K, dim_suffix))
def load_color_desc((vidname, classno)):
fv = load_bigimbaz_blocks("%s/%s_%%d.fvecs" % (sstats_path, vidname), fisher_dim + spatial_fisher_dim)
print vidname, "%d * %d" % fv.shape
fv = normalize_fisher(fv)
# Average of descriptors.
desc = fv.sum(axis=0) / fv.shape[0]
return vidname, classno, desc
nthread = yael.count_cpu()
for vidname, classno, desc in threads.ParallelIter(nthread, parse_listfile(split), load_color_desc):
video_labels.append(classno)
video_names.append(vidname)
video_data.append(desc)
video_data = np.vstack(video_data)
video_labels = np.array(video_labels)
with open(filename, 'w') as ff:
np.save(ff, video_data)
np.save(ff, video_labels)
cPickle.dump(video_names, ff)
return video_data[:, :dim], video_labels, video_names
def get_per_video_sift_data(split, desc_key = 'framestep60_dense_siftnonorm_lpca32_gmm256_w_mu_sigma',
dimensions = (255, 255 + 32 * 256),
ssl = None, nchunks = 1, v14 = False):
"""
The descriptor key is a directory name. The dimensions can be used to select mu/w/sigma
"""
# cache
# filename = dan_data + "per_video_cache/SIFT_%s_%s.raw" % (desc_key, split)
if nchunks > 1:
suffix = "_nchunks%d" % nchunks
else:
suffix = ""
if v14:
if split in ('train_balanced', 'test_balanced', 'trecvid11_more_null'):
outdir = "/home/clear/dataset/trecvid14/tv11/PS-Training/SIFT"
elif split.startswith("trecvid11_test_"):
outdir = "/home/clear/dataset/trecvid14/tv11/Test/SIFT"
elif split == "tv14test_positives":
outdir = "/home/clear/dataset/trecvid14/MED14-Test/PS-Training/SIFT"
elif split == "tv14test_eventbg":
outdir = "/home/clear/dataset/trecvid14/MED14-Test/Event-BG/SIFT"
else:
assert False, "split " + split + " unknown"
filename = "%s/%s%s_%s.raw" % (outdir, desc_key, suffix, split)
else:
filename = "data/per_video_cache/SIFT_%s%s_%s.raw" % (desc_key, suffix, split)
if os.path.exists(filename):
# Load data from cache file.
if ssl == None:
print " load per video data", filename
with open(filename, "r") as ff:
video_data = np.load(ff)
video_labels = np.load(ff)
video_names = cPickle.load(ff)
else:
ff = open(filename, 'r')
skip_npy(ff)
labels = np.load(ff)
names = cPickle.load(ff)
data = np.load(filename, mmap_mode = 'r')
assert data.shape[0] == labels.size == len(names)
n = data.shape[0]
slno, slb = ssl # read fraction i out of slb
i0, i1 = slno * n / slb, (slno + 1) * n / slb
# select the fraction
video_data = data[i0:i1, :].copy()
video_names = names[i0:i1]
video_labels = labels[i0:i1]
d0, d1 = dimensions
return video_data[:, d0:d1], video_labels, video_names
# to build the index
# k=framestep60_dense_siftnonorm_lpca32_gmm256_w_mu_sigma;
# k=framestep60_dense_siftnonorm_lpca32_gmm1024_w_mu_sigma;
# (find /scratch2/clear/dataset/trecvid11/med//desc/frames/$k/* ; find /scratch/clear/dataset/trecvid12/desc/events/$k/E* /scratch/clear/dataset/trecvid12/desc/PROGTEST/$k/ ) | grep _0.fvecs > $k.index
assert ssl == None
if split.startswith("tv14test"):
single_location = outdir + "/" + desc_key
elif "sfv" in desc_key:
# slightly different subsets used for storing SIFT
subsets = ['trecvid13_train', 'trecvid12_test', 'trecvid11_balanced', 'trecvid13_adhoc', 'trecvid11_more_null', 'trecvid11_test']
if split == 'train_balanced' or split == 'test_balanced':
subset_name = 'trecvid11_balanced'
else:
subset_name = None
for s in subsets:
if split.startswith(s):
subset_name = s
break
basedir = "/home/lear/potapov/data/med11/static/desc/frames"
final_dir= "framestep60_dense_siftnonorm_lpca64_gmm256_sfv2"
if subset_name:
single_location = "%s/%s/%s" % (basedir, subset_name, final_dir)
else:
single_location = None
else:
single_location = None
location = {}
for l in open(matthijs_data + "per_video_cache/" + desc_key + ".index", "r"):
i = l.rfind('/')
d, f = l[:i], l[i+1:-9]
location[f] = d
print "Caching video descriptors, will be put in", filename
video_names = []
video_labels = []
video_data = []
d = -1 # unknown... Let's hope the first video does not have 0 frames
def load_sift_desc((vidname, classno)):
# Fisher descriptors of frames in a video may be spread over several blocks
dirname = single_location or location[vidname]
fv = load_bigimbaz_blocks("%s/%s_%%d.fvecs" % (dirname, vidname), d)
print vidname, "%d * %d" % fv.shape
# normalize per frame
fv = normalize_fisher(fv)
if nchunks > 1:
# average descriptors inside chunks
fv = average_inside_chunks(fv, nchunks)
# normalize "per chunk"
fv = power_normalize(fv, 0.5)
fv = L2_normalize(fv)
# average of frame (or chunk) descriptors
desc = fv.sum(axis = 0) / fv.shape[0]
return vidname, classno, desc
# Identify the dimension
parsed_listfile = parse_listfile(split)
vidname, classno, desc = load_sift_desc(parsed_listfile[0])
d = desc.shape[0] # desc is a 1-dimensional vector
nthread = yael.count_cpu()
for vidname, classno, desc in threads.ParallelIter(nthread, parsed_listfile, load_sift_desc):
video_labels.append(classno)
video_names.append(vidname)
video_data.append(desc)
video_data = np.vstack(video_data)
video_labels = np.array(video_labels)
with open(filename, "w") as ff:
np.save(ff, video_data)
np.save(ff, video_labels)
cPickle.dump(video_names, ff)
d0, d1 = dimensions
return video_data[:, d0:d1], video_labels, video_names
# File format
fmt_b = 'fvecs'
fmt_q = 'fvecs'
fmt_nn = 'ivecs'
fmt_dis = 'fvecs'
fb_name = None # database filename
fq_name = None # query filename
fnn_name = "nn.out"
# output filename for indexes
fdis_name = "dis.out"
# output filenames for distances
# Main processing - parameters
ma = 1 # multiple assignment
nt = yael.count_cpu() # automatic detection of number of cores
# Parse argument (and override default)
args = sys.argv[1:]
while args:
a = args.pop(0)
if a == '-b':
fb_name = args.pop(0)
elif a == '-brawf':
fb_name = args.pop(0)
fmt_b = 'rawf'
elif a == '-q':
fq_name = args.pop(0)
elif a == '-qrawf':
fq_name = args.pop(0)
fmt_q = 'rawf'
for fold in range(nfold)]
if it == 0:
# add the baseline, which has not been evaluated so far
todo = [(None, 0, fold) for fold in range(nfold)] + todo
# filter out configurations that have been visited already
todo = [(pname, pm, fold) for (pname, pm, fold) in todo
if (params_to_tuple(self.params_step(pname, pm)),
0) not in self.cache]
# use multithreading if training itself is not threaded
if self.constant_parameters['n_thread']:
n_thread = 1
else:
n_thread = yael.count_cpu()
# perform all experiments
src = threads.ParallelIter(n_thread, todo, self.do_exp)
while True:
# pull a series of nfold results
try:
allstats = [next(src) for j in range(nfold)]
except StopIteration:
break
(pname, pm, zero), stats = allstats[0]
assert zero == 0
params_i = self.params_step(pname, pm)
# File format
fmt_b = 'fvecs'
fmt_q = 'fvecs'
fmt_nn = 'ivecs'
fmt_dis = 'fvecs'
fb_name = None # database filename
fq_name = None # query filename
fnn_name = "nn.out"; # output filename for indexes
fdis_name = "dis.out"; # output filenames for distances
# Main processing - parameters
ma = 1 # multiple assignment
nt = yael.count_cpu() # automatic detection of number of cores
# Parse argument (and override default)
args = sys.argv[1:]
while args:
a = args.pop (0)
if a == '-b':
fb_name = args.pop(0)
elif a == '-brawf':
fb_name = args.pop(0)
fmt_b = 'rawf'
elif a == '-q':
fq_name = args.pop(0)
elif a == '-qrawf':
fq_name = args.pop(0)
fmt_q = 'rawf'
