def test_fname_pairs_log_files(self):
a = fs.gen_paths(self.dir_, is_caffe_info_log)
b = fs.gen_paths(self.dir_tmp, is_caffe_info_log)
pairs = fs.fname_pairs(a, b)
for x, y in pairs:
assert_in(x, a)
assert_in(y, b)
def walk_id_loc_to_loc(dir_src, key_dst):
def runner(fpath):
if filter_is_h5(fpath):
id_loc_to_loc(fpath, key_dst)
return True # otherwise gen_paths won't append to list
flist = gen_paths(dir_src, func_filter=runner)
return flist
def LearningCurveFromPath(p):
if os.path.isfile(p):
return LearningCurve(p)
elif os.path.isdir(p):
log_paths = fs.gen_paths(p, func_filter=is_caffe_info_log)
if len(log_paths) > 0:
return LearningCurve(log_paths[-1])
else:
return None
else:
raise IOError("%s: No such file or directory" % (p, ))
def LearningCurveFromPath(p):
if os.path.isfile(p):
return LearningCurve(p)
elif os.path.isdir(p):
log_paths = fs.gen_paths(p, func_filter=is_caffe_info_log)
if len(log_paths) > 0:
return LearningCurve(log_paths[-1])
else:
return None
else:
raise IOError("%s: No such file or directory" % (p,))
def lower_samplerate_cmds(dir_src, samplerate, dir_dst, fpath_exec_script):
paths = fs.gen_paths(dir_src, is_wav)
commands = []
for fpath in paths:
path_dst = os.path.join(dir_dst, os.path.basename(fpath))
cmd = ['ffmpeg', '-i', fpath, '-ac', '2', '-ar', "%d" % (samplerate,), path_dst]
commands.append(cmd)
commands = [list2cmdline(cmd) for cmd in commands]
with open(fpath_exec_script, 'w') as f:
for cmd in commands:
f.write('%s\n' % (cmd,))
def test_gen_paths_no_imgs_found(self):
flist = fs.gen_paths(self.dir_, fs.filter_is_img)
assert_equal(len(flist), 0)
def test_gen_paths_is_caffe_log(self):
flist = fs.gen_paths(self.dir_, is_caffe_info_log)
assert_is_instance(flist, list)
assert_equal(len(flist), 1)
assert_true('.log.' in flist[0] and '.INFO.' in flist[0])
def test_gen_paths_no_filter(self):
flist = fs.gen_paths(self.dir_)
assert_is_instance(flist, list)
assert_greater(len(flist), 0)
def get_paths_au_labels(self):
p = fs.gen_paths(self.dir_au_labels, AMFED.is_au_label)
self.log_num_paths(p)
return p
def pascal_context_to_lmdb(dir_imgs,
dir_segm_labels,
fpath_labels_list,
fpath_labels_list_subset,
dst_prefix,
dir_dst,
val_list=None):
'''
Fine intersection of filename in both directories and create
one lmdb directory for each
val_list - list of entities to exclude from train (validation subset e.g. ['2008_000002', '2010_000433'])
'''
if dst_prefix is None:
dst_prefix = ''
labels_list = get_labels_list(fpath_labels_list)
labels_59_list = get_labels_list(fpath_labels_list_subset)
# print labels_list
# print labels_59_list
labels_lut = du.get_labels_lut(labels_list, labels_59_list)
def apply_labels_lut(m):
return labels_lut[m]
paths_imgs = fs.gen_paths(dir_imgs, fs.filter_is_img)
paths_segm_labels = fs.gen_paths(dir_segm_labels)
paths_pairs = fs.fname_pairs(paths_imgs, paths_segm_labels)
paths_imgs, paths_segm_labels = map(list, zip(*paths_pairs))
# for a, b in paths_pairs:
# print a,b
if val_list is not None:
# do train/val split
train_idx, val_idx = du.get_train_val_split_from_names(paths_imgs, val_list)
# images
paths_imgs_train = [paths_imgs[i] for i in train_idx]
fpath_lmdb_imgs_train = os.path.join(dir_dst,
'%scontext_imgs_train_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs_train,
fpath_lmdb_imgs_train)
paths_imgs_val = [paths_imgs[i] for i in val_idx]
fpath_lmdb_imgs_val = os.path.join(dir_dst,
'%scontext_imgs_val_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs_val,
fpath_lmdb_imgs_val)
# ground truth
paths_segm_labels_train = [paths_segm_labels[i] for i in train_idx]
fpath_lmdb_segm_labels_train = os.path.join(dir_dst,
'%scontext_labels_train_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels_train,
fpath_lmdb_segm_labels_train,
'LabelMap',
lut=apply_labels_lut)
paths_segm_labels_val = [paths_segm_labels[i] for i in val_idx]
fpath_lmdb_segm_labels_val = os.path.join(dir_dst,
'%scontext_labels_val_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels_val,
fpath_lmdb_segm_labels_val,
'LabelMap',
lut=apply_labels_lut)
return len(paths_imgs_train), len(paths_imgs_val), \
fpath_lmdb_imgs_train, fpath_lmdb_segm_labels_train, fpath_lmdb_imgs_val, fpath_lmdb_segm_labels_val
else:
fpath_lmdb_imgs = os.path.join(dir_dst,
'%scontext_imgs_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs,
fpath_lmdb_imgs)
fpath_lmdb_segm_labels = os.path.join(dir_dst,
'%scontext_labels_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels,
fpath_lmdb_segm_labels,
'LabelMap',
lut=apply_labels_lut)
return len(paths_imgs), fpath_lmdb_imgs, fpath_lmdb_segm_labels
def get_paths_landmarks(self):
p = fs.gen_paths(self.dir_landmarks, AMFED.is_landmarks)
self.log_num_paths(p)
return p
def get_paths_au_labels(self):
p = fs.gen_paths(self.dir_au_labels, AMFED.is_au_label)
self.log_num_paths(p)
return p
def get_paths_videos(self):
p = fs.gen_paths(self.dir_videos, AMFED.is_video)
self.log_num_paths(p)
return p
def get_paths_landmarks(self):
p = fs.gen_paths(self.dir_landmarks, AMFED.is_landmarks)
self.log_num_paths(p)
return p
def test_hash_file(self):
p = fs.gen_paths(self.dir_, is_caffe_info_log)[0]
h = fs.hashfile(p)
assert_is_instance(h, str)
assert_greater(len(h), 0)
def pascal_context_to_lmdb(dir_imgs,
dir_segm_labels,
fpath_labels_list,
fpath_labels_list_subset,
dst_prefix,
dir_dst,
val_list=None):
'''
Fine intersection of filename in both directories and create
one lmdb directory for each
val_list - list of entities to exclude from train (validation subset e.g. ['2008_000002', '2010_000433'])
'''
if dst_prefix is None:
dst_prefix = ''
labels_list = get_labels_list(fpath_labels_list)
labels_59_list = get_labels_list(fpath_labels_list_subset)
# print labels_list
# print labels_59_list
labels_lut = du.get_labels_lut(labels_list, labels_59_list)
def apply_labels_lut(m):
return labels_lut[m]
paths_imgs = fs.gen_paths(dir_imgs, fs.filter_is_img)
paths_segm_labels = fs.gen_paths(dir_segm_labels)
paths_pairs = fs.fname_pairs(paths_imgs, paths_segm_labels)
paths_imgs, paths_segm_labels = map(list, zip(*paths_pairs))
# for a, b in paths_pairs:
# print a,b
if val_list is not None:
# do train/val split
train_idx, val_idx = du.get_train_val_split_from_names(
paths_imgs, val_list)
# images
paths_imgs_train = [paths_imgs[i] for i in train_idx]
fpath_lmdb_imgs_train = os.path.join(
dir_dst, '%scontext_imgs_train_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs_train, fpath_lmdb_imgs_train)
paths_imgs_val = [paths_imgs[i] for i in val_idx]
fpath_lmdb_imgs_val = os.path.join(
dir_dst, '%scontext_imgs_val_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs_val, fpath_lmdb_imgs_val)
# ground truth
paths_segm_labels_train = [paths_segm_labels[i] for i in train_idx]
fpath_lmdb_segm_labels_train = os.path.join(
dir_dst, '%scontext_labels_train_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels_train,
fpath_lmdb_segm_labels_train,
'LabelMap',
lut=apply_labels_lut)
paths_segm_labels_val = [paths_segm_labels[i] for i in val_idx]
fpath_lmdb_segm_labels_val = os.path.join(
dir_dst, '%scontext_labels_val_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels_val,
fpath_lmdb_segm_labels_val,
'LabelMap',
lut=apply_labels_lut)
return len(paths_imgs_train), len(paths_imgs_val), \
fpath_lmdb_imgs_train, fpath_lmdb_segm_labels_train, fpath_lmdb_imgs_val, fpath_lmdb_segm_labels_val
else:
fpath_lmdb_imgs = os.path.join(dir_dst,
'%scontext_imgs_lmdb' % dst_prefix)
to_lmdb.imgs_to_lmdb(paths_imgs, fpath_lmdb_imgs)
fpath_lmdb_segm_labels = os.path.join(
dir_dst, '%scontext_labels_lmdb' % dst_prefix)
to_lmdb.matfiles_to_lmdb(paths_segm_labels,
fpath_lmdb_segm_labels,
'LabelMap',
lut=apply_labels_lut)
return len(paths_imgs), fpath_lmdb_imgs, fpath_lmdb_segm_labels
def get_paths_videos(self):
p = fs.gen_paths(self.dir_videos, AMFED.is_video)
self.log_num_paths(p)
return p
