def batch_generate_pymeta(data_root_folder, force_to_generate=False):
"""
This function will convert batches into pymeta style which can be loaded by convnet
"""
allfolder = iu.getfolderlist(data_root_folder)
print "Get %d folders" % len(allfolder)
l = []
import sys
for fn in allfolder:
a = DHMLPE()
fp = iu.fullfile(data_root_folder, fn, "matlab_meta.mat")
if iu.exists(fp, "file"):
save_fp = iu.fullfile(data_root_folder, fn, "batches.meta")
print "-----------------------------"
print "Processing ", fp
if iu.exists(save_fp, "file") and not force_to_generate:
print "Ha ha, it exists!"
else:
meta = a.get_convnet_meta(fp)
mio.pickle(save_fp, meta)
print "Saved %s" % save_fp
else:
l = l + [fp]
print "=============\n"
print "Here is what I cannot find (%d in total)" % len(l)
print l
def batch_generate_pymeta(data_root_folder, force_to_generate=False):
"""
This function will convert batches into pymeta style which can be loaded by convnet
"""
allfolder = iu.getfolderlist(data_root_folder)
print 'Get %d folders' % len(allfolder)
l = []
import sys
for fn in allfolder:
a = DHMLPE()
fp = iu.fullfile(data_root_folder, fn, 'matlab_meta.mat')
if iu.exists(fp, 'file'):
save_fp = iu.fullfile(data_root_folder, fn, 'batches.meta')
print '-----------------------------'
print 'Processing ', fp
if iu.exists(save_fp, 'file') and not force_to_generate:
print 'Ha ha, it exists!'
else:
meta = a.get_convnet_meta(fp)
mio.pickle(save_fp, meta)
print 'Saved %s' % save_fp
else:
l = l + [fp]
print '=============\n'
print 'Here is what I cannot find (%d in total)' % len(l)
print l
def merge_batch_data(data_dir_list,
save_dir,
is_symbolic=True,
batch_start_num=1):
"""
This function will merge all the data_batches in data_dir into one folder
and rename them accordining.
Of cause, meta data will be updated
"""
import os
import shutil
iu.ensure_dir(save_dir)
meta = None
for ddir in data_dir_list:
cur_meta = myio.unpickle(iu.fullfile(ddir, 'batches.meta'))
meta = HMLPE.merge_meta(meta, cur_meta)
myio.pickle(iu.fullfile(save_dir, 'batches.meta'), meta)
cur_id = batch_start_num
for ddir in data_dir_list:
all_file = iu.getfilelist(ddir, 'data_batch_\d+')
print 'I find %d batches in %s' % (len(all_file), ddir)
if is_symbolic:
for fn in all_file:
sn = iu.fullfile(save_dir, 'data_batch_%d' % cur_id)
if iu.exists(sn, 'file'):
os.remove(sn)
os.symlink(iu.fullfile(ddir, fn), sn)
cur_id = cur_id + 1
else:
for fn in all_file:
shutil.copyfile(
iu.fullfile(ddir, fn),
iu.fullfile(save_dir, 'data_batch_%d' % cur_id))
cur_id = cur_id + 1
def add_part_indicatormap(data_dir, save_dir, mdim, rate, filter_size, stride):
"""
This function is used for generating part indicator map for old data
data_dir is the directory that you put all batch_datayes
"""
allfile = iu.getfilelist(data_dir, 'data_batch_\d+')
meta_path = iu.fullfile(data_dir, 'batches.meta')
iu.ensure_dir(save_dir)
if iu.exists(meta_path, 'file'):
d_meta = myio.unpickle(meta_path)
if 'savedata_info' not in d_meta:
d_meta['savedata_info'] = dict()
d_meta['savedata_info']['indmap_para'] = dict()
d_meta['savedata_info']['indmap_para']['filter_size'] = filter_size
d_meta['savedata_info']['indmap_para']['stride'] = stride
d_meta['savedata_info']['indmap_para']['rate'] = rate
myio.pickle(iu.fullfile(save_dir, 'batches.meta'), d_meta)
for fn in allfile:
print 'Processing %s' % fn
d = myio.unpickle(iu.fullfile(data_dir, fn))
ndata = d['data'].shape[-1]
nparts = 7
d['indmap'] = np.zeros((nparts, mdim[0], mdim[1], ndata),
dtype=np.bool)
for i in range(ndata):
jts = d['joints8'][..., i]
d['indmap'][..., i] = HMLPE.create_part_indicatormap(
jts, part_idx, mdim, rate, filter_size, stride)
myio.pickle(iu.fullfile(save_dir, fn), d)
def merge_batch_data(data_dir_list, save_dir, is_symbolic = True, batch_start_num = 1):
"""
This function will merge all the data_batches in data_dir into one folder
and rename them accordining.
Of cause, meta data will be updated
"""
import os
import shutil
iu.ensure_dir(save_dir)
meta = None
for ddir in data_dir_list:
cur_meta = myio.unpickle(iu.fullfile(ddir, 'batches.meta'))
meta = HMLPE.merge_meta(meta, cur_meta)
myio.pickle(iu.fullfile(save_dir, 'batches.meta'), meta)
cur_id = batch_start_num
for ddir in data_dir_list:
all_file = iu.getfilelist(ddir, 'data_batch_\d+')
print 'I find %d batches in %s' % (len(all_file), ddir)
if is_symbolic:
for fn in all_file:
sn = iu.fullfile(save_dir, 'data_batch_%d' % cur_id)
if iu.exists(sn, 'file'):
os.remove(sn)
os.symlink(iu.fullfile(ddir, fn), sn)
cur_id = cur_id + 1
else:
for fn in all_file:
shutil.copyfile(iu.fullfile(ddir, fn), iu.fullfile(save_dir, 'data_batch_%d' % cur_id))
cur_id = cur_id + 1
def add_part_indicatormap(data_dir, save_dir, mdim, rate, filter_size, stride):
"""
This function is used for generating part indicator map for old data
data_dir is the directory that you put all batch_datayes
"""
allfile = iu.getfilelist(data_dir, 'data_batch_\d+')
meta_path = iu.fullfile(data_dir, 'batches.meta')
iu.ensure_dir(save_dir)
if iu.exists(meta_path, 'file'):
d_meta = myio.unpickle(meta_path)
if 'savedata_info' not in d_meta:
d_meta['savedata_info'] = dict()
d_meta['savedata_info']['indmap_para'] = dict()
d_meta['savedata_info']['indmap_para']['filter_size'] = filter_size
d_meta['savedata_info']['indmap_para']['stride'] = stride
d_meta['savedata_info']['indmap_para']['rate'] = rate
myio.pickle(iu.fullfile(save_dir, 'batches.meta'), d_meta)
for fn in allfile:
print 'Processing %s' % fn
d = myio.unpickle(iu.fullfile(data_dir, fn))
ndata = d['data'].shape[-1]
nparts = 7
d['indmap'] = np.zeros((nparts, mdim[0], mdim[1], ndata), dtype=np.bool)
for i in range(ndata):
jts = d['joints8'][...,i]
d['indmap'][...,i] = HMLPE.create_part_indicatormap(jts, part_idx, mdim, rate, filter_size, stride)
myio.pickle(iu.fullfile(save_dir, fn), d)
