def batch_generate_pymeta(data_root_folder, force_to_generate=False): """ This function will convert batches into pymeta style which can be loaded by convnet """ allfolder = iu.getfolderlist(data_root_folder) print "Get %d folders" % len(allfolder) l = [] import sys for fn in allfolder: a = DHMLPE() fp = iu.fullfile(data_root_folder, fn, "matlab_meta.mat") if iu.exists(fp, "file"): save_fp = iu.fullfile(data_root_folder, fn, "batches.meta") print "-----------------------------" print "Processing ", fp if iu.exists(save_fp, "file") and not force_to_generate: print "Ha ha, it exists!" else: meta = a.get_convnet_meta(fp) mio.pickle(save_fp, meta) print "Saved %s" % save_fp else: l = l + [fp] print "=============\n" print "Here is what I cannot find (%d in total)" % len(l) print l
def batch_generate_pymeta(data_root_folder, force_to_generate=False): """ This function will convert batches into pymeta style which can be loaded by convnet """ allfolder = iu.getfolderlist(data_root_folder) print 'Get %d folders' % len(allfolder) l = [] import sys for fn in allfolder: a = DHMLPE() fp = iu.fullfile(data_root_folder, fn, 'matlab_meta.mat') if iu.exists(fp, 'file'): save_fp = iu.fullfile(data_root_folder, fn, 'batches.meta') print '-----------------------------' print 'Processing ', fp if iu.exists(save_fp, 'file') and not force_to_generate: print 'Ha ha, it exists!' else: meta = a.get_convnet_meta(fp) mio.pickle(save_fp, meta) print 'Saved %s' % save_fp else: l = l + [fp] print '=============\n' print 'Here is what I cannot find (%d in total)' % len(l) print l
def merge_batch_data(data_dir_list, save_dir, is_symbolic=True, batch_start_num=1): """ This function will merge all the data_batches in data_dir into one folder and rename them accordining. Of cause, meta data will be updated """ import os import shutil iu.ensure_dir(save_dir) meta = None for ddir in data_dir_list: cur_meta = myio.unpickle(iu.fullfile(ddir, 'batches.meta')) meta = HMLPE.merge_meta(meta, cur_meta) myio.pickle(iu.fullfile(save_dir, 'batches.meta'), meta) cur_id = batch_start_num for ddir in data_dir_list: all_file = iu.getfilelist(ddir, 'data_batch_\d+') print 'I find %d batches in %s' % (len(all_file), ddir) if is_symbolic: for fn in all_file: sn = iu.fullfile(save_dir, 'data_batch_%d' % cur_id) if iu.exists(sn, 'file'): os.remove(sn) os.symlink(iu.fullfile(ddir, fn), sn) cur_id = cur_id + 1 else: for fn in all_file: shutil.copyfile( iu.fullfile(ddir, fn), iu.fullfile(save_dir, 'data_batch_%d' % cur_id)) cur_id = cur_id + 1
def add_part_indicatormap(data_dir, save_dir, mdim, rate, filter_size, stride): """ This function is used for generating part indicator map for old data data_dir is the directory that you put all batch_datayes """ allfile = iu.getfilelist(data_dir, 'data_batch_\d+') meta_path = iu.fullfile(data_dir, 'batches.meta') iu.ensure_dir(save_dir) if iu.exists(meta_path, 'file'): d_meta = myio.unpickle(meta_path) if 'savedata_info' not in d_meta: d_meta['savedata_info'] = dict() d_meta['savedata_info']['indmap_para'] = dict() d_meta['savedata_info']['indmap_para']['filter_size'] = filter_size d_meta['savedata_info']['indmap_para']['stride'] = stride d_meta['savedata_info']['indmap_para']['rate'] = rate myio.pickle(iu.fullfile(save_dir, 'batches.meta'), d_meta) for fn in allfile: print 'Processing %s' % fn d = myio.unpickle(iu.fullfile(data_dir, fn)) ndata = d['data'].shape[-1] nparts = 7 d['indmap'] = np.zeros((nparts, mdim[0], mdim[1], ndata), dtype=np.bool) for i in range(ndata): jts = d['joints8'][..., i] d['indmap'][..., i] = HMLPE.create_part_indicatormap( jts, part_idx, mdim, rate, filter_size, stride) myio.pickle(iu.fullfile(save_dir, fn), d)
def merge_batch_data(data_dir_list, save_dir, is_symbolic = True, batch_start_num = 1): """ This function will merge all the data_batches in data_dir into one folder and rename them accordining. Of cause, meta data will be updated """ import os import shutil iu.ensure_dir(save_dir) meta = None for ddir in data_dir_list: cur_meta = myio.unpickle(iu.fullfile(ddir, 'batches.meta')) meta = HMLPE.merge_meta(meta, cur_meta) myio.pickle(iu.fullfile(save_dir, 'batches.meta'), meta) cur_id = batch_start_num for ddir in data_dir_list: all_file = iu.getfilelist(ddir, 'data_batch_\d+') print 'I find %d batches in %s' % (len(all_file), ddir) if is_symbolic: for fn in all_file: sn = iu.fullfile(save_dir, 'data_batch_%d' % cur_id) if iu.exists(sn, 'file'): os.remove(sn) os.symlink(iu.fullfile(ddir, fn), sn) cur_id = cur_id + 1 else: for fn in all_file: shutil.copyfile(iu.fullfile(ddir, fn), iu.fullfile(save_dir, 'data_batch_%d' % cur_id)) cur_id = cur_id + 1
def add_part_indicatormap(data_dir, save_dir, mdim, rate, filter_size, stride): """ This function is used for generating part indicator map for old data data_dir is the directory that you put all batch_datayes """ allfile = iu.getfilelist(data_dir, 'data_batch_\d+') meta_path = iu.fullfile(data_dir, 'batches.meta') iu.ensure_dir(save_dir) if iu.exists(meta_path, 'file'): d_meta = myio.unpickle(meta_path) if 'savedata_info' not in d_meta: d_meta['savedata_info'] = dict() d_meta['savedata_info']['indmap_para'] = dict() d_meta['savedata_info']['indmap_para']['filter_size'] = filter_size d_meta['savedata_info']['indmap_para']['stride'] = stride d_meta['savedata_info']['indmap_para']['rate'] = rate myio.pickle(iu.fullfile(save_dir, 'batches.meta'), d_meta) for fn in allfile: print 'Processing %s' % fn d = myio.unpickle(iu.fullfile(data_dir, fn)) ndata = d['data'].shape[-1] nparts = 7 d['indmap'] = np.zeros((nparts, mdim[0], mdim[1], ndata), dtype=np.bool) for i in range(ndata): jts = d['joints8'][...,i] d['indmap'][...,i] = HMLPE.create_part_indicatormap(jts, part_idx, mdim, rate, filter_size, stride) myio.pickle(iu.fullfile(save_dir, fn), d)