import apt_expts import os import ast import apt_expts import os import pickle os.environ['CUDA_VISIBLE_DEVICES'] = '' gt_lbl = None lbl_file = '/groups/branson/bransonlab/apt/experiments/data/roian_apt_dlstripped.lbl' op_af_graph = '\(0,1\),\(0,2\),\(0,3\),\(1,2\),\(1,3\),\(2,3\)' lbl = h5py.File(lbl_file, 'r') proj_name = apt.read_string(lbl['projname']) nviews = int(apt.read_entry(lbl['cfg']['NumViews'])) lbl.close() cache_dir = '/nrs/branson/mayank/apt_cache' all_models = ['openpose'] gpu_model = 'GeForceRTX2080Ti' sdir = '/groups/branson/home/kabram/bransonlab/APT/deepnet/singularity_stuff' n_splits = 3 common_conf = {} common_conf['rrange'] = 10 common_conf['trange'] = 5 common_conf['mdn_use_unet_loss'] = True common_conf['dl_steps'] = 100000 common_conf['decay_steps'] = 20000 common_conf['save_step'] = 5000
def compute_peformance(args): H = h5py.File(args.lbl_file, 'r') nviews = int(apt.read_entry(H['cfg']['NumViews'])) dir_name = args.whose if len(args.nets) == 0: all_nets = methods else: all_nets = args.nets all_preds = {} for view in range(nviews): db_file = os.path.join(out_dir, args.name, args.gt_name) + '_view{}.tfrecords'.format(view) conf = apt.create_conf(args.lbl_file, view, name='a', net_type=all_nets[0], cache_dir=os.path.join(out_dir, args.name, dir_name)) conf.labelfile = args.gt_lbl if not (os.path.exists(db_file) and args.skip_gt_db): print('Creating GT DB file {}'.format(db_file)) apt.create_tfrecord(conf, split=False, on_gt=True, db_files=(db_file, )) for curm in all_nets: all_preds[curm] = [] for view in range(nviews): cur_out = [] db_file = os.path.join( out_dir, args.name, args.gt_name) + '_view{}.tfrecords'.format(view) if args.split_type is None: cachedir = os.path.join(out_dir, args.name, dir_name, '{}_view_{}'.format(curm, view), 'full') conf = apt.create_conf(args.lbl_file, view, name='a', net_type=curm, cache_dir=cachedir) model_files, ts = get_model_files(conf, cachedir, curm) for mndx, m in enumerate(model_files): out_file = m + '_' + args.gt_name load = False if curm == 'unet' or curm == 'deeplabcut': mm = m + '.index' else: mm = m if os.path.exists(out_file + '.mat') and os.path.getmtime( out_file + '.mat') > os.path.getmtime(mm): load = True if load: H = sio.loadmat(out_file) pred = H['pred_locs'] - 1 label = H['labeled_locs'] - 1 gt_list = H['list'] - 1 else: # pred, label, gt_list = apt.classify_gt_data(conf, curm, out_file, m) tf_iterator = multiResData.tf_reader( conf, db_file, False) tf_iterator.batch_size = 1 read_fn = tf_iterator.next pred_fn, close_fn, _ = apt.get_pred_fn(curm, conf, m) pred, label, gt_list = apt.classify_db( conf, read_fn, pred_fn, tf_iterator.N) close_fn() mat_pred_locs = pred + 1 mat_labeled_locs = np.array(label) + 1 mat_list = gt_list sio.savemat( out_file, { 'pred_locs': mat_pred_locs, 'labeled_locs': mat_labeled_locs, 'list': mat_list }) cur_out.append( [pred, label, gt_list, m, out_file, view, 0, ts[mndx]]) else: for cur_split in range(nsplits): cachedir = os.path.join(out_dir, args.name, '{}_view_{}'.format(curm, view), 'cv_{}'.format(cur_split)) conf = apt.create_conf(args.lbl_file, view, name='a', net_type=curm, cache_dir=cachedir) model_files, ts = get_model_files(conf, cachedir, curm) db_file = os.path.join(cachedir, 'val_TF.tfrecords') for mndx, m in enumerate(model_files): out_file = m + '.gt_data' load = False if curm == 'unet' or curm == 'deeplabcut': mm = m + '.index' else: mm = m if os.path.exists( out_file + '.mat') and os.path.getmtime( out_file + '.mat') > os.path.getmtime(mm): load = True if load: H = sio.loadmat(out_file) pred = H['pred_locs'] - 1 label = H['labeled_locs'] - 1 gt_list = H['list'] - 1 else: tf_iterator = multiResData.tf_reader( conf, db_file, False) tf_iterator.batch_size = 1 read_fn = tf_iterator.next pred_fn, close_fn, _ = apt.get_pred_fn( curm, conf, m) pred, label, gt_list = apt.classify_db( conf, read_fn, pred_fn, tf_iterator.N) close_fn() mat_pred_locs = pred + 1 mat_labeled_locs = np.array(label) + 1 mat_list = gt_list sio.savemat( out_file, { 'pred_locs': mat_pred_locs, 'labeled_locs': mat_labeled_locs, 'list': mat_list }) cur_out.append([ pred, label, gt_list, m, out_file, view, cur_split, ts[mndx] ]) all_preds[curm].append(cur_out) with open( os.path.join(out_dir, args.name, dir_name, args.gt_name + '_results.p'), 'w') as f: pickle.dump(all_preds, f)
def train_theirs(args): H = h5py.File(args.lbl_file, 'r') nviews = int(apt.read_entry(H['cfg']['NumViews'])) all_nets = args.nets for curm in all_nets: for view in range(nviews): if args.split_type is None: cachedir = os.path.join(out_dir, args.name, 'theirs', '{}_view_{}'.format(curm, view), 'full') singularity_script = os.path.join(cachedir, 'singularity.sh') singularity_logfile = os.path.join(cachedir, 'singularity.log') f = open(singularity_script, 'w') f.write('#!/bin/bash\n') f.write('. /opt/venv/bin/activate\n') if curm == 'unet': f.write('cd {}\n'.format(unet_dir)) cmd = 'APT_interface.py -view {} -cache {} -type unet {} train -skip_db'.format( view + 1, cachedir, args.lbl_file) f.write('python {}'.format(cmd)) elif curm == 'openpose': f.write('cd {}\n'.format(openpose_dir)) cmd = 'train_pose.py {} {} {}'.format( args.lbl_file, cachedir, view) f.write('python {}'.format(cmd)) elif curm == 'leap': f.write('cd {}\n'.format(leap_dir)) data_path = os.path.join(cachedir, 'leap_train.h5') cmd = 'leap/training_MK.py {}'.format(data_path) f.write('python {}'.format(cmd)) elif curm == 'deeplabcut': f.write('cd {}\n'.format(cachedir)) cmd = os.path.join(deepcut_dir, 'pose-tensorflow', 'train.py') f.write('python {}'.format(cmd)) else: raise ValueError('Undefined net type: {}'.format(curm)) f.close() os.chmod(singularity_script, stat.S_IEXEC) cmd = '''ssh 10.36.11.34 '. /misc/lsf/conf/profile.lsf; bsub -oo {} -n4 -gpu "num=1" -q gpu_any "singularity exec --nv /misc/local/singularity/branson_v2.simg {}"' '''.format( singularity_logfile, singularity_script ) # -n4 because we use 4 preprocessing threads subprocess.call(cmd, shell=True) print('Submitted job: {}'.format(cmd)) else: for cur_split in range(nsplits): cachedir = os.path.join(out_dir, args.name, '{}_view_{}'.format(curm, view), 'cv_{}'.format(cur_split)) singularity_script = os.path.join(cachedir, 'singularity.sh') singularity_logfile = os.path.join(cachedir, 'singularity.log') f = open(singularity_script, 'w') f.write('#!/bin/bash\n') f.write('. /opt/venv/bin/activate\n') args.skip_db = True if curm == 'unet': f.write('cd {}\n'.format(unet_dir)) cmd = 'APT_interface_mdn.py {} -view {} -cache {} -type unet train -skip_db'.format( args.lbl_file, view + 1, cachedir) f.write('python {}'.format(cmd)) elif curm == 'openpose': f.write('cd {}\n'.format(openpose_dir)) cmd = 'train_pose.py {} {} {}'.format( args.lbl_file, cachedir, view) f.write('python {}'.format(cmd)) elif curm == 'leap': f.write('cd {}\n'.format(leap_dir)) data_path = os.path.join(cachedir, 'leap_train.h5') cmd = 'leap/training_MK.py {}'.format(data_path) f.write('python {}'.format(cmd)) elif curm == 'deeplabcut': f.write('cd {}\n'.format(cachedir)) cmd = os.path.join(deepcut_dir, 'pose-tensorflow', 'train.py') f.write('python {}'.format(cmd)) else: raise ValueError('Undefined net type: {}'.format(curm)) f.close() os.chmod(singularity_script, stat.S_IEXEC) cmd = '''ssh 10.36.11.34 '. /misc/lsf/conf/profile.lsf; bsub -oo {} -n4 -gpu "num=1" -q gpu_tesla "singularity exec --nv /misc/local/singularity/branson_v2.simg {}"' '''.format( singularity_logfile, singularity_script ) # -n4 because we use 4 preprocessing threads subprocess.call(cmd, shell=True) print('Submitted job: {}'.format(cmd))
def train_ours(args): H = h5py.File(args.lbl_file, 'r') nviews = int(apt.read_entry(H['cfg']['NumViews'])) dir_name = 'ours_default' if len(args.nets) == 0: all_nets = methods else: all_nets = args.nets for curm in all_nets: for view in range(nviews): if args.split_type is None: cachedir = os.path.join(out_dir, args.name, dir_name, '{}_view_{}'.format(curm, view), 'full') singularity_script = os.path.join(cachedir, 'singularity.sh') singularity_logfile = os.path.join(cachedir, 'singularity.log') f = open(singularity_script, 'w') f.write('#!/bin/bash\n') f.write('. /opt/venv/bin/activate\n') f.write('cd {}\n'.format(unet_dir)) cmd = 'APT_interface_mdn.py {} -view {} -cache {} -type {} train -skip_db'.format( args.lbl_file, view + 1, cachedir, curm) if args.whose == 'ours_default': cmd += ' -use_defaults' f.write('python {}'.format(cmd)) f.close() os.chmod(singularity_script, 0o755) cmd = '''ssh 10.36.11.34 '. /misc/lsf/conf/profile.lsf; bsub -oo {} -n4 -gpu "num=1" -q gpu_tesla "singularity exec --nv /misc/local/singularity/branson_v2.simg {}"' '''.format( singularity_logfile, singularity_script ) # -n4 because we use 4 preprocessing threads subprocess.call(cmd, shell=True) print('Submitted job: {}'.format(cmd)) else: for cur_split in range(nsplits): cachedir = os.path.join(out_dir, args.name, '{}_view_{}'.format(curm, view), 'cv_{}'.format(cur_split)) singularity_script = os.path.join(cachedir, 'singularity.sh') singularity_logfile = os.path.join(cachedir, 'singularity.log') f = open(singularity_script, 'w') f.write('#!/bin/bash\n') f.write('. /opt/venv/bin/activate\n') f.write('cd {}\n'.format(unet_dir)) cmd = 'APT_interface_mdn.py {} -view {} -cache {} -type {} train -skip_db'.format( args.lbl_file, view + 1, cachedir, curm) if args.whose == 'ours_default': cmd += ' -use_defaults' f.write('python {}'.format(cmd)) f.close() os.chmod(singularity_script, 0o755) cmd = '''ssh 10.36.11.34 '. /misc/lsf/conf/profile.lsf; bsub -oo {} -n4 -gpu "num=1" -q gpu_tesla "singularity exec --nv /misc/local/singularity/branson_v2.simg {}"' '''.format( singularity_logfile, singularity_script ) # -n4 because we use 4 preprocessing threads subprocess.call(cmd, shell=True) print('Submitted job: {}'.format(cmd))
def create_db(args): H = h5py.File(args.lbl_file, 'r') nviews = int(apt.read_entry(H['cfg']['NumViews'])) all_nets = args.nets all_split_files = [] for view in range(nviews): if args.split_type is not None and not args.split_type.startswith( 'prog'): cachedir = os.path.join(out_dir, args.name, 'common') if not os.path.exists(cachedir): os.mkdir(cachedir) cachedir = os.path.join(out_dir, args.name, 'common', 'splits_{}'.format(view)) if not os.path.exists(cachedir): os.mkdir(cachedir) conf = apt.create_conf(args.lbl_file, view, args.name, cache_dir=cachedir) conf.splitType = args.split_type print("Split type is {}".format(conf.splitType)) if args.do_split: train_info, val_info, split_files = apt.create_cv_split_files( conf, nsplits) else: split_files = [ os.path.join(conf.cachedir, 'cv_split_fold_{}.json'.format(ndx)) for ndx in range(nsplits) ] all_split_files.append(split_files) for curm in all_nets: for view in range(nviews): if args.split_type is None: cachedir = os.path.join(out_dir, args.name, 'common', '{}_view_{}'.format(curm, view), 'full') conf = apt.create_conf(args.lbl_file, view, args.name, cache_dir=cachedir) if not args.only_check: if not os.path.exists(conf.cachedir): os.makedirs(conf.cachedir) if curm == 'unet' or curm == 'openpose': apt.create_tfrecord(conf, False) elif curm == 'leap': apt.create_leap_db(conf, False) elif curm == 'deeplabcut': apt.create_deepcut_db(conf, False) create_deepcut_cfg(conf) else: raise ValueError('Undefined net type: {}'.format(curm)) check_db(curm, conf) elif args.split_type.startswith('prog'): split_type = args.split_type[5:] all_info = get_increasing_splits(conf, split_type) for cur_tr in prog_thresholds: cachedir = os.path.join(out_dir, args.name, 'common', '{}_view_{}'.format(curm, view), '{}'.format(cur_tr)) conf = apt.create_conf(args.lbl_file, view, args.name, cache_dir=cachedir) split_ndx = round(len(all_info) / cur_tr) cur_train = all_info[:split_ndx] cur_val = all_info[split_ndx:] split_file = os.path.join(cachedir, 'splitdata.json') with open(split_file, 'w') as f: json.dump([cur_train, cur_val], f) if not args.only_check: if curm == 'unet' or curm == 'openpose': apt.create_tfrecord(conf, True, split_file) elif curm == 'leap': apt.create_leap_db(conf, True, split_file) elif curm == 'deeplabcut': apt.create_deepcut_db(conf, True, split_file) create_deepcut_cfg(conf) else: raise ValueError( 'Undefined net type: {}'.format(curm)) check_db(curm, conf) else: split_files = all_split_files[view] for cur_split in range(nsplits): conf.cachedir = os.path.join( out_dir, args.name, 'common', '{}_view_{}'.format(curm, view)) if not os.path.exists(conf.cachedir): os.mkdir(conf.cachedir) conf.cachedir = os.path.join( out_dir, args.name, 'common', '{}_view_{}'.format(curm, view), 'cv_{}'.format(cur_split)) if not os.path.exists(conf.cachedir): os.mkdir(conf.cachedir) conf.splitType = 'predefined' split_file = split_files[cur_split] if not args.only_check: if curm == 'unet' or curm == 'openpose': apt.create_tfrecord(conf, True, split_file) elif curm == 'leap': apt.create_leap_db(conf, True, split_file) elif curm == 'deeplabcut': apt.create_deepcut_db(conf, True, split_file) create_deepcut_cfg(conf) else: raise ValueError( 'Undefined net type: {}'.format(curm)) check_db(curm, conf) base_dir = os.path.join(out_dir, args.name, 'common') their_dir = os.path.join(out_dir, args.name, 'theirs') our_dir = os.path.join(out_dir, args.name, 'ours') our_default_dir = os.path.join(out_dir, args.name, 'ours_default') cmd = 'cp -rs {} {}'.format(base_dir, their_dir) os.system(cmd) cmd = 'cp -rs {} {}'.format(base_dir, our_dir) os.system(cmd) cmd = 'cp -rs {} {}'.format(base_dir, our_default_dir) os.system(cmd)