def on_epoch_end(self, epoch, logs={}): model = self.model scores = measures.eval_human36m_sc_error(model, self.num_blocks, self.x, self.pw, self.afmat, self.rootz, self.scam, pose_only=self.pose_only, batch_size=self.batch_size) epoch += 1 if self.logdir is not None: if not hasattr(self, 'logarray'): self.logarray = {} self.logarray[epoch] = scores with open(os.path.join(self.logdir, 'h36m_val.json'), 'w') as f: f.write(json.dumps(self.logarray)) cur_best = min(scores) self.scores[epoch] = cur_best log.printcn( log.OKBLUE, 'Best score is %.1f at epoch %d' % (self.best_score, self.best_epoch))
def compute_mpjpe(nb_blocks, pred, pose_w, afmat, rootz, scam, resol_z=2000., verbose=True): ''' smaller version of eval_human36m_sc_error takes the prediction nb_blocks: nb pose prediction blocks of the model (there is one y_pred per block) pred: the prediction tensor (pose part only) shape : (nb_blocks, batch_size, nb_joints, dim + 1) i.e (2, 16, 17, 4) pose_w: the ground truth pose in the world in millimeters shape : (batch_size, nb_joints, dim) i.e (16, 17, 3) afmat: transformation matrix used to convert uvd coordinates of the crop to the uvd coordinates of the original image rootz: origin of the z axis scam: camera used to convert uvd coordinates (uv pixels and depth in mm) to world (millimeters) coordinates ''' print("MPJPE: pred shape %s" % pred.shape) print("MPJPE: pose_w shape %s" % pose_w.shape) print("MPJPE: afmat shape %s" % afmat.shape) print("MPJPE: scam shape %s" % scam.shape) # assert len(pred) == len(pose_w) == len(afmat) == len(scam) # y true in the world y_true_w = pose_w if rootz.get_shape().ndims == 1: rootz = tf.expand_dims(rootz, axis=-1) # Move the root joints from GT poses to origin y_true_w -= y_true_w[:, 0:1, :] if verbose: log.printc(log.WARNING, 'Avg. mm. error:') lower_err = np.inf scores = [] for b in range(nb_blocks): y_pred = pred[b] y_pred_w = pred_to_world_tf(y_pred, y_true_w, afmat, scam, rootz) err_w = mean_distance_error(y_true_w[:, 0:, :], y_pred_w[:, 0:, :]) scores.append(err_w) if verbose: log.printc(log.WARNING, ' %.1f' % err_w) # Keep the best prediction if err_w < lower_err: lower_err = err_w if verbose: log.printcn('', '') log.printcn(log.WARNING, 'Final averaged error (mm): %.3f' % lower_err) return scores
def build_callbacks(self, prop, relabel_step=None): ''' prop = proportion of known labels of current run TensorBoard MAPCallback SaveModel LearningRateScheduler ''' log.printcn(log.OKBLUE, 'Building callbacks') cb_list = list() # # tensorboard # logs_folder = os.environ['HOME'] + '/partial_experiments/tensorboard/' + self.exp_folder.split('/')[-1] + '/prop%s' % prop # log.printcn(log.OKBLUE, 'Tensorboard log folder %s' % logs_folder) # tensorboard = TensorBoard(log_dir=os.path.join(logs_folder, 'tensorboard')) # cb_list.append(tensorboard) # Validation callback if cfg.CALLBACK.VAL_CB is not None: cb_list.append( self.build_val_cb(cfg.CALLBACK.VAL_CB, p=prop, relabel_step=relabel_step)) else: log.printcn(log.WARNING, 'Skipping validation callback') # Save Model cb_list.append( SaveModel(self.exp_folder, prop, relabel_step=relabel_step)) # Learning rate scheduler cb_list.append(LearningRateScheduler(lr_scheduler)) return cb_list
def eval_baseline_h36m(): weights_path = sys.args[1] eval_model = MultiBranchModel( dim=3, n_joints=17, nb_pose_blocks=1) # TODO retrieve from config eval_model.load_weights(weights_path, pose_only=True) # local loading local_h36m_path = '/home/caleml/datasets/h36m' local_h36m = Human36M(local_h36m_path, dataconf=config.human36m_dataconf, poselayout=pose_format.pa17j3d, topology='frames') h36m_val = BatchLoader(local_h36m, ['frame'], ['pose_w', 'pose_uvd', 'afmat', 'camera'], VALID_MODE, batch_size=local_h36m.get_length(VALID_MODE), shuffle=True) log.printcn(log.OKBLUE, 'Preloading Human3.6M validation samples...') [x_val], [pw_val, puvd_val, afmat_val, scam_val] = h36m_val[0] scores = eval_human36m_sc_error(eval_model.model, x_val, pw_val, afmat_val, puvd_val[:, 0, 2], scam_val, batch_size=24) pprint(scores)
def lr_scheduler(epoch, lr): if epoch in [20, 30]: newlr = 0.5 * lr log.printcn(log.WARNING, 'lr_scheduler: lr %g -> %g @ %d' % (lr, newlr, epoch)) else: newlr = lr log.printcn(log.OKBLUE, 'lr_scheduler: lr %g @ %d' % (newlr, epoch)) return newlr
def lr_scheduler(epoch, lr): if epoch in cfg.CALLBACK.LR_TRIGGER: newlr = cfg.CALLBACK.LR_FACTOR * lr log.printcn(log.WARNING, 'lr_scheduler: lr %g -> %g @ %d' % (lr, newlr, epoch)) else: newlr = lr log.printcn(log.OKBLUE, 'lr_scheduler: lr %g @ %d' % (newlr, epoch)) return newlr
def finish_step(self, relabel_step): assert relabel_step == self.relabel_step relabel_logpath = os.path.join(self.exp_folder, 'relabeling', 'log_relabeling.csv') with open(relabel_logpath, 'a') as f_log: f_log.write('%s,%s,%s,%s,%s\n' % (self.p, self.relabel_step, self.total_added, self.positive_added, self.negative_added)) log.printcn( log.OKBLUE, '\tAdded %s labels during relabeling, logging into %s' % (self.total_added, relabel_logpath)) log.printcn(log.OKBLUE, '\tNew dataset path %s' % (self.targets_path)) self.f_relabel.close()
def launch_percentage_relabel(self, p): ''' For a given known label percentage p: 1. load dataset 3. callbacks 4. load / build model 5. train ''' self.dataset_train = self.load_dataset(mode=cfg.DATASET.TRAIN, batch_size=cfg.BATCH_SIZE, p=p) self.dataset_test = self.load_dataset(mode=cfg.DATASET.TEST, batch_size=cfg.TEST_BATCH_SIZE) # model self.build_model(self.dataset_train.nb_classes, p) self.relabelator = self.load_relabelator(p, self.dataset_train.nb_classes) for relabel_step in range(cfg.RELABEL.STEPS): log.printcn(log.OKBLUE, '\nDoing relabel step %s' % (relabel_step)) # callbacks cb_list = self.build_callbacks(p, relabel_step=relabel_step) # actual training n_epochs = cfg.TRAINING.N_EPOCHS if cfg.RELABEL.EPOCHS is None else cfg.RELABEL.EPOCHS[ relabel_step] steps_per_epoch = len( self.dataset_train ) if not cfg.TRAINING.STEPS_PER_EPOCH else cfg.TRAINING.STEPS_PER_EPOCH self.model.train(self.dataset_train, steps_per_epoch=steps_per_epoch, cb_list=cb_list, n_epochs=n_epochs, dataset_val=self.dataset_test) # relabeling self.relabel_dataset(relabel_step) # cleaning (to release memory before next launch) K.clear_session() del self.model
def finish_step(self, relabel_step): # log relabelling stats relabel_logpath = os.path.join(self.exp_folder, 'relabeling', 'log_relabeling.csv') os.makedirs(os.path.dirname(relabel_logpath), exist_ok=True) with open(relabel_logpath, 'a') as f_log: f_log.write('{},{},{},{}\n'.format(self.p, relabel_step, self.TP, self.FP)) log.printcn( log.OKBLUE, '\tAdded {} TP and {} FP, logging into {}'.format( self.TP, self.FP, relabel_logpath)) # Setting the path to the new targets self.targets_path = os.path.join(self.data_dir, 'relabeled_annotations', self.exp_name, '{}', str(self.p), str(relabel_step))
def build_val_cb(self, cb_name, p, relabel_step=None): ''' Validation callback Different datasets require different validations, like mAP, DICE, etc This callback is instanciated here ''' if cb_name == 'map': log.printcn(log.OKBLUE, 'loading mAP callback') X_test, Y_test = self.dataset_test[0] map_cb = MAPCallback(X_test, Y_test, self.exp_folder, p, relabel_step=relabel_step) return map_cb else: raise Exception('Invalid validation callback %s' % cb_name)
def exp_init(cmd, exps_folder=None, exp_name=None): ''' common actions for setuping an experiment: - create experiment folder - dump config in it - create cmd file - dump current model code in it (because for now we only save weights) ''' if exps_folder is None: exps_folder = os.path.join(os.environ['HOME'], 'prior_experiments') # model folder name_suffix = ('_%s' % exp_name) if exp_name else '' model_folder = '%s/exp_%s_%s%s' % (exps_folder, datetime.datetime.now( ).strftime("%Y%m%d_%H%M"), cfg.ARCHI.NAME, name_suffix) os.makedirs(model_folder) n_epochs = cfg.TRAINING.NB_EPOCH if cfg.RELABEL.EPOCHS is None else sum( cfg.RELABEL.EPOCHS) log.printcn( log.OKBLUE, "Conducting experiment for %s epochs in folder %s" % (n_epochs, model_folder)) # config config_path = os.path.join(model_folder, 'config.yaml') with open(config_path, 'w+') as f_conf: yaml.dump(cfg, f_conf, default_flow_style=False) # cmd cmd_path = os.path.join(model_folder, 'cmd.txt') with open(cmd_path, 'w+') as f_cmd: f_cmd.write(cmd + '\n') # model src_folder = os.path.dirname(os.path.realpath(__file__)) + '/..' dst_folder = os.path.join(model_folder, 'model_src/') shutil.copytree(src_folder, dst_folder, ignore=shutil.ignore_patterns('.*')) return model_folder
def launch_training(dataset_path, model_folder, n_epochs, batch_size, pose_blocks): # loading dataset h36m_path = dataset_path h36m = Human36M(h36m_path, dataconf=config.human36m_dataconf, poselayout=pose_format.pa17j3d, topology='frames') data_tr_h36m = BatchLoader(h36m, ['frame'], ['pose'] * pose_blocks, TRAIN_MODE, batch_size=batch_size, shuffle=True) # validation h36m_val = BatchLoader(h36m, ['frame'], ['pose_w', 'pose_uvd', 'afmat', 'camera'], VALID_MODE, batch_size=h36m.get_length(VALID_MODE), shuffle=True) log.printcn(log.OKBLUE, 'Preloading Human3.6M validation samples...') [x_val], [pw_val, puvd_val, afmat_val, scam_val] = h36m_val[0] eval_callback = callbacks.H36MEvalCallback(pose_blocks, x_val, pw_val, afmat_val, puvd_val[:, 0, 2], scam_val, pose_only=True, logdir=model_folder) # model model = MultiBranchModel(dim=3, n_joints=17, nb_pose_blocks=pose_blocks) model.build_pose_only() model.add_callback(eval_callback) model.train(data_tr_h36m, steps_per_epoch=len(data_tr_h36m), model_folder=model_folder, n_epochs=n_epochs)
def relabel_dataset(self, relabel_step): ''' Use model to make predictions Use predictions to relabel elements (create a new relabeled csv dataset) Use created csv to update dataset train ''' log.printcn(log.OKBLUE, '\nDoing relabeling inference step %s' % relabel_step) self.relabelator.init_step(relabel_step) # predict for i in range(len(self.dataset_train)): x_batch, y_batch = self.dataset_train[i] y_pred = self.model.predict(x_batch) # TODO not the logits!!!!!!!! self.relabelator.relabel(x_batch, y_batch, y_pred) self.relabelator.finish_step(relabel_step) targets_path = self.relabelator.targets_path # update dataset self.dataset_train.update_targets(targets_path)
help= 'the specific percentage of known labels. When not specified all percentages are sequentially launched' ) parser.add_argument('--exp_name', '-n', help='optional experiment name') parser.add_argument( '--initial_weights', '-w', help='optional path to the weights that should be loaded') # options management args = parser.parse_args() options = config_utils.parse_options_file(args.options) config_utils.update_config(options) # init exp_folder = utils.exp_init(' '.join(sys.argv), exp_name=(args.exp_name or args.options)) os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu try: launcher = Launcher(exp_folder, percent=args.percent, initial_weights=args.initial_weights) launcher.launch() finally: # cleanup if needed (test folders) if cfg.CLEANUP is True: log.printcn(log.OKBLUE, 'Cleaning folder %s' % (exp_folder)) shutil.rmtree(exp_folder)
def launch(self): ''' main entrypoint 1. training data 2. validation data 3. callbacks 4. model building 5. launch actual training ''' h36m = Human36M(self.dataset_path, dataconf=config.human36m_dataconf, poselayout=pose_format.pa17j3d, topology='frames') # training data dataset_output = self.get_h36m_outputs() data_tr_h36m = BatchLoader(h36m, ['frame'], dataset_output, TRAIN_MODE, batch_size=self.batch_size, shuffle=True) # validation data h36m_val = BatchLoader(h36m, ['frame'], ['pose_w', 'pose_uvd', 'afmat', 'camera'], VALID_MODE, batch_size=h36m.get_length(VALID_MODE), shuffle=True) log.printcn(log.OKBLUE, 'Preloading Human3.6M validation samples...') [x_val], [pw_val, puvd_val, afmat_val, scam_val] = h36m_val[0] # assert rootz == puvd_val[:,0,2] # callbacks cb_list = list() eval_callback = callbacks.H36MEvalCallback(self.pose_blocks, x_val, pw_val, afmat_val, puvd_val[:, 0, 2], scam_val, pose_only=self.pose_only, logdir=self.model_folder) logs_folder = os.environ[ 'HOME'] + '/pe_experiments/tensorboard/' + self.model_folder.split( '/')[-1] print('Tensorboard log folder %s' % logs_folder) tensorboard = TensorBoard( log_dir=os.path.join(logs_folder, 'tensorboard')) cb_list.append(tensorboard) cb_list.append(eval_callback) cb_list.append(LearningRateScheduler(lr_scheduler)) cb_list.append(callbacks.SaveModel(self.model_folder)) # model self.build_model() # train self.model.train(data_tr_h36m, steps_per_epoch=len(data_tr_h36m), model_folder=self.model_folder, n_epochs=self.n_epochs, cb_list=cb_list, n_workers=self.nb_workers)
def log(self, msg): if self.verbose: log.printcn(log.HEADER, msg)
def build_model(self): if self.exp_type == 'baseline': self.model = MultiBranchModel(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build_pose_only() elif self.exp_type == 'hybrid': self.model = MultiBranchModel(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build() elif self.exp_type == 'hybrid_reduced': assert isinstance(self.zp_depth, int), 'wrong zp_depth %s' % self.zp_depth log.printcn( log.OKBLUE, 'launching hybrid_reduced model with zp_depth = %s' % self.zp_depth) self.model = MultiBranchReduced(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks, zp_depth=self.zp_depth) self.model.build() elif self.exp_type == 'hybrid_vgg': self.model = MultiBranchVGGModel(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build() elif self.exp_type == 'hybrid_stop': self.model = MultiBranchStopped(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks, zp_depth=self.zp_depth) self.model.build() elif self.exp_type == 'hybrid_r_bb': self.model = MBMReducedBB(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks, zp_depth=self.zp_depth) self.model.build() elif self.exp_type == 'cycle': self.model = CycleModel(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build() elif self.exp_type == 'cycle_reduced': self.model = CycleReduced(dim=self.dim, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build() elif self.exp_type == 'cycle_r_bb': self.model = CycleReducedBB(dim=self.dim, cut_zp=self.cut_zp, n_joints=self.nb_joints, nb_pose_blocks=self.pose_blocks) self.model.build() else: raise Exception('Unknown exp type %s' % self.exp_type)
def eval_human36m_sc_error(model, num_blocks, x, pose_w, afmat, rootz, scam, pose_only=False, resol_z=2000., batch_size=8, logdir=None, verbose=True): assert len(x) == len(pose_w) == len(afmat) == len(scam) y_true_w = pose_w.copy() y_pred_w = np.zeros((num_blocks,) + y_true_w.shape) if rootz.ndim == 1: rootz = np.expand_dims(rootz, axis=-1) pred = model.predict(x, batch_size=batch_size, verbose=1) # Move the root joints from GT poses to origin y_true_w -= y_true_w[:, 0:1, :] if verbose: log.printc(log.WARNING, 'Avg. mm. error:') lower_err = np.inf scores = [] for b in range(num_blocks): if pose_only: y_pred = pred[b] else: y_pred = pred[b + 1] # first output is image and pose output start after # take only the spatial dims y_pred = y_pred[:, :, 0:3] # project normalized coordiates to the image plane y_pred[:, :, 0:2] = transform.transform_pose_sequence(afmat.copy(), y_pred[:, :, 0:2], inverse=True) # Recover the absolute Z y_pred[:, :, 2] = (resol_z * (y_pred[:, :, 2] - 0.5)) + rootz y_pred_uvd = y_pred[:, :, 0:3] # camera inverse projection for j in range(len(y_pred_uvd)): cam = camera.camera_deserialize(scam[j]) y_pred_w[b, j, :, :] = cam.inverse_project(y_pred_uvd[j]) # Move the root joint from predicted poses to the origin y_pred_w[b, :, :, :] -= y_pred_w[b, :, 0:1, :] err_w = mean_distance_error(y_true_w[:, 0:, :], y_pred_w[b, :, 0:, :]) scores.append(err_w) if verbose: log.printc(log.WARNING, ' %.1f' % err_w) # Keep the best prediction if err_w < lower_err: lower_err = err_w if verbose: log.printcn('', '') if logdir is not None: np.save('%s/y_pred_w.npy' % logdir, y_pred_w) np.save('%s/y_true_w.npy' % logdir, y_true_w) log.printcn(log.WARNING, 'Final averaged error (mm): %.3f' % lower_err) return scores