def main(args): cfg = setup(args) if args.eval_only: model = Trainer.build_model(cfg) if args.eval_iter != -1: # load checkpoint at specified iteration ckpt_file = os.path.join( cfg.OUTPUT_DIR, 'model_{:07d}.pth'.format(args.eval_iter - 1)) resume = False else: # load checkpoint at last iteration ckpt_file = cfg.MODEL.WEIGHTS resume = True DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( ckpt_file, resume=resume) res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) # save evaluation results in json os.makedirs(os.path.join(cfg.OUTPUT_DIR, 'inference'), exist_ok=True) with open( os.path.join(cfg.OUTPUT_DIR, 'inference', 'res_final.json'), 'w') as fp: json.dump(res, fp) return res elif args.eval_all: tester = Tester(cfg) all_ckpts = sorted(tester.check_pointer.get_all_checkpoint_files()) for i, ckpt in enumerate(all_ckpts): ckpt_iter = ckpt.split('model_')[-1].split('.pth')[0] if ckpt_iter.isnumeric() and int(ckpt_iter) + 1 < args.start_iter: # skip evaluation of checkpoints before start iteration continue if args.end_iter != -1: if not ckpt_iter.isnumeric( ) or int(ckpt_iter) + 1 > args.end_iter: # skip evaluation of checkpoints after end iteration break tester.test(ckpt) return best_res elif args.eval_during_train: tester = Tester(cfg) saved_checkpoint = None while True: if tester.check_pointer.has_checkpoint(): current_ckpt = tester.check_pointer.get_checkpoint_file() if saved_checkpoint is None or current_ckpt != saved_checkpoint: saved_checkpoint = current_ckpt tester.test(current_ckpt) time.sleep(10) else: if comm.is_main_process(): print( 'Please specify --eval-only, --eval-all, or --eval-during-train' )
def train(self): """ Run training. Returns: OrderedDict of results, if evaluation is enabled. Otherwise None. """ super().train(self.start_iter, self.max_iter) if hasattr(self, "_last_eval_results") and comm.is_main_process(): verify_results(self.cfg, self._last_eval_results) return self._last_eval_results
def main(args): cfg = setup(args) if args.eval_only: model = Trainer.build_model(cfg) DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) return res """ If you'd like to do anything fancier than the standard training logic, consider writing your own training loop or subclassing the trainer. """ trainer = Trainer(cfg) trainer.resume_or_load(resume=args.resume) return trainer.train()
def test(self, ckpt): self.check_pointer._load_model(self.check_pointer._load_file(ckpt)) print('evaluating checkpoint {}'.format(ckpt)) res = Trainer.test(self.cfg, self.model) if comm.is_main_process(): verify_results(self.cfg, res) print(res) if (self.best_res is None) or ( self.best_res is not None and self.best_res['bbox']['AP'] < res['bbox']['AP']): self.best_res = res self.best_file = ckpt print('best results from checkpoint {}'.format(self.best_file)) print(self.best_res) self.all_res["best_file"] = self.best_file self.all_res["best_res"] = self.best_res self.all_res[ckpt] = res os.makedirs(os.path.join(self.cfg.OUTPUT_DIR, 'inference'), exist_ok=True) with open( os.path.join(self.cfg.OUTPUT_DIR, 'inference', 'all_res.json'), 'w') as fp: json.dump(self.all_res, fp)