# read config
if args.resumed:
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
continue_state_object = torch.load(args.resumed,
map_location=torch.device("cpu"))
config = continue_state_object['config']
solver.init_from_checkpoint(
continue_state_object=continue_state_object)
if is_main_process:
snap_dir = args.resumed[:-len(args.resumed.split('/')[-1])]
if not os.path.exists(snap_dir):
logging.error('[Error] {} is not existed.'.format(snap_dir))
raise FileNotFoundError
else:
config = load_config(args.config)
pathcfg = load_config(args.pathcfg)
config = merge_config(config, pathcfg)
solver.init_from_scratch(config)
if is_main_process:
exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime())
if isinstance(config['data']['name'], list):
snap_dir = os.path.join(config["snap"]["path"],
config['data']['name'][0],
config['model']['name'], exp_time)
else:
snap_dir = os.path.join(config["snap"]["path"],
config['data']['name'],
config['model']['name'], exp_time)
if not os.path.exists(snap_dir):
logging.error(
'args --config and --resumed should at least one value available.')
raise ValueError
is_main_process = True if args.local_rank == 0 else False
solver = Solver()
# read config
if args.resumed:
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
continue_state_object = torch.load(args.resumed,
map_location=torch.device("cpu"))
config = continue_state_object['config']
### Minghan: only use this line when the trained model and the evaluation is not on the same machine
if args.config:
override_cfg = load_config(args.config)
config.update(override_cfg)
solver.init_from_checkpoint(continue_state_object=continue_state_object)
if is_main_process:
snap_dir = args.resumed[:-len(args.resumed.split('/')[-1])]
if not os.path.exists(snap_dir):
logging.error('[Error] {} is not existed.'.format(snap_dir))
raise FileNotFoundError
else:
config = load_config(args.config)
pathcfg = load_config(args.pathcfg)
config = merge_config(config, pathcfg)
solver.init_from_scratch(config)
if is_main_process:
solver = Solver()
# read config
if args.resumed:
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
continue_state_object = torch.load(args.resumed,
map_location=torch.device("cpu"))
config = continue_state_object['config']
solver.init_from_checkpoint(continue_state_object=continue_state_object)
if is_main_process:
snap_dir = args.resumed[:-len(args.resumed.split('/')[-1])]
if not os.path.exists(snap_dir):
logging.error('[Error] {} is not existed.'.format(snap_dir))
raise FileNotFoundError
else:
config = load_config(args.config)
solver.init_from_scratch(config)
if is_main_process:
exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime())
snap_dir = os.path.join(config["snap"]["path"], config['data']['name'],
config['model']['name'], exp_time)
if not os.path.exists(snap_dir):
os.makedirs(snap_dir)
if is_main_process:
print_config(config)
# dataset
tr_loader, sampler, niter_per_epoch = build_loader(config,
True,
solver.world_size,