def test_checkpoint(self): savedir_base = '.results' # create exp folder exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} savedir = os.path.join(savedir_base, hu.hash_dict(exp_dict)) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) hu.torch_save(os.path.join(savedir, "model.pth"), torch.zeros(10)) hu.torch_load(os.path.join(savedir, "model.pth")) assert(os.path.exists(savedir)) # delete exp folder hc.delete_experiment(savedir) assert(not os.path.exists(savedir)) # check backup folder os.rmdir(savedir_base)
def test_checkpoint(): savedir_base = ".results" # create exp folder exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } savedir = os.path.join(savedir_base, hu.hash_dict(exp_dict)) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) hu.torch_save(os.path.join(savedir, "model.pth"), torch.zeros(10)) hu.torch_load(os.path.join(savedir, "model.pth")) hc.load_checkpoint(exp_dict, savedir_base, fname="model.pth") assert os.path.exists(savedir) # delete exp folder hc.delete_experiment(savedir) assert not os.path.exists(savedir) # check backup folder os.rmdir(savedir_base)
def trainval(exp_dict, savedir_base, reset=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) # Dataset # ----------- # train loader train_loader = datasets.get_loader(dataset_name=exp_dict['dataset'], datadir=savedir_base, split='train') # val loader val_loader = datasets.get_loader(dataset_name=exp_dict['dataset'], datadir=savedir_base, split='val') # Model # ----------- model = models.get_model(model_name=exp_dict['model']) # Checkpoint # ----------- model_path = os.path.join(savedir, 'model.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print('Starting experiment at epoch %d' % (s_epoch)) for e in range(s_epoch, 10): score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate the model val_dict = model.val_on_loader(val_loader) # Get metrics score_dict['train_loss'] = train_dict['train_loss'] score_dict['val_acc'] = val_dict['val_acc'] score_dict['epoch'] = e # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print(score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print('Checkpoint Saved: %s' % savedir) print('experiment completed')
def trainval(exp_dict, savedir_base, datadir_base, reset=False, num_workers=0, pin_memory=False, ngpu=1, cuda_deterministic=False): # bookkeeping # ================== # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) if DEVICE.type == "cuda": if cuda_deterministic: cudnn.benchmark = False cudnn.deterministic = True else: cudnn.benchmark = True # Dataset # ================== trainset = get_dataset(exp_dict['dataset'], 'train', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['train'], transform_lvl=exp_dict['dataset']['transform_lvl'], colorjitter=exp_dict['dataset'].get('colorjitter') ) valset = get_dataset(exp_dict['dataset'], 'validation', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['train'], transform_lvl=0, val_transform=exp_dict['dataset']['val_transform']) testset = get_dataset(exp_dict['dataset'], 'test', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['test'], transform_lvl=0, val_transform=exp_dict['dataset']['val_transform']) print("Dataset defined.") # define dataloaders if exp_dict['dataset']['name'] == 'bach': testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) else: testloader = torch.utils.data.DataLoader(testset, batch_size=exp_dict['batch']['size'], shuffle=False, num_workers=num_workers, pin_memory=pin_memory) print("Testloader defined.") # Model # ================== model = get_model(exp_dict, trainset, device=DEVICE) print("Model loaded") model_path = os.path.join(savedir, 'model.pth') model_best_path = os.path.join(savedir, 'model_best.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') # checkpoint management if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = len(score_list) else: # restart experiment score_list = [] s_epoch = 0 # define and log random seed for reproducibility assert('fixedSeed' in exp_dict) seed = exp_dict['fixedSeed'] random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) np.random.seed(seed) print("Seed defined.") # Train & Val # ================== print("Starting experiment at epoch %d/%d" % (s_epoch, exp_dict['niter'])) for epoch in range(s_epoch, exp_dict['niter']): s_time = time.time() # Sample new train val trainloader, valloader = get_train_val_dataloader(exp_dict, trainset, valset, mixtrainval=exp_dict['mixTrainVal'], num_workers=num_workers, pin_memory=pin_memory) # Train & validate train_dict = model.train_on_loader(trainloader, valloader, epoch=epoch, exp_dict=exp_dict) # Test phase train_dict_2 = model.test_on_loader(trainloader) val_dict = model.test_on_loader(valloader) test_dict = model.test_on_loader(testloader) # Vis phase model.vis_on_loader('train', trainset, savedir_images=os.path.join( savedir, 'images'), epoch=epoch) score_dict = {} score_dict["epoch"] = epoch score_dict["test_acc"] = test_dict['acc'] score_dict["val_acc"] = val_dict['acc'] score_dict["train_acc"] = train_dict_2['acc'] score_dict["train_loss"] = train_dict['loss'] score_dict["time_taken"] = time.time() - s_time score_dict["netC_lr"] = train_dict['netC_lr'] if exp_dict['model']['netA'] is not None: if 'transformations_mean' in train_dict: for i in range(len(train_dict['transformations_mean'])): score_dict[str( i) + "_mean"] = train_dict['transformations_mean'][i].item() if 'transformations_std' in train_dict: for i in range(len(train_dict['transformations_std'])): score_dict[str( i) + "_std"] = train_dict['transformations_std'][i].item() # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Update best score if epoch == 0 or (score_dict["test_acc"] >= score_df["test_acc"][:-1].max()): hu.save_pkl(os.path.join( savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) print('experiment completed')
def test(exp_dict, savedir_base, datadir, num_workers=0, model_path=None, scan_id=None): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ================== # val set test_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) if str(scan_id) != 'None': test_set.active_data = test_set.get_scan(scan_id) test_sampler = torch.utils.data.SequentialSampler(test_set) test_loader = DataLoader(test_set, sampler=test_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) # Model # ================== # chk = torch.load('best_model.ckpt') model = models.get_model_for_onnx_export(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=test_set).cuda() epoch = -1 if str(model_path) != 'None': model_path = model_path model.load_state_dict(hu.torch_load(model_path)) else: try: exp_dict_train = copy.deepcopy(exp_dict) del exp_dict_train['test_mode'] savedir_train = os.path.join(savedir_base, hu.hash_dict(exp_dict_train)) model_path = os.path.join(savedir_train, "model_best.pth") score_list = hu.load_pkl( os.path.join(savedir_train, 'score_list_best.pkl')) epoch = score_list[-1]['epoch'] print('Loaded model at epoch %d with score %.3f' % epoch) model.load_state_dict(hu.torch_load(model_path)) except: pass s_time = time.time() savedir_images = os.path.join(savedir, 'images') # delete image folder if exists if os.path.exists(savedir_images): shutil.rmtree(savedir_images) os.makedirs(savedir_images, exist_ok=True) # for i in range(20): # score_dict = model.train_on_loader(test_loader) score_dict = model.val_on_loader(test_loader, savedir_images=savedir_images, n_images=30000, save_preds=True) score_dict['epoch'] = epoch score_dict["time"] = time.time() - s_time score_dict["saved_at"] = hu.time_to_montreal() # save test_score_list test_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(test_path): test_score_list = [ sd for sd in hu.load_pkl(test_path) if sd['epoch'] != epoch ] else: test_score_list = [] # append score_dict to last result test_score_list += [score_dict] hu.save_pkl(test_path, test_score_list) print('Final Score is ', str(score_dict["val_score"]) + "\n")
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # set seed # ================== seed = 42 np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) # Dataset # ================== # train set train_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="train", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val set val_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # test set test_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="test", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val_sampler = torch.utils.data.SequentialSampler(val_set) val_loader = DataLoader( val_set, # sampler=val_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) test_loader = DataLoader( test_set, # sampler=val_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) # Model # ================== model = models.get_model(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() # model.opt = optimizers.get_optim(exp_dict['opt'], model) model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) model.waiting = 0 model.val_score_best = -np.inf train_sampler = torch.utils.data.RandomSampler(train_set, replacement=True, num_samples=2 * len(test_set)) train_loader = DataLoader(train_set, sampler=train_sampler, collate_fn=ut.collate_fn, batch_size=exp_dict["batch_size"], drop_last=True, num_workers=num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} test_dict = model.val_on_loader(test_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) # Train the model train_dict = model.train_on_loader(train_loader) # Validate the model val_dict = model.val_on_loader(val_loader) score_dict["val_score"] = val_dict["val_score"] # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = e score_dict["waiting"] = model.waiting model.waiting += 1 # Add to score_list and save checkpoint score_list += [score_dict] # Save Best Checkpoint score_df = pd.DataFrame(score_list) if score_dict["val_score"] >= model.val_score_best: test_dict = model.val_on_loader(test_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) score_dict.update(test_dict) hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) # score_df.to_csv(os.path.join(savedir, "score_best_df.csv")) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) model.waiting = 0 model.val_score_best = score_dict["val_score"] print("Saved Best: %s" % savedir) # Report & Save score_df = pd.DataFrame(score_list) # score_df.to_csv(os.path.join(savedir, "score_df.csv")) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) if model.waiting > 100: break print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, datadir_base, reset=False): # bookkeeping stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ================== # load train and acrtive set train_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="train", datadir_base=datadir_base, exp_dict=exp_dict) active_set = ActiveLearningDataset(train_set, random_state=42) # val set val_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="val", datadir_base=datadir_base, exp_dict=exp_dict) val_loader = DataLoader(val_set, batch_size=exp_dict["batch_size"]) # Model # ================== model = models.get_model(model_name=exp_dict['model']['name'], exp_dict=exp_dict).cuda() model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) active_set.load_state_dict( hu.load_pkl(os.path.join(savedir, "active_set.pkl"))) score_list = hu.load_pkl(score_list_path) inner_s_epoch = score_list[-1]['inner_epoch'] + 1 s_cycle = score_list[-1]['cycle'] else: # restart experiment score_list = [] inner_s_epoch = 0 s_cycle = 0 # Train & Val # ================== print("Starting experiment at cycle %d epoch %d" % (s_cycle, inner_s_epoch)) for c in range(s_cycle, exp_dict['max_cycle']): # Set seed np.random.seed(c) torch.manual_seed(c) torch.cuda.manual_seed_all(c) if inner_s_epoch == 0: active_set.label_next_batch(model) hu.save_pkl(os.path.join(savedir, "active_set.pkl"), active_set.state_dict()) train_loader = DataLoader(active_set, sampler=samplers.get_sampler( exp_dict['sampler']['train'], active_set), batch_size=exp_dict["batch_size"]) # Visualize the model model.vis_on_loader(vis_loader, savedir=os.path.join(savedir, "images")) for e in range(inner_s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} if e == 0: score_dict.update(model.val_on_loader(val_loader)) # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model score_dict["epoch"] = len(score_list) score_dict["inner_epoch"] = e score_dict["cycle"] = c score_dict['n_ratio'] = active_set.n_labelled_ratio score_dict["n_train"] = len(train_loader.dataset) score_dict["n_pool"] = len(train_loader.dataset.pool) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) inner_s_epoch = 0
def trainval(exp_dict, savedir_base, data_root, reset=False, tensorboard=True): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) np.random.seed(exp_dict["seed"]) torch.manual_seed(exp_dict["seed"]) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) writer = tensorboardX.SummaryWriter(savedir) \ if tensorboard == 1 else None # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) pprint.pprint(exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ----------- train_dataset, val_dataset = get_dataset(['train', 'val'], data_root, exp_dict) # val_dataset = get_dataset('val', exp_dict) # train and val loader if exp_dict["episodic"] == False: train_loader = DataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) val_loader = DataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) else: # to support episodes TODO: move inside each model from datasets.episodic_dataset import EpisodicDataLoader train_loader = EpisodicDataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) val_loader = EpisodicDataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict, labelset=train_dataset.raw_labelset, writer=writer) print("Model with:", sum(p.numel() for p in model.parameters() if p.requires_grad), "parameters") # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(e, train_loader)) # Validate the model score_dict.update(model.val_on_loader(e, val_loader)) score_dict["epoch"] = e if tensorboard: for key, value in score_dict.items(): writer.add_scalar(key, value, e) writer.flush() # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # if model.is_end(): # print("Early stopping") # break print('experiment completed') # Cleanup if tensorboard == 1: writer.close()
def trainval(exp_dict, savedir_base, reset=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print(exp_dict) print("Experiment saved in %s" % savedir) # Set Seed # ------- seed = exp_dict.get('seed') np.random.seed(seed) torch.manual_seed(seed) # Dataset # ----------- train_dataset = get_dataset('train', exp_dict['dataset']) val_dataset = get_dataset('test', exp_dict['dataset']) # train and val loader train_loader = DataLoader( train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x if exp_dict['batch_size'] == 1 else default_collate, # to handle episodes num_workers=args.num_workers) val_loader = DataLoader( val_dataset, batch_size=exp_dict['batch_size'], collate_fn=lambda x: x if exp_dict['batch_size'] == 1 else default_collate, shuffle=True, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict) # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model savepath = os.path.join(savedir_base, exp_dict['dataset']['name']) score_dict.update(model.val_on_loader(val_loader, savedir=savepath)) model.on_train_end(savedir=savedir, epoch=e) score_dict["epoch"] = e # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) print('experiment completed')
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ================== # train set train_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="train", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val set val_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) val_sampler = torch.utils.data.SequentialSampler(val_set) val_loader = DataLoader(val_set, sampler=val_sampler, batch_size=1, num_workers=num_workers) # Model # ================== model = models.get_model(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() # model.opt = optimizers.get_optim(exp_dict['opt'], model) model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) train_sampler = torch.utils.data.RandomSampler(train_set, replacement=True, num_samples=2 * len(val_set)) train_loader = DataLoader(train_set, sampler=train_sampler, batch_size=exp_dict["batch_size"], drop_last=True, num_workers=num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate and Visualize the model val_dict = model.val_on_loader(val_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) score_dict.update(val_dict) # model.vis_on_loader( # vis_loader, savedir=os.path.join(savedir, "images")) # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = len(score_list) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Save Best Checkpoint if e == 0 or (score_dict.get("val_score", 0) > score_df["val_score"][:-1].fillna(0).max()): hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, data_root, reset=False, wandb='None', wandb_key='None'): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print(exp_dict) print("Experiment saved in %s" % savedir) model_name = exp_dict['model'] + \ "_lr_" + str(exp_dict['lr']) +\ "_hs_" + str(exp_dict['backbone']['hidden_size']) +\ "_pa_" + str(exp_dict['patience']) if exp_dict['model'] == 'MAML': model_name += "_ilr_" + str(exp_dict['inner_lr']) +\ "_nii_" + str(exp_dict['n_inner_iter']) #TODO add seed if wandb is not 'None': # https://docs.wandb.com/quickstart import wandb as logger if wandb_key is not 'None': logger.login(key=wandb_key) logger.init(project=wandb, group=model_name) logger.config.update(exp_dict) # Dataset # ----------- train_dataset = get_dataset('train', data_root, exp_dict) val_dataset = get_dataset('val', data_root, exp_dict) test_dataset = get_dataset('test', data_root, exp_dict) if 'ood' in exp_dict['dataset']['task']: ood_dataset = get_dataset('ood', data_root, exp_dict) ood = True else: ood = False # train and val loader if exp_dict["episodic"] == False: train_loader = DataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) val_loader = DataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) test_loader = DataLoader(test_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) else: # to support episodes TODO: move inside each model from datasets.episodic_dataset import EpisodicDataLoader train_loader = EpisodicDataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) val_loader = EpisodicDataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) test_loader = EpisodicDataLoader(test_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) if ood: ood_loader = EpisodicDataLoader(ood_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict) # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 patience_counter = 0 # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate and Test the model score_dict.update( model.val_on_loader(val_loader, mode='val', savedir=os.path.join( savedir_base, exp_dict['dataset']['name']))) score_dict.update(model.val_on_loader(test_loader, mode='test')) if ood: score_dict.update(model.val_on_loader(ood_loader, mode='ood')) score_dict["epoch"] = e # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Test error at best validation: if score_dict["val_accuracy"] > model.best_val: score_dict["test_accuracy_at_best_val"] = score_dict[ "test_accuracy"] score_dict["ood_accuracy_at_best_val"] = score_dict["ood_accuracy"] model.best_val = score_dict["val_accuracy"] patience_counter = 0 # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) if wandb is not 'None': for key, values in score_dict.items(): logger.log({key: values}) patience_counter += 1 # Patience: if patience_counter > exp_dict['patience'] * 3: print('training done, out of patience') break print('experiment completed')
def train(exp_dict, savedir_base, reset, compute_fid=False): # Book keeping pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: ut.rmtree(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) print('Experiment saved in %s' % savedir) device = \ torch.device('cuda:' + exp_dict['gpu'] if torch.cuda.is_available() else 'cpu') # 1. Load dataset and loader train_set, test_set, num_channels, num_train_classes, num_test_classes = \ datasets.get_dataset(exp_dict['dataset'], dataset_path=savedir_base, image_size=exp_dict['image_size']) train_loader, test_loader = \ dataloaders.get_dataloader(exp_dict['dataloader'], train_set, test_set, exp_dict) # 2. Fetch model to train model = models.get_model(exp_dict['model'], num_train_classes, num_test_classes, num_channels, device, exp_dict) # 3. Resume experiment or start from scratch score_list_path = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_path): # Resume experiment if it exists model_path = os.path.join(savedir, 'model_state_dict.pth') model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) meta_dict_path = os.path.join(savedir, 'meta_dict.pkl') meta_dict = hu.load_pkl(meta_dict_path) print('Resuming experiment at episode %d epoch %d' % (meta_dict['episode'], meta_dict['epoch'])) else: # Start experiment from scratch meta_dict = {'episode': 1, 'epoch': 1} score_list = [] # Remove TensorBoard logs from previous runs ut.rmtree(os.path.join(savedir, 'tensorboard_logs')) print('Starting experiment at episode %d epoch %d' % (meta_dict['episode'], meta_dict['epoch'])) # 4. Train and eval loop s_epoch = meta_dict['epoch'] for e in range(s_epoch, exp_dict['num_epochs'] + 1): # 0. Initialize dicts score_dict = {'epoch': e} meta_dict['epoch'] = e # 1. Train on loader train_dict = model.train_on_loader(train_loader) # 1b. Compute FID if compute_fid == 1: if e % 20 == 0 or e == 1 or e == exp_dict['num_epochs']: print('Starting FID computation...') train_dict['fid'] = fid(model, train_loader.dataset, train_loader.sampler, save_dir) score_dict.update(train_dict) # 2. Eval on loader eval_dict = model.val_on_loader(test_loader, savedir, e) score_dict.update(eval_dict) # 3. Report and save model state, optimizer state, and scores score_list += [score_dict] score_df = pd.DataFrame(score_list) print('\n', score_df.tail(), '\n') if e % 10 == 0: hu.torch_save(os.path.join(savedir, 'model_state_dict.pth'), model.get_state_dict()) hu.save_pkl(os.path.join(savedir, 'score_list.pkl'), score_list) hu.save_pkl(os.path.join(savedir, 'meta_dict.pkl'), meta_dict)
def trainval(exp_dict, savedir_base, data_root, reset=False, test_only=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) np.random.seed(exp_dict["seed"]) torch.manual_seed(exp_dict["seed"]) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) pprint.pprint(exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ----------- # train and val loader if exp_dict["episodic"] == False: if (int(test_only) == 0): train_dataset, val_dataset, test_dataset = get_dataset( ['train', 'val', 'test'], data_root, exp_dict) train_loader = DataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) val_loader = DataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) test_loader = DataLoader(test_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) if hasattr(train_dataset, "mask"): # assert((train_dataset.mask == val_dataset.mask).all()) # assert((train_dataset.mask == test_dataset.mask).all()) np.save(os.path.join(savedir, "mask.npy"), train_dataset.mask) else: test_dataset, = get_dataset(['test'], exp_dict) test_loader = DataLoader(test_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) else: # to support episodes TODO: move inside each model from datasets.episodic_dataset import EpisodicDataLoader train_loader = EpisodicDataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) val_loader = EpisodicDataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict) print("Parameters: ", sum([torch.numel(v) for v in model.parameters()])) # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment print("Resuming from", model_path) model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 if int(test_only) == 0: # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model score_dict.update( model.val_on_loader(val_loader, savedir=os.path.join( savedir_base, exp_dict['dataset']['name']))) score_dict["epoch"] = e # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) if model.is_end(): print("Early stopping") break print('experiment completed') print("Testing...") score_dict = model.test_on_loader(train_loader, tag="train") score_dict.update(model.test_on_loader(val_loader, tag="val")) score_dict.update(model.test_on_loader(test_loader, tag="test")) # Report & Save score_list_path = os.path.join(savedir, "score_list_test.pkl") hu.save_pkl(score_list_path, score_dict) else: print("Testing...") score_dict = model.test_on_loader(test_loader, "test") # Report & Save score_list_path = os.path.join(savedir, "score_list_test.pkl") hu.save_pkl(score_list_path, score_dict)
def trainval(exp_dict, savedir, args): """ exp_dict: dictionary defining the hyperparameters of the experiment savedir: the directory where the experiment will be saved args: arguments passed through the command line """ # Dataset # ================== # train set train_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="train", datadir=args.datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val set val_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=args.datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) val_sampler = torch.utils.data.SequentialSampler(val_set) val_loader = DataLoader(val_set, sampler=val_sampler, batch_size=1, num_workers=args.num_workers) # Model # ================== model = models.CountingModel(exp_dict=exp_dict, train_set=train_set).cuda() # model.opt = optimizers.get_optim(exp_dict['opt'], model) model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) train_sampler = torch.utils.data.RandomSampler(train_set, replacement=True, num_samples=2 * len(val_set)) train_loader = DataLoader(train_set, sampler=train_sampler, batch_size=exp_dict["batch_size"], drop_last=True, num_workers=args.num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate and Visualize the model val_dict = model.val_on_loader(val_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) score_dict.update(val_dict) # model.vis_on_loader( # vis_loader, savedir=os.path.join(savedir, "images")) # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = len(score_list) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Save Best Checkpoint if e == 0 or (score_dict.get("val_score", 0) > score_df["val_score"][:-1].fillna(0).max()): hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== savedir = os.path.join(savedir_base, hu.hash_dict(exp_dict)) os.makedirs(savedir, exist_ok=True) if reset: hc.delete_and_backup_experiment(savedir) print("Experiment saved in %s" % savedir) # Dataset # ================== # train set data_transform = A.Compose( [ A.Flip(p=0.3), A.IAAAffine(p=0.3), A.Rotate(p=0.3), A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=15, val_shift_limit=10, p=0.3), A.GaussianBlur(3, p=0.3), A.GaussNoise(30, p=0.3) ], keypoint_params=A.KeypointParams(format='xy'), additional_targets={ 'mask0': 'mask', 'mask1': 'mask', 'mask2': 'mask', 'keypoints0': 'keypoints', 'keypoints1': 'keypoints', 'keypoints2': 'keypoints', 'keypoints3': 'keypoints', 'keypoints4': 'keypoints', 'keypoints5': 'keypoints' }) # random.seed(20201009) random_seed = random.randint(0, 20201009) train_set = HEDataset_Fast(data_dir=datadir, n_classes=exp_dict["n_classes"], transform=data_transform, option="Train", random_seed=random_seed, obj_option=exp_dict["obj"], patch_size=exp_dict["patch_size"], bkg_option=exp_dict["bkg"]) test_transform = A.Compose([A.Resize(1024, 1024)], keypoint_params=A.KeypointParams(format='xy'), additional_targets={ 'mask0': 'mask', 'mask1': 'mask' }) # val set val_set = HEDataset(data_dir=datadir, transform=test_transform, option="Validation") val_loader = DataLoader(val_set, batch_size=1, num_workers=num_workers) # test set test_set = HEDataset(data_dir=datadir, transform=test_transform, option="Test") test_loader = DataLoader(test_set, batch_size=1, num_workers=num_workers) # Model # ================== # torch.manual_seed(20201009) model = models.get_model(exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) # train_sampler = torch.utils.data.RandomSampler( # train_set, replacement=True, num_samples=2*len(val_set)) train_loader = DataLoader(train_set, batch_size=exp_dict["batch_size"], shuffle=True, num_workers=num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate and Visualize the model val_dict = model.val_on_loader(val_loader, savedir_images=os.path.join( savedir, "images"), n_images=7) score_dict.update(val_dict) # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = len(score_list) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Save Best Checkpoint if e == 0 or (score_dict.get("val_score", 0) > score_df["val_score"][:-1].fillna(0).max()): hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) # if s_epoch==exp_dict['max_epoch']: # e = s_epoch model.load_state_dict( hu.torch_load(os.path.join(savedir, "model_best.pth"))) test_dict = model.test_on_loader(test_loader) hu.save_pkl(os.path.join(savedir, 'test_iou.pkl'), test_dict) print('Test IoU:{}'.format(test_dict["test_iou"])) print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, reset=False, num_workers=0, run_ssl=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) # load datasets # ========================== train_set = datasets.get_dataset( dataset_name=exp_dict["dataset_train"], data_root=exp_dict["dataset_train_root"], split="train", transform=exp_dict["transform_train"], classes=exp_dict["classes_train"], support_size=exp_dict["support_size_train"], query_size=exp_dict["query_size_train"], n_iters=exp_dict["train_iters"], unlabeled_size=exp_dict["unlabeled_size_train"]) val_set = datasets.get_dataset( dataset_name=exp_dict["dataset_val"], data_root=exp_dict["dataset_val_root"], split="val", transform=exp_dict["transform_val"], classes=exp_dict["classes_val"], support_size=exp_dict["support_size_val"], query_size=exp_dict["query_size_val"], n_iters=exp_dict["val_iters"], unlabeled_size=exp_dict["unlabeled_size_val"]) test_set = datasets.get_dataset( dataset_name=exp_dict["dataset_test"], data_root=exp_dict["dataset_test_root"], split="test", transform=exp_dict["transform_val"], classes=exp_dict["classes_test"], support_size=exp_dict["support_size_test"], query_size=exp_dict["query_size_test"], n_iters=exp_dict["test_iters"], unlabeled_size=exp_dict["unlabeled_size_test"]) # get dataloaders # ========================== train_loader = torch.utils.data.DataLoader( train_set, batch_size=exp_dict["batch_size"], shuffle=True, num_workers=num_workers, collate_fn=ut.get_collate(exp_dict["collate_fn"]), drop_last=True) val_loader = torch.utils.data.DataLoader(val_set, batch_size=1, shuffle=False, num_workers=num_workers, collate_fn=lambda x: x, drop_last=True) test_loader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False, num_workers=num_workers, collate_fn=lambda x: x, drop_last=True) # create model and trainer # ========================== # Create model, opt, wrapper backbone = backbones.get_backbone( backbone_name=exp_dict['model']["backbone"], exp_dict=exp_dict) model = models.get_model(model_name=exp_dict["model"]['name'], backbone=backbone, n_classes=exp_dict["n_classes"], exp_dict=exp_dict) if run_ssl: # runs the SSL experiments score_list_path = os.path.join(savedir, 'score_list.pkl') if not os.path.exists(score_list_path): test_dict = model.test_on_loader(test_loader, max_iter=None) hu.save_pkl(score_list_path, [test_dict]) return # Checkpoint # ----------- checkpoint_path = os.path.join(savedir, 'checkpoint.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(checkpoint_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Run training and validation for epoch in range(s_epoch, exp_dict["max_epoch"]): score_dict = {"epoch": epoch} score_dict.update(model.get_lr()) # train score_dict.update(model.train_on_loader(train_loader)) # validate score_dict.update(model.val_on_loader(val_loader)) score_dict.update(model.test_on_loader(test_loader)) # Add score_dict to score_list score_list += [score_dict] # Report score_df = pd.DataFrame(score_list) print(score_df.tail()) # Save checkpoint hu.save_pkl(score_list_path, score_list) hu.torch_save(checkpoint_path, model.get_state_dict()) print("Saved: %s" % savedir) if "accuracy" in exp_dict["target_loss"]: is_best = score_dict[exp_dict["target_loss"]] >= score_df[ exp_dict["target_loss"]][:-1].max() else: is_best = score_dict[exp_dict["target_loss"]] <= score_df[ exp_dict["target_loss"]][:-1].min() # Save best checkpoint if is_best: hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "checkpoint_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) # Check for end of training conditions if model.is_end_of_training(): break
test_loader = DataLoader(test_set, batch_size=1, collate_fn=ut.collate_fn, num_workers=0) pprint.pprint(exp_dict) # Model # ================== model = models.get_model(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() model_path = os.path.join(savedir_base, hash_id, 'model_best.pth') # load best model model.load_state_dict(hu.torch_load(model_path)) # loop over the val_loader and saves image # get counts habitats = [] for i, batch in enumerate(test_loader): habitat = batch['meta'][0]['habitat'] habitats += [habitat] habitats = np.array(habitats) val_dict = {} val_dict_lst = [] for h in np.unique(habitats): val_meter = metrics.SegMeter(split=test_loader.dataset.split) for i, batch in enumerate(tqdm.tqdm(test_loader)): habitat = batch['meta'][0]['habitat']
def trainval(exp_dict, savedir_base, reset): # ================== # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = savedir_base + "/%s/" % exp_id if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(savedir + "exp_dict.json", exp_dict) print("Experiment saved in %s" % savedir) # ================== # Dataset # ================== transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), torchvision.transforms.Normalize((0.5, ), (0.5, )) ]) # train set train_set = torchvision.datasets.MNIST(savedir_base, train=True, download=True, transform=transform) train_loader = DataLoader(train_set, shuffle=True, batch_size=exp_dict["batch_size"]) # val set val_set = torchvision.datasets.MNIST(savedir_base, train=False, download=True, transform=transform) val_loader = DataLoader(val_set, shuffle=False, batch_size=exp_dict["batch_size"]) # ================== # Model # ================== model = MLP(n_classes=10).cuda() model.opt = torch.optim.Adam(model.parameters(), lr=exp_dict["lr"]) model_path = savedir + "/model.pth" score_list_path = savedir + "/score_list.pkl" if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = len(score_list) else: # restart experiment score_list = [] s_epoch = 0 # ================== # Train & Val # ================== print("Starting experiment at epoch %d" % s_epoch) for e in range(s_epoch, 100): score_dict = {} # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model score_dict.update(model.val_on_loader(val_loader)) score_dict["epoch"] = e # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()[["epoch", "train_loss", "val_acc"]], "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir_base)