def test_get_plot(self): # save a score_list savedir_base = '.tmp' exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) fig, axis = hr.get_plot(exp_list, savedir_base=savedir_base, filterby_list=[({'model':{'name':'mlp'}}, {'style':{'color':'red'}})], x_metric='epoch', y_metric='acc') # fig, axis = hr.get_plot(exp_list, # savedir_base=savedir_base, # x_metric='epoch', # y_metric='acc', # mode='pretty_plot') fig, axis = hr.get_plot(exp_list, savedir_base=savedir_base, x_metric='epoch', y_metric='acc', mode='bar') fig.savefig(os.path.join('.tmp', 'test.png')) shutil.rmtree('.tmp')
def test_get_score_lists(): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert score_lists[0][0]["acc"] == 0.5 assert score_lists[0][1]["acc"] == 0.9 shutil.rmtree(savedir_base)
def test_zipdir(self): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hr.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert score_lists[0][0]["acc"] == 0.5 assert score_lists[0][1]["acc"] == 0.9 from haven import haven_dropbox as hd hd.zipdir([hu.hash_dict(exp_dict) for exp_dict in exp_list], savedir_base, src_fname=".tmp/results.zip") shutil.rmtree(savedir_base)
def test_get_score_df(self): # save a score_list savedir_base = '.tmp' exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} exp_dict2 = {'model':{'name':'mlp2', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 0, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict2), 'exp_dict.json'), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base) assert(np.array(score_df['dataset'])[0].strip("'") == 'mnist') shutil.rmtree('.tmp')
def test_get_score_lists(self): # save a score_list savedir_base = '.tmp' exp_dict = { 'model': { 'name': 'mlp', 'n_layers': 30 }, 'dataset': 'mnist', 'batch_size': 1 } score_list = [{'epoch': 0, 'acc': 0.5}, {'epoch': 0, 'acc': 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hr.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert (score_lists[0][0]['acc'] == 0.5) assert (score_lists[0][1]['acc'] == 0.9) shutil.rmtree(savedir_base)
def compute_fstar(self, model_func, loss_function, fname): if os.path.exists(fname): fstar_list = hu.load_pkl(fname) else: fstar_list = np.ones(len(self)) * -1 for i in range(len(self)): batch = self[i] images, labels = batch['images'][None].cuda(), batch['labels'][None].cuda() model = model_func() opt = torch.optim.Adam(model.parameters()) for j in range(10000): opt.zero_grad() closure = lambda : loss_function(model, images, labels, backwards=True) loss = opt.step(closure).item() grad_current = sps.get_grad_list(model.parameters()) grad_norm = sps.compute_grad_norm(grad_current) if np.isnan(loss): print('nan') # print(i, loss) if grad_norm < 1e-6: break if j > 0 and abs(loss_old - loss) < 1e-6: break loss_old = loss print("%d/%d - converged:%d - %.6f"% (i, len(self), j, loss)) fstar_list[i] = loss hu.save_pkl(fname, fstar_list) self.fstar_list = fstar_list
def __init__( self, split, datadir, exp_dict, ): self.exp_dict = exp_dict self.datadir = datadir self.split = split self.n_classes = 5 self.img_path = os.path.join(datadir, 'OpenSourceDCMs') self.lung_path = os.path.join(datadir, 'LungMasks') self.tgt_path = os.path.join(datadir, 'InfectionMasks') self.img_tgt_dict = [] for tgt_name in os.listdir(self.tgt_path): lung_name = os.path.join(self.lung_path, tgt_name) scan_id, slice_id = tgt_name.split('_') slice_id = str(int(slice_id.replace('z', '').replace('.png', ''))).zfill(4) img_name = [ f for f in os.listdir( os.path.join(self.img_path, 'DCM' + scan_id)) if 's%s' % slice_id in f ][0] img_name = os.path.join('DCM' + scan_id, img_name) self.img_tgt_dict += [{ 'img': img_name, 'tgt': tgt_name, 'lung': lung_name }] # get label_meta fname = os.path.join(datadir, 'tmp', 'labels_array.pkl') if not os.path.exists(fname): labels_array = np.zeros((len(self.img_tgt_dict), 3)) for i, idict in enumerate(tqdm.tqdm(self.img_tgt_dict)): img_name, tgt_name = idict['img'], idict['tgt'] mask = np.array( Image.open(os.path.join(self.tgt_path, tgt_name))) uniques = np.unique(mask) if 0 in uniques: labels_array[i, 0] = 1 if 127 in uniques: labels_array[i, 1] = 1 if 255 in uniques: labels_array[i, 2] = 1 hu.save_pkl(fname, labels_array) labels_array = hu.load_pkl(fname) # self.np.where(labels_array[:,1:].max(axis=1)) ind_list = np.where(labels_array[:, 1:].max(axis=1))[0] self.img_tgt_dict = np.array(self.img_tgt_dict)[ind_list] if split == 'train': self.img_tgt_dict = self.img_tgt_dict[:300] elif split == 'val': self.img_tgt_dict = self.img_tgt_dict[300:]
def test_get_plot(): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 1, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) fig, axis = hr.get_plot( exp_list, savedir_base=savedir_base, filterby_list=[({ "model": { "name": "mlp" } }, { "style": { "color": "red" } })], x_metric="epoch", y_metric="acc", ) # fig, axis = hr.get_plot(exp_list, # savedir_base=savedir_base, # x_metric='epoch', # y_metric='acc', # mode='pretty_plot') fig, axis = hr.get_plot(exp_list, savedir_base=savedir_base, x_metric="epoch", y_metric="acc", mode="bar") fig.savefig(os.path.join(".tmp", "test.png")) shutil.rmtree(".tmp")
def test_get_best_exp_dict(self): savedir_base = '.tmp' exp_dict_1 = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_1), 'score_list.pkl'), score_list) exp_dict_2 = {'model':{'name':'mlp', 'n_layers':35}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.6}, {'epoch': 1, 'acc':1.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_2), 'score_list.pkl'), score_list) best_exp_list = hu.filter_exp_list([exp_dict_1, exp_dict_2], savedir_base=savedir_base, filterby_list=[({'model.name':'mlp'}, {'best':{'avg_across':'run', 'metric':'acc', 'metric_agg':'max'}} )]) assert len(best_exp_list) == 1 assert best_exp_list[0]['model']['n_layers'] == 35 best_exp_list = hu.filter_exp_list([exp_dict_1, exp_dict_2], savedir_base=savedir_base, filterby_list=[({'model.name':'mlp'}, {'best':{'avg_across':'run', 'metric':'acc', 'metric_agg':'min'}} )]) assert best_exp_list[0]['model']['n_layers'] == 30 # exp 2 exp_dict_2 = {'model':{'name':'mlp2', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1, 'run':0} score_list = [{'epoch': 0, 'acc':1.5}, {'epoch': 1, 'acc':1.8}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_2), 'score_list.pkl'), score_list) # exp 3 exp_dict_3 = {'model':{'name':'mlp2', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1, 'run':1} score_list = [{'epoch': 0, 'acc':1.5}, {'epoch': 1, 'acc':1.3}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_3), 'score_list.pkl'), score_list) exp_list = [exp_dict_1, exp_dict_2, exp_dict_3] best_exp_dict = hu.get_best_exp_dict(exp_list, savedir_base=savedir_base, metric='acc', avg_across='run', metric_agg='max', ) assert(best_exp_dict['model']['name'] == 'mlp2')
def trainval(exp_dict, savedir, args): """ exp_dict: dictionary defining the hyperparameters of the experiment savedir: the directory where the experiment will be saved args: arguments passed through the command line """ # -- Datasets train_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], train_flag=True, datadir=args.datadir) val_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], train_flag=False, datadir=args.datadir) # -- Model model = models.Model(exp_dict, device=torch.device('cuda')) # -- Train & Val Loop score_list = [] for e in range(0, 50): # Compute metrics score_dict = {"epoch": e} score_dict["train_loss"] = model.val_on_dataset( val_set, metric_name='softmax_loss') score_dict["val_acc"] = model.val_on_dataset(val_set, metric_name='softmax_acc') score_list += [score_dict] # Train model for one epoch model.train_on_dataset(train_set) # Visualize images = model.vis_on_dataset(val_set, fname=os.path.join( savedir, 'images', 'results.png')) # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.save_pkl(os.path.join(savedir, 'score_list.pkl'), score_list) hu.torch_save(os.path.join(savedir, 'model.pth'), model.state_dict()) print("Checkpoint Saved: %s" % savedir) print('Experiment completed et epoch %d' % e)
def save_example_results(savedir_base="results"): import os import pandas import requests import io import matplotlib.pyplot as plt from .. import haven_results as hr from .. import haven_utils as hu from PIL import Image # create hyperparameters exp_list = [{ "dataset": "mnist", "model": "mlp", "lr": lr } for lr in [1e-1, 1e-2, 1e-3]] for i, exp_dict in enumerate(exp_list): # get hash for experiment exp_id = hu.hash_dict(exp_dict) # add scores for loss, and accuracy score_list = [] for e in range(1, 10): score_list += [{ "epoch": e, "loss": 1 - e * exp_dict["lr"] * 0.9, "acc": e * exp_dict["lr"] * 0.1 }] # save scores and images hu.save_json(os.path.join(savedir_base, exp_id, "exp_dict.json"), exp_dict) hu.save_pkl(os.path.join(savedir_base, exp_id, "score_list.pkl"), score_list) url = "https://raw.githubusercontent.com/haven-ai/haven-ai/master/haven/haven_examples/data/%d.png" % ( i + 1) response = requests.get(url).content img = plt.imread(io.BytesIO(response), format="JPG") hu.save_image(os.path.join(savedir_base, exp_id, "images/1.png"), img[:, :, :3])
def test_get_best_exp_dict(self): savedir_base = '.tmp' exp_dict_1 = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_1), 'score_list.pkl'), score_list) exp_dict_2 = {'model':{'name':'mlp2', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':1.2}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict_2), 'score_list.pkl'), score_list) exp_list = [exp_dict_1, exp_dict_2] best_exp_dict = hr.get_best_exp_dict(exp_list, savedir_base=savedir_base, metric='acc', min_or_max='max') assert(best_exp_dict['model']['name'] == 'mlp2')
def test_get_score_df(): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } exp_dict2 = { "model": { "name": "mlp2", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict2), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base) assert np.array(score_df["dataset"])[0].strip("'") == "mnist" shutil.rmtree(".tmp")
def save_example_results(savedir_base='results'): import os, pandas import requests, io import matplotlib.pyplot as plt from .. import haven_results as hr from .. import haven_utils as hu from PIL import Image # create hyperparameters exp_list = [{ 'dataset': 'mnist', 'model': 'mlp', 'lr': lr } for lr in [1e-1, 1e-2, 1e-3]] for i, exp_dict in enumerate(exp_list): # get hash for experiment exp_id = hu.hash_dict(exp_dict) # add scores for loss, and accuracy score_list = [] for e in range(1, 10): score_list += [{ 'epoch': e, 'loss': 1 - e * exp_dict['lr'] * 0.9, 'acc': e * exp_dict['lr'] * 0.1 }] # save scores and images hu.save_json(os.path.join(savedir_base, exp_id, 'exp_dict.json'), exp_dict) hu.save_pkl(os.path.join(savedir_base, exp_id, 'score_list.pkl'), score_list) url = 'https://raw.githubusercontent.com/haven-ai/haven-ai/master/haven/haven_examples/data/%d.png' % ( i + 1) response = requests.get(url).content img = plt.imread(io.BytesIO(response), format='JPG') hu.save_image(os.path.join(savedir_base, exp_id, 'images/1.png'), img[:, :, :3])
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) if not os.path.join(savedir, "exp_dict.json"): hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # BCD train # ================== # Ignore the following combinations if not ut.is_valid_exp(exp_dict): return score_list_fname = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_fname): score_list = hu.load_pkl(score_list_fname) else: score_list = train(dataset_name=exp_dict['dataset']['name'], loss_name=exp_dict['dataset']['loss'], block_size=exp_dict['block_size'], partition_rule=exp_dict['partition'], selection_rule=exp_dict['selection'], update_rule=exp_dict['update'], n_iters=exp_dict['max_iters'], L1=exp_dict.get('l1', 0), L2=0, datasets_path=datadir) hu.save_pkl(score_list_fname, score_list) print('Experiment completed.') return score_list
def trainval(exp_dict, savedir, args): """ exp_dict: dictionary defining the hyperparameters of the experiment savedir: the directory where the experiment will be saved args: arguments passed through the command line """ # set seed # ================== seed = 42 np.random.seed(seed) torch.manual_seed(seed) if args.use_cuda: device = 'cuda' torch.cuda.manual_seed_all(seed) assert torch.cuda.is_available( ), 'cuda is not, available please run with "-c 0"' else: device = 'cpu' print('Running on device: %s' % device) # Dataset # Load val set and train set val_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="val", transform=exp_dict.get("transform"), datadir=args.datadir) train_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="train", transform=exp_dict.get("transform"), datadir=args.datadir) # Load train loader, val loader, and vis loader train_loader = DataLoader(train_set, sampler=RandomSampler( train_set, replacement=True, num_samples=max(min(500, len(train_set)), len(val_set))), batch_size=exp_dict["batch_size"]) val_loader = DataLoader(val_set, shuffle=False, batch_size=exp_dict["batch_size"]) vis_loader = DataLoader(val_set, sampler=ut.SubsetSampler(train_set, indices=[0, 1, 2]), batch_size=1) # Create model, opt, wrapper model_original = models.get_model(exp_dict["model"], exp_dict=exp_dict).cuda() opt = torch.optim.Adam(model_original.parameters(), lr=1e-5, weight_decay=0.0005) model = wrappers.get_wrapper(exp_dict["wrapper"], model=model_original, opt=opt).cuda() score_list = [] # Checkpointing # ============= score_list_path = os.path.join(savedir, "score_list.pkl") model_path = os.path.join(savedir, "model_state_dict.pth") opt_path = os.path.join(savedir, "opt_state_dict.pth") if os.path.exists(score_list_path): # resume experiment score_list = hu.load_pkl(score_list_path) model.load_state_dict(torch.load(model_path)) opt.load_state_dict(torch.load(opt_path)) s_epoch = score_list[-1]["epoch"] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Run training and validation for epoch in range(s_epoch, exp_dict["max_epoch"]): score_dict = {"epoch": epoch} # visualize model.vis_on_loader(vis_loader, savedir=os.path.join(savedir, "images")) # validate score_dict.update(model.val_on_loader(val_loader)) # train score_dict.update(model.train_on_loader(train_loader)) # Add score_dict to score_list score_list += [score_dict] # Report and save print(pd.DataFrame(score_list).tail()) hu.save_pkl(score_list_path, score_list) hu.torch_save(model_path, model.state_dict()) hu.torch_save(opt_path, opt.state_dict()) print("Saved in %s" % savedir)
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== savedir = os.path.join(savedir_base, hu.hash_dict(exp_dict)) os.makedirs(savedir, exist_ok=True) if reset: hc.delete_and_backup_experiment(savedir) print("Experiment saved in %s" % savedir) # Dataset # ================== # train set data_transform = A.Compose( [ A.Flip(p=0.3), A.IAAAffine(p=0.3), A.Rotate(p=0.3), A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=15, val_shift_limit=10, p=0.3), A.GaussianBlur(3, p=0.3), A.GaussNoise(30, p=0.3) ], keypoint_params=A.KeypointParams(format='xy'), additional_targets={ 'mask0': 'mask', 'mask1': 'mask', 'mask2': 'mask', 'keypoints0': 'keypoints', 'keypoints1': 'keypoints', 'keypoints2': 'keypoints', 'keypoints3': 'keypoints', 'keypoints4': 'keypoints', 'keypoints5': 'keypoints' }) # random.seed(20201009) random_seed = random.randint(0, 20201009) train_set = HEDataset_Fast(data_dir=datadir, n_classes=exp_dict["n_classes"], transform=data_transform, option="Train", random_seed=random_seed, obj_option=exp_dict["obj"], patch_size=exp_dict["patch_size"], bkg_option=exp_dict["bkg"]) test_transform = A.Compose([A.Resize(1024, 1024)], keypoint_params=A.KeypointParams(format='xy'), additional_targets={ 'mask0': 'mask', 'mask1': 'mask' }) # val set val_set = HEDataset(data_dir=datadir, transform=test_transform, option="Validation") val_loader = DataLoader(val_set, batch_size=1, num_workers=num_workers) # test set test_set = HEDataset(data_dir=datadir, transform=test_transform, option="Test") test_loader = DataLoader(test_set, batch_size=1, num_workers=num_workers) # Model # ================== # torch.manual_seed(20201009) model = models.get_model(exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) # train_sampler = torch.utils.data.RandomSampler( # train_set, replacement=True, num_samples=2*len(val_set)) train_loader = DataLoader(train_set, batch_size=exp_dict["batch_size"], shuffle=True, num_workers=num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate and Visualize the model val_dict = model.val_on_loader(val_loader, savedir_images=os.path.join( savedir, "images"), n_images=7) score_dict.update(val_dict) # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = len(score_list) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Save Best Checkpoint if e == 0 or (score_dict.get("val_score", 0) > score_df["val_score"][:-1].fillna(0).max()): hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) # if s_epoch==exp_dict['max_epoch']: # e = s_epoch model.load_state_dict( hu.torch_load(os.path.join(savedir, "model_best.pth"))) test_dict = model.test_on_loader(test_loader) hu.save_pkl(os.path.join(savedir, 'test_iou.pkl'), test_dict) print('Test IoU:{}'.format(test_dict["test_iou"])) print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, reset=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) # Dataset # ----------- # train loader train_loader = datasets.get_loader(dataset_name=exp_dict['dataset'], datadir=savedir_base, split='train') # val loader val_loader = datasets.get_loader(dataset_name=exp_dict['dataset'], datadir=savedir_base, split='val') # Model # ----------- model = models.get_model(model_name=exp_dict['model']) # Checkpoint # ----------- model_path = os.path.join(savedir, 'model.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print('Starting experiment at epoch %d' % (s_epoch)) for e in range(s_epoch, 10): score_dict = {} # Train the model train_dict = model.train_on_loader(train_loader) # Validate the model val_dict = model.val_on_loader(val_loader) # Get metrics score_dict['train_loss'] = train_dict['train_loss'] score_dict['val_acc'] = val_dict['val_acc'] score_dict['epoch'] = e # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print(score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print('Checkpoint Saved: %s' % savedir) print('experiment completed')
def trainval(exp_dict, savedir_base, datadir_base, reset=False, num_workers=0, pin_memory=False, ngpu=1, cuda_deterministic=False): # bookkeeping # ================== # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) if DEVICE.type == "cuda": if cuda_deterministic: cudnn.benchmark = False cudnn.deterministic = True else: cudnn.benchmark = True # Dataset # ================== trainset = get_dataset(exp_dict['dataset'], 'train', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['train'], transform_lvl=exp_dict['dataset']['transform_lvl'], colorjitter=exp_dict['dataset'].get('colorjitter') ) valset = get_dataset(exp_dict['dataset'], 'validation', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['train'], transform_lvl=0, val_transform=exp_dict['dataset']['val_transform']) testset = get_dataset(exp_dict['dataset'], 'test', exp_dict=exp_dict, datadir_base=datadir_base, n_samples=exp_dict['dataset_size']['test'], transform_lvl=0, val_transform=exp_dict['dataset']['val_transform']) print("Dataset defined.") # define dataloaders if exp_dict['dataset']['name'] == 'bach': testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) else: testloader = torch.utils.data.DataLoader(testset, batch_size=exp_dict['batch']['size'], shuffle=False, num_workers=num_workers, pin_memory=pin_memory) print("Testloader defined.") # Model # ================== model = get_model(exp_dict, trainset, device=DEVICE) print("Model loaded") model_path = os.path.join(savedir, 'model.pth') model_best_path = os.path.join(savedir, 'model_best.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') # checkpoint management if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = len(score_list) else: # restart experiment score_list = [] s_epoch = 0 # define and log random seed for reproducibility assert('fixedSeed' in exp_dict) seed = exp_dict['fixedSeed'] random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) np.random.seed(seed) print("Seed defined.") # Train & Val # ================== print("Starting experiment at epoch %d/%d" % (s_epoch, exp_dict['niter'])) for epoch in range(s_epoch, exp_dict['niter']): s_time = time.time() # Sample new train val trainloader, valloader = get_train_val_dataloader(exp_dict, trainset, valset, mixtrainval=exp_dict['mixTrainVal'], num_workers=num_workers, pin_memory=pin_memory) # Train & validate train_dict = model.train_on_loader(trainloader, valloader, epoch=epoch, exp_dict=exp_dict) # Test phase train_dict_2 = model.test_on_loader(trainloader) val_dict = model.test_on_loader(valloader) test_dict = model.test_on_loader(testloader) # Vis phase model.vis_on_loader('train', trainset, savedir_images=os.path.join( savedir, 'images'), epoch=epoch) score_dict = {} score_dict["epoch"] = epoch score_dict["test_acc"] = test_dict['acc'] score_dict["val_acc"] = val_dict['acc'] score_dict["train_acc"] = train_dict_2['acc'] score_dict["train_loss"] = train_dict['loss'] score_dict["time_taken"] = time.time() - s_time score_dict["netC_lr"] = train_dict['netC_lr'] if exp_dict['model']['netA'] is not None: if 'transformations_mean' in train_dict: for i in range(len(train_dict['transformations_mean'])): score_dict[str( i) + "_mean"] = train_dict['transformations_mean'][i].item() if 'transformations_std' in train_dict: for i in range(len(train_dict['transformations_std'])): score_dict[str( i) + "_std"] = train_dict['transformations_std'][i].item() # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # Update best score if epoch == 0 or (score_dict["test_acc"] >= score_df["test_acc"][:-1].max()): hu.save_pkl(os.path.join( savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) print('experiment completed')
def trainval(exp_dict, savedir_base, reset=False, num_workers=0, run_ssl=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) # load datasets # ========================== train_set = datasets.get_dataset( dataset_name=exp_dict["dataset_train"], data_root=exp_dict["dataset_train_root"], split="train", transform=exp_dict["transform_train"], classes=exp_dict["classes_train"], support_size=exp_dict["support_size_train"], query_size=exp_dict["query_size_train"], n_iters=exp_dict["train_iters"], unlabeled_size=exp_dict["unlabeled_size_train"]) val_set = datasets.get_dataset( dataset_name=exp_dict["dataset_val"], data_root=exp_dict["dataset_val_root"], split="val", transform=exp_dict["transform_val"], classes=exp_dict["classes_val"], support_size=exp_dict["support_size_val"], query_size=exp_dict["query_size_val"], n_iters=exp_dict["val_iters"], unlabeled_size=exp_dict["unlabeled_size_val"]) test_set = datasets.get_dataset( dataset_name=exp_dict["dataset_test"], data_root=exp_dict["dataset_test_root"], split="test", transform=exp_dict["transform_val"], classes=exp_dict["classes_test"], support_size=exp_dict["support_size_test"], query_size=exp_dict["query_size_test"], n_iters=exp_dict["test_iters"], unlabeled_size=exp_dict["unlabeled_size_test"]) # get dataloaders # ========================== train_loader = torch.utils.data.DataLoader( train_set, batch_size=exp_dict["batch_size"], shuffle=True, num_workers=num_workers, collate_fn=ut.get_collate(exp_dict["collate_fn"]), drop_last=True) val_loader = torch.utils.data.DataLoader(val_set, batch_size=1, shuffle=False, num_workers=num_workers, collate_fn=lambda x: x, drop_last=True) test_loader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False, num_workers=num_workers, collate_fn=lambda x: x, drop_last=True) # create model and trainer # ========================== # Create model, opt, wrapper backbone = backbones.get_backbone( backbone_name=exp_dict['model']["backbone"], exp_dict=exp_dict) model = models.get_model(model_name=exp_dict["model"]['name'], backbone=backbone, n_classes=exp_dict["n_classes"], exp_dict=exp_dict) if run_ssl: # runs the SSL experiments score_list_path = os.path.join(savedir, 'score_list.pkl') if not os.path.exists(score_list_path): test_dict = model.test_on_loader(test_loader, max_iter=None) hu.save_pkl(score_list_path, [test_dict]) return # Checkpoint # ----------- checkpoint_path = os.path.join(savedir, 'checkpoint.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(checkpoint_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Run training and validation for epoch in range(s_epoch, exp_dict["max_epoch"]): score_dict = {"epoch": epoch} score_dict.update(model.get_lr()) # train score_dict.update(model.train_on_loader(train_loader)) # validate score_dict.update(model.val_on_loader(val_loader)) score_dict.update(model.test_on_loader(test_loader)) # Add score_dict to score_list score_list += [score_dict] # Report score_df = pd.DataFrame(score_list) print(score_df.tail()) # Save checkpoint hu.save_pkl(score_list_path, score_list) hu.torch_save(checkpoint_path, model.get_state_dict()) print("Saved: %s" % savedir) if "accuracy" in exp_dict["target_loss"]: is_best = score_dict[exp_dict["target_loss"]] >= score_df[ exp_dict["target_loss"]][:-1].max() else: is_best = score_dict[exp_dict["target_loss"]] <= score_df[ exp_dict["target_loss"]][:-1].min() # Save best checkpoint if is_best: hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) hu.torch_save(os.path.join(savedir, "checkpoint_best.pth"), model.get_state_dict()) print("Saved Best: %s" % savedir) # Check for end of training conditions if model.is_end_of_training(): break
def test(exp_dict, savedir_base, datadir, num_workers=0, model_path=None, scan_id=None): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ================== # val set test_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) if str(scan_id) != 'None': test_set.active_data = test_set.get_scan(scan_id) test_sampler = torch.utils.data.SequentialSampler(test_set) test_loader = DataLoader(test_set, sampler=test_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) # Model # ================== # chk = torch.load('best_model.ckpt') model = models.get_model_for_onnx_export(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=test_set).cuda() epoch = -1 if str(model_path) != 'None': model_path = model_path model.load_state_dict(hu.torch_load(model_path)) else: try: exp_dict_train = copy.deepcopy(exp_dict) del exp_dict_train['test_mode'] savedir_train = os.path.join(savedir_base, hu.hash_dict(exp_dict_train)) model_path = os.path.join(savedir_train, "model_best.pth") score_list = hu.load_pkl( os.path.join(savedir_train, 'score_list_best.pkl')) epoch = score_list[-1]['epoch'] print('Loaded model at epoch %d with score %.3f' % epoch) model.load_state_dict(hu.torch_load(model_path)) except: pass s_time = time.time() savedir_images = os.path.join(savedir, 'images') # delete image folder if exists if os.path.exists(savedir_images): shutil.rmtree(savedir_images) os.makedirs(savedir_images, exist_ok=True) # for i in range(20): # score_dict = model.train_on_loader(test_loader) score_dict = model.val_on_loader(test_loader, savedir_images=savedir_images, n_images=30000, save_preds=True) score_dict['epoch'] = epoch score_dict["time"] = time.time() - s_time score_dict["saved_at"] = hu.time_to_montreal() # save test_score_list test_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(test_path): test_score_list = [ sd for sd in hu.load_pkl(test_path) if sd['epoch'] != epoch ] else: test_score_list = [] # append score_dict to last result test_score_list += [score_dict] hu.save_pkl(test_path, test_score_list) print('Final Score is ', str(score_dict["val_score"]) + "\n")
def trainval(exp_dict, savedir_base, datadir, reset=False, num_workers=0): # bookkeepting stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # set seed # ================== seed = 42 np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) # Dataset # ================== # train set train_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="train", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val set val_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="val", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # test set test_set = datasets.get_dataset(dataset_dict=exp_dict["dataset"], split="test", datadir=datadir, exp_dict=exp_dict, dataset_size=exp_dict['dataset_size']) # val_sampler = torch.utils.data.SequentialSampler(val_set) val_loader = DataLoader( val_set, # sampler=val_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) test_loader = DataLoader( test_set, # sampler=val_sampler, batch_size=1, collate_fn=ut.collate_fn, num_workers=num_workers) # Model # ================== model = models.get_model(model_dict=exp_dict['model'], exp_dict=exp_dict, train_set=train_set).cuda() # model.opt = optimizers.get_optim(exp_dict['opt'], model) model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.load_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ================== print("Starting experiment at epoch %d" % (s_epoch)) model.waiting = 0 model.val_score_best = -np.inf train_sampler = torch.utils.data.RandomSampler(train_set, replacement=True, num_samples=2 * len(test_set)) train_loader = DataLoader(train_set, sampler=train_sampler, collate_fn=ut.collate_fn, batch_size=exp_dict["batch_size"], drop_last=True, num_workers=num_workers) for e in range(s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} test_dict = model.val_on_loader(test_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) # Train the model train_dict = model.train_on_loader(train_loader) # Validate the model val_dict = model.val_on_loader(val_loader) score_dict["val_score"] = val_dict["val_score"] # Get new score_dict score_dict.update(train_dict) score_dict["epoch"] = e score_dict["waiting"] = model.waiting model.waiting += 1 # Add to score_list and save checkpoint score_list += [score_dict] # Save Best Checkpoint score_df = pd.DataFrame(score_list) if score_dict["val_score"] >= model.val_score_best: test_dict = model.val_on_loader(test_loader, savedir_images=os.path.join( savedir, "images"), n_images=3) score_dict.update(test_dict) hu.save_pkl(os.path.join(savedir, "score_list_best.pkl"), score_list) # score_df.to_csv(os.path.join(savedir, "score_best_df.csv")) hu.torch_save(os.path.join(savedir, "model_best.pth"), model.get_state_dict()) model.waiting = 0 model.val_score_best = score_dict["val_score"] print("Saved Best: %s" % savedir) # Report & Save score_df = pd.DataFrame(score_list) # score_df.to_csv(os.path.join(savedir, "score_df.csv")) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) if model.waiting > 100: break print('Experiment completed et epoch %d' % e)
def trainval(exp_dict, savedir_base, reset=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, 'exp_dict.json'), exp_dict) pprint.pprint(exp_dict) print('Experiment saved in %s' % savedir) # set seed # --------------- seed = 42 + exp_dict['runs'] np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) # Dataset # ----------- # train loader train_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], train_flag=True, datadir=savedir_base, exp_dict=exp_dict) train_loader = torch.utils.data.DataLoader( train_set, drop_last=True, shuffle=True, batch_size=exp_dict["batch_size"]) # val set val_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], train_flag=False, datadir=savedir_base, exp_dict=exp_dict) # Model # ----------- model = models.get_model(exp_dict["model"], train_set=train_set).cuda() # Choose loss and metric function loss_function = metrics.get_metric_function(exp_dict["loss_func"]) # Compute fstar # ------------- if exp_dict['opt'].get('fstar_flag'): ut.compute_fstar(train_set, loss_function, savedir_base, exp_dict) # Load Optimizer n_batches_per_epoch = len(train_set) / float(exp_dict["batch_size"]) opt = optimizers.get_optimizer(opt_dict=exp_dict["opt"], params=model.parameters(), n_batches_per_epoch=n_batches_per_epoch) # Checkpoint # ----------- model_path = os.path.join(savedir, 'model.pth') score_list_path = os.path.join(savedir, 'score_list.pkl') opt_path = os.path.join(savedir, 'opt_state_dict.pth') if os.path.exists(score_list_path): # resume experiment score_list = hu.load_pkl(score_list_path) model.load_state_dict(torch.load(model_path)) opt.load_state_dict(torch.load(opt_path)) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print('Starting experiment at epoch %d/%d' % (s_epoch, exp_dict['max_epoch'])) for e in range(s_epoch, exp_dict['max_epoch']): # Set seed seed = e + exp_dict['runs'] np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) score_dict = {} # Compute train loss over train set score_dict["train_loss"] = metrics.compute_metric_on_dataset( model, train_set, metric_name=exp_dict["loss_func"]) # Compute val acc over val set score_dict["val_acc"] = metrics.compute_metric_on_dataset( model, val_set, metric_name=exp_dict["acc_func"]) # Train over train loader model.train() print("%d - Training model with %s..." % (e, exp_dict["loss_func"])) # train and validate s_time = time.time() for batch in tqdm.tqdm(train_loader): images, labels = batch["images"].cuda(), batch["labels"].cuda() opt.zero_grad() # closure def closure(): return loss_function(model, images, labels, backwards=True) opt.step(closure) e_time = time.time() # Record metrics score_dict["epoch"] = e score_dict["step_size"] = opt.state["step_size"] score_dict["step_size_avg"] = opt.state["step_size_avg"] score_dict["n_forwards"] = opt.state["n_forwards"] score_dict["n_backwards"] = opt.state["n_backwards"] score_dict["grad_norm"] = opt.state["grad_norm"] score_dict["batch_size"] = train_loader.batch_size score_dict["train_epoch_time"] = e_time - s_time score_list += [score_dict] # Report and save print(pd.DataFrame(score_list).tail()) hu.save_pkl(score_list_path, score_list) hu.torch_save(model_path, model.state_dict()) hu.torch_save(opt_path, opt.state_dict()) print("Saved: %s" % savedir) print('Experiment completed')
def trainval(exp_dict, savedir_base, datadir_base, reset=False): # bookkeeping stuff # ================== pprint.pprint(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: hc.delete_and_backup_experiment(savedir) os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ================== # load train and acrtive set train_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="train", datadir_base=datadir_base, exp_dict=exp_dict) active_set = ActiveLearningDataset(train_set, random_state=42) # val set val_set = datasets.get_dataset(dataset_name=exp_dict["dataset"], split="val", datadir_base=datadir_base, exp_dict=exp_dict) val_loader = DataLoader(val_set, batch_size=exp_dict["batch_size"]) # Model # ================== model = models.get_model(model_name=exp_dict['model']['name'], exp_dict=exp_dict).cuda() model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) active_set.load_state_dict( hu.load_pkl(os.path.join(savedir, "active_set.pkl"))) score_list = hu.load_pkl(score_list_path) inner_s_epoch = score_list[-1]['inner_epoch'] + 1 s_cycle = score_list[-1]['cycle'] else: # restart experiment score_list = [] inner_s_epoch = 0 s_cycle = 0 # Train & Val # ================== print("Starting experiment at cycle %d epoch %d" % (s_cycle, inner_s_epoch)) for c in range(s_cycle, exp_dict['max_cycle']): # Set seed np.random.seed(c) torch.manual_seed(c) torch.cuda.manual_seed_all(c) if inner_s_epoch == 0: active_set.label_next_batch(model) hu.save_pkl(os.path.join(savedir, "active_set.pkl"), active_set.state_dict()) train_loader = DataLoader(active_set, sampler=samplers.get_sampler( exp_dict['sampler']['train'], active_set), batch_size=exp_dict["batch_size"]) # Visualize the model model.vis_on_loader(vis_loader, savedir=os.path.join(savedir, "images")) for e in range(inner_s_epoch, exp_dict['max_epoch']): # Validate only at the start of each cycle score_dict = {} if e == 0: score_dict.update(model.val_on_loader(val_loader)) # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model score_dict["epoch"] = len(score_list) score_dict["inner_epoch"] = e score_dict["cycle"] = c score_dict['n_ratio'] = active_set.n_labelled_ratio score_dict["n_train"] = len(train_loader.dataset) score_dict["n_pool"] = len(train_loader.dataset.pool) # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail(), "\n") hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) inner_s_epoch = 0
def trainval(exp_dict, savedir_base, data_root, reset=False, tensorboard=True): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) np.random.seed(exp_dict["seed"]) torch.manual_seed(exp_dict["seed"]) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) writer = tensorboardX.SummaryWriter(savedir) \ if tensorboard == 1 else None # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) pprint.pprint(exp_dict) print("Experiment saved in %s" % savedir) # Dataset # ----------- train_dataset, val_dataset = get_dataset(['train', 'val'], data_root, exp_dict) # val_dataset = get_dataset('val', exp_dict) # train and val loader if exp_dict["episodic"] == False: train_loader = DataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) val_loader = DataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, num_workers=args.num_workers) else: # to support episodes TODO: move inside each model from datasets.episodic_dataset import EpisodicDataLoader train_loader = EpisodicDataLoader(train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) val_loader = EpisodicDataLoader(val_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict, labelset=train_dataset.raw_labelset, writer=writer) print("Model with:", sum(p.numel() for p in model.parameters() if p.requires_grad), "parameters") # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(e, train_loader)) # Validate the model score_dict.update(model.val_on_loader(e, val_loader)) score_dict["epoch"] = e if tensorboard: for key, value in score_dict.items(): writer.add_scalar(key, value, e) writer.flush() # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) # if model.is_end(): # print("Early stopping") # break print('experiment completed') # Cleanup if tensorboard == 1: writer.close()
def get_dataset(dataset_name, train_flag, datadir, exp_dict): if dataset_name == "mnist": dataset = torchvision.datasets.MNIST(datadir, train=train_flag, download=True, transform=torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), torchvision.transforms.Normalize( (0.5,), (0.5,)) ])) if dataset_name == "cifar10": if train_flag: transform_function = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: transform_function = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) dataset = torchvision.datasets.CIFAR10( root=datadir, train=train_flag, download=True, transform=transform_function) if dataset_name == "cifar100": if train_flag: transform_function = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: transform_function = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) dataset = torchvision.datasets.CIFAR100( root=datadir, train=train_flag, download=True, transform=transform_function) if dataset_name in ['syn']: bias = 1; scaling = 10; sparsity = 10; solutionSparsity = 0.1; n = 1000 p = 100 A = np.random.randn(n,p)+bias; A = A.dot(np.diag(scaling* np.random.randn(p))) A = A * (np.random.rand(n,p) < (sparsity*np.log(n)/n)); w = np.random.randn(p) * (np.random.rand(p) < solutionSparsity); b = np.sign(A.dot(w)); b = b * np.sign(np.random.rand(n)-0.1); labels = np.unique(b) A = A / np.linalg.norm(A, axis=1)[:, None].clip(min=1e-6) A = A * 2 b[b==labels[0]] = 0 b[b==labels[1]] = 1 dataset = torch.utils.data.TensorDataset(torch.FloatTensor(A), torch.FloatTensor(b)) return DatasetWrapper(dataset) if dataset_name in ['mushrooms', 'w8a', 'rcv1', 'ijcnn']: sigma_dict = {"mushrooms": 0.5, "w8a":20.0, "rcv1":0.25 , "ijcnn":0.05} X, y = load_libsvm(dataset_name, data_dir=datadir) labels = np.unique(y) y[y==labels[0]] = 0 y[y==labels[1]] = 1 # splits used in experiments splits = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=9513451) X_train, X_test, Y_train, Y_test = splits if train_flag: # fname_rbf = "%s/rbf_%s_%s_train.pkl" % (datadir, dataset_name, sigma_dict[dataset_name]) fname_rbf = "%s/rbf_%s_%s_train.npy" % (datadir, dataset_name, sigma_dict[dataset_name]) if os.path.exists(fname_rbf): k_train_X = np.load(fname_rbf) else: k_train_X = rbf_kernel(X_train, X_train, sigma_dict[dataset_name]) np.save(fname_rbf, k_train_X) print('%s saved' % fname_rbf) X_train = k_train_X X_train = torch.FloatTensor(X_train) Y_train = torch.FloatTensor(Y_train) dataset = torch.utils.data.TensorDataset(X_train, Y_train) else: fname_rbf = "%s/rbf_%s_%s_test.npy" % (datadir, dataset_name, sigma_dict[dataset_name]) if os.path.exists(fname_rbf): k_test_X = np.load(fname_rbf) else: k_test_X = rbf_kernel(X_test, X_train, sigma_dict[dataset_name]) # hu.save_pkl(fname_rbf, k_test_X) np.save(fname_rbf, k_test_X) print('%s saved' % fname_rbf) X_test = k_test_X X_test = torch.FloatTensor(X_test) Y_test = torch.FloatTensor(Y_test) dataset = torch.utils.data.TensorDataset(X_test, Y_test) if dataset_name == "matrix_fac": fname = datadir + 'matrix_fac.pkl' if not os.path.exists(fname): data = generate_synthetic_matrix_factorization_data() hu.save_pkl(fname, data) A, y = hu.load_pkl(fname) X_train, X_test, y_train, y_test = train_test_split(A, y, test_size=0.2, random_state=9513451) training_set = torch.utils.data.TensorDataset(torch.tensor(X_train, dtype=torch.float), torch.tensor(y_train, dtype=torch.float)) test_set = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.float), torch.tensor(y_test, dtype=torch.float)) if train_flag: dataset = training_set else: dataset = test_set return DatasetWrapper(dataset)
def test_get_result_manager(self): # save a score_list savedir_base = '.tmp_plots' if os.path.exists(savedir_base): shutil.rmtree(savedir_base) exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'cifar10', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.25}, {'epoch': 1, 'acc':1.24}, {'epoch': 2, 'acc':1.5}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) exp_dict = {'model':{'name':'lenet', 'n_layers':30}, 'dataset':'cifar10', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.35}, {'epoch': 1, 'acc':1.2}, {'epoch': 2, 'acc':1.3}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) exp_dict = {'model':{'name':'lenet', 'n_layers':30}, 'dataset':'cifar10', 'batch_size':5} score_list = [{'epoch': 0, 'acc':0.15}, {'epoch': 1, 'acc':1.21}, {'epoch': 2, 'acc':1.7}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) rm = hr.ResultManager(savedir_base=savedir_base) # assert(len(rm.exp_groups) == 2) # for exp_list in rm.exp_groups: # assert(exp_list[0]['dataset'] in ['mnist', 'cifar10']) rm.get_exp_list_df() rm.get_score_df(avg_across='dataset') rm.get_score_df(avg_across='dataset', add_prefix=True) rm.get_score_df() rm.get_score_lists() rm.get_images() table = rm.get_score_table() table = rm.get_exp_table() fig_list = rm.get_plot(x_metric='epoch', y_metric='acc', title_list=['dataset'], legend_list=['model']) for i, fig in enumerate(fig_list): fig.savefig(os.path.join(savedir_base, '%d.png' % i)) order = 'groups_by_metrics' fig_list = rm.get_plot_all(order=order, x_metric='epoch', y_metric_list=['acc', 'epoch'], title_list=['dataset'], legend_list=['model'], groupby_list=['dataset'], log_metric_list=['acc'], map_title_list=[{'mnist':'MNIST'}, {'cifar10':'CIFAR-10'}], map_xlabel_list=[{'epoch':'EPOCHS'}], map_ylabel_list=[{'acc':'Score'}], ylim_list=[[(0.5, 0.8),(0.5, 0.8)], [(0.5, 0.8),(0.5, 0.8)]]) for i, fig in enumerate(fig_list): fig.savefig(os.path.join(savedir_base, '%s_%d.png' % (order, i))) order = 'metrics_by_groups' fig_list = rm.get_plot_all(order=order, x_metric='epoch', y_metric_list=['acc', 'epoch'], title_list=['dataset'], legend_list=['model'], avg_across='batch_size') for i, fig in enumerate(fig_list): fig.savefig(os.path.join(savedir_base, '%s_%d.png' % (order, i)))
def trainval(exp_dict, savedir_base, reset=False): # bookkeeping # --------------- # get experiment directory exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) if reset: # delete and backup experiment hc.delete_experiment(savedir, backup_flag=True) # create folder and save the experiment dictionary os.makedirs(savedir, exist_ok=True) hu.save_json(os.path.join(savedir, "exp_dict.json"), exp_dict) print(exp_dict) print("Experiment saved in %s" % savedir) # Set Seed # ------- seed = exp_dict.get('seed') np.random.seed(seed) torch.manual_seed(seed) # Dataset # ----------- train_dataset = get_dataset('train', exp_dict['dataset']) val_dataset = get_dataset('test', exp_dict['dataset']) # train and val loader train_loader = DataLoader( train_dataset, batch_size=exp_dict['batch_size'], shuffle=True, collate_fn=lambda x: x if exp_dict['batch_size'] == 1 else default_collate, # to handle episodes num_workers=args.num_workers) val_loader = DataLoader( val_dataset, batch_size=exp_dict['batch_size'], collate_fn=lambda x: x if exp_dict['batch_size'] == 1 else default_collate, shuffle=True, num_workers=args.num_workers) # Model # ----------- model = get_model(exp_dict) # Checkpoint # ----------- model_path = os.path.join(savedir, "model.pth") score_list_path = os.path.join(savedir, "score_list.pkl") if os.path.exists(score_list_path): # resume experiment model.set_state_dict(hu.torch_load(model_path)) score_list = hu.load_pkl(score_list_path) s_epoch = score_list[-1]['epoch'] + 1 else: # restart experiment score_list = [] s_epoch = 0 # Train & Val # ------------ print("Starting experiment at epoch %d" % (s_epoch)) for e in range(s_epoch, exp_dict['max_epoch']): score_dict = {} # Train the model score_dict.update(model.train_on_loader(train_loader)) # Validate the model savepath = os.path.join(savedir_base, exp_dict['dataset']['name']) score_dict.update(model.val_on_loader(val_loader, savedir=savepath)) model.on_train_end(savedir=savedir, epoch=e) score_dict["epoch"] = e # Visualize the model # model.vis_on_loader(vis_loader, savedir=savedir+"/images/") # Add to score_list and save checkpoint score_list += [score_dict] # Report & Save score_df = pd.DataFrame(score_list) print("\n", score_df.tail()) hu.torch_save(model_path, model.get_state_dict()) hu.save_pkl(score_list_path, score_list) print("Checkpoint Saved: %s" % savedir) print('experiment completed')
val_dict_dfc.rename(columns={ 'test_score': 'mIoU', 'test_class0': 'IoU class 0', 'test_class1': 'IoU class 1', 'test_mae': 'MAE', 'test_game': 'GAME' }, inplace=True) val_dict_lst.append(val_dict_dfc) val_dict_df = pd.concat(val_dict_lst, axis=0) val_dict_df.to_csv(os.path.join( '/mnt/public/predictions/habitat/', "%s_habitat_score_df.csv" % hash_id), index=False) val_dict_df.to_latex(os.path.join( '/mnt/public/predictions/habitat/', "%s_habitat_score_df.tex" % hash_id), index=False, caption=hash_dct[hash_id], label=hash_dct[hash_id]) hu.save_pkl(fname, val_dict) val_dict['model'] = exp_dict['model'] score_list += [val_dict] print(pd.DataFrame(score_list)) # score_df = pd.DataFrame(score_list) # score_df.to_csv(os.path.join('/mnt/public/predictions/habitat/', "habitat_score_df.csv")) # score_df.to_latex(os.path.join('/mnt/public/predictions/habitat/', "habitat_score_df.tex"))
if __name__ == "__main__": savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 4, "acc": 0.5}, {"epoch": 6, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) exp_dict = { "model": { "name": "mlp2", "n_layers": 35 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 2, "acc": 0.1}, {"epoch": 6, "acc": 0.3}]