def analyze_experiments(experiment_data_entries, hyperparams_selection=None, plot_seq_acc=True, plot_seq_forgetting=False, save_img_parent_dir=None, img_extention='png', legend_location='top', all_diff_color_force=False, ylim=None, taskcount=10): """ Pipeline data collection and plotting/summary.""" # Collect data experiment_data_entries, hyperparams_counts, max_task_count = collect_dataframe(experiment_data_entries, hyperparams_selection, taskcount) # Pad entries pad_dataframe(experiment_data_entries, hyperparams_counts) # Plot if save_img_parent_dir is not None: filename_template = save_img_parent_dir + "_TASK{}." + img_extention filename_template = filename_template.replace(" ", "") save_img_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'imgs', save_img_parent_dir, img_extention) utils.create_dir(save_img_path) save_img_path = os.path.join(save_img_path, filename_template) plot_multigraphs(experiment_data_entries, save_img_path, max_task_count, plot_seq_forgetting=plot_seq_forgetting, plot_seq_acc=plot_seq_acc, legend_location=legend_location, all_diff_color_force=all_diff_color_force, ylim=ylim, taskcount=taskcount) # Collect some statistics print_exp_statistics(experiment_data_entries)
def saved_for_eval(eval_loader, result, output_path, epoch=None, vs=False): """ save a results file in the format accepted by the submission server. input result : [ans_idxs, acc, q_ids, att_weights, spatials, mask_ans_idxs] """ label2ans = eval_loader.dataset.label2ans results, mask_results = [], [] for a, q_id in zip(result[0], result[2]): results.append({'question_id': int(q_id), 'answer': label2ans[a]}) # mask_results.append({'question_id': int(q_id), 'answer': label2ans[ma]}) if config.mode != 'trainval': name = 'eval_results.json' else: name = 'eval_results_epoch{}.json'.format(epoch) # results_path = os.path.join(output_path, name) # weights_path = os.path.join(output_path, 'att_weights.h5') # mask_results_path = os.path.join(output_path, 'mask_{}_eval_results.json'.format(config.masks)) # # mask_results_path = os.path.join(output, 'att_random_eval_results.json') # with open(mask_results_path, 'w') as fd: # json.dump(mask_results, fd) if vs: path = os.path.join(output_path, "vs/") utils.create_dir(path) with open(os.path.join(path, name), 'w') as fd: json.dump(results, fd) with h5py.File(os.path.join(path, 'att_weights.h5'), 'w') as f: f.create_dataset('weights', data=result[3]) f.create_dataset('spatials', data=result[4]) f.create_dataset('hints', data=result[6]) else: with open(os.path.join(output_path, name), 'w') as fd: json.dump(results, fd)
def save_hyperparams(self, output_dir, hyperparams): """ Add extra stats (memory,...) and save to output_dir. :param output_dir: Dir to export the dictionary to. :param hyperparams: Dictionary with hyperparams to save """ utils.create_dir(output_dir) hyperparams_outpath = os.path.join( output_dir, utils.get_hyperparams_output_filename()) torch.save(hyperparams, hyperparams_outpath) print("Saved hyperparams to: ", hyperparams_outpath)
def download_dset(path, location="eu"): """ Europe links are used, replace if Asia or North America. Location: eu (Europe), asia, """ assert location in ["eu", "asia", "us"] utils.create_dir(path) # TRAIN/VAL IMAGES train_link = "https://storage.googleapis.com/inat_data_2018_{}/train_val2018.tar.gz".format( location) train_tarname = train_link.split('/')[-1] # train_val2018.tar.gz train_dirname = train_tarname.split('.')[0] # train_val2018 if not os.path.exists(os.path.join(path, train_tarname)): download(path, train_link) print("Succesfully downloaded train+val dataset iNaturalist.") if not os.path.exists(os.path.join(path, train_dirname)): extract(path, os.path.join(path, train_tarname)) print("Succesfully extracted train+val dataset iNaturalist.") # TRAIN JSON trainjson_link = "https://storage.googleapis.com/inat_data_2018_{}/train2018.json.tar.gz".format( location) trainjson_tarname = trainjson_link.split('/')[-1] # train2018.json.tar.gz trainjson_filename = trainjson_link.split('.')[0] # train2018 if not os.path.exists(os.path.join(path, trainjson_tarname)): download(path, train_link) print("Succesfully downloaded train json iNaturalist.") if not os.path.exists(os.path.join(path, trainjson_filename)): extract(path, os.path.join(path, os.path.join(path, trainjson_tarname))) print("Succesfully extracted train json iNaturalist.") # VAL JSON trainjson_link = "https://storage.googleapis.com/inat_data_2018_{}/train2018.json.tar.gz".format( location) trainjson_tarname = trainjson_link.split('/')[-1] # train2018.json.tar.gz trainjson_filename = trainjson_link.split('.')[0] # train2018 if not os.path.exists(os.path.join(path, trainjson_tarname)): download(path, train_link) print("Succesfully downloaded train json iNaturalist.") if not os.path.exists(os.path.join(path, trainjson_filename)): extract(path, os.path.join(path, os.path.join(path, trainjson_tarname))) print("Succesfully extracted train json iNaturalist.")
def __init__(self, crop=False, create=False): super().__init__(crop=crop, create=create) self.original_dataset_root = self.dataset_root self.dataset_root = os.path.join(self.original_dataset_root, self.suffix) utils.create_dir(self.dataset_root) print(self.dataset_root) # Create symbolic links if non-existing for task in range(1, self.task_count + 1): src_taskdir = os.path.join(self.original_dataset_root, str(self.task_ordering[task - 1])) dst_tasklink = os.path.join(self.dataset_root, str(task)) if not os.path.exists(dst_tasklink): os.symlink(src_taskdir, dst_tasklink) print("CREATE LINK: {} -> {}".format(dst_tasklink, src_taskdir)) else: print("EXISTING LINK: {} -> {}".format(dst_tasklink, src_taskdir))
def load_chkpt(self, manager): """ Load checkpoint hyperparams from convergence. """ utils.create_dir(manager.heuristic_exp_dir) hyperparams_path = os.path.join( manager.heuristic_exp_dir, utils.get_hyperparams_output_filename()) try: print("Initiating framework chkpt:{}".format(hyperparams_path)) chkpt = torch.load(hyperparams_path) except: print( "CHECKPOINT LOAD FAILED: No state to restore, starting from scratch." ) return False self._restore_state(chkpt['state']) print("SUCCESSFUL loading framework chkpt:{}".format(hyperparams_path)) return True
def download_dset(path): utils.create_dir(path) if not os.path.exists(os.path.join(path, 'tiny-imagenet-200.zip')): subprocess.call( "wget -P {} http://cs231n.stanford.edu/tiny-imagenet-200.zip".format(path), shell=True) print("Succesfully downloaded TinyImgnet dataset.") else: print("Already downloaded TinyImgnet dataset in {}".format(path)) if not os.path.exists(os.path.join(path, 'tiny-imagenet-200')): subprocess.call( "unzip {} -d {}".format(os.path.join(path, 'tiny-imagenet-200.zip'), path), shell=True) print("Succesfully extracted TinyImgnet dataset.") else: print("Already extracted TinyImgnet dataset in {}".format(os.path.join(path, 'tiny-imagenet-200')))
def main(): args = parse_args() seed_torch(args.seed) print("epochs", args.epochs) print("lr", args.lr) print("optimizer", args.optimizer) os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu train_loader = data.get_loader('train', args.batch_size) val_loader = data.get_loader('test', args.batch_size) output_path = 'saved_models/{}/{}'.format(config.dataset, args.output) utils.create_dir(output_path) torch.backends.cudnn.benchmark = True embeddings = np.load(os.path.join(config.dataroot, 'glove6b_init_300d.npy')) constructor = 'build_baseline_with_dl' model = getattr(base_model, constructor)(embeddings).cuda() model = nn.DataParallel(model).cuda() if args.optimizer == 'adadelta': optimizer = optim.Adadelta(model.parameters(), rho=0.95, eps=1e-6, weight_decay=args.weight_decay) elif args.optimizer == 'rmsprop': optimizer = optim.RMSprop(model.parameters(), lr=0.01, alpha=0.99, eps=1e-08, weight_decay=args.weight_decay, momentum=0, centered=False) elif args.optimizer == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=args.weight_decay) elif args.optimizer == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.weight_decay) else: optimizer = optim.Adamax(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) start_epoch = 0 f1_val_best = 0 tracker = utils.Tracker() model_path = os.path.join(output_path, 'model.pth') for epoch in range(start_epoch, args.epochs): run(model, train_loader, optimizer, tracker, train=True, prefix='train', epoch=epoch) r = run(model, val_loader, optimizer, tracker, train=False, prefix='test', epoch=epoch) if r[4].mean() > f1_val_best: f1_val_best = r[4].mean() best_results = r utils.print_results(best_results[2], best_results[3])
def compute_target(answers_dset, ans2label, name): """Augment answers_dset with soft score as label ***answers_dset should be preprocessed*** Write result into a cache file """ target = [] for ans_entry in answers_dset: answers = ans_entry['answers'] answer_count = {} for answer in answers: answer_ = preprocess_answer(answer['answer']) answer_count[answer_] = answer_count.get(answer_, 0) + 1 labels = [] scores = [] for answer in answer_count: if answer not in ans2label: continue labels.append(ans2label[answer]) score = get_score(answer_count[answer]) scores.append(score) label_counts = {} for k, v in answer_count.items(): if k in ans2label: label_counts[ans2label[k]] = v # when all the answers are removed in training mode, skip this entrance target.append({ 'question_id': ans_entry['question_id'], 'question_type': ans_entry['question_type'], 'image_id': ans_entry['image_id'], 'labels': labels, 'label_counts': label_counts, 'scores': scores }) utils.create_dir(config.cache_root) cache_file = os.path.join(config.cache_root, name + '_target.json') json.dump(target, open(cache_file, 'w')) return target
def create_ans2label(occurence, name): """Note that this will also create label2ans.pkl at the same time occurence: dict {answer -> whatever} name: prefix of the output file cache_root: str """ ans2label = {} label2ans = [] label = 0 for answer in occurence: label2ans.append(answer) ans2label[answer] = label label += 1 utils.create_dir(config.cache_root) cache_file = os.path.join(config.cache_root, name + '_ans2label.json') json.dump(ans2label, open(cache_file, 'w')) cache_file = os.path.join(config.cache_root, name + '_label2ans.json') json.dump(label2ans, open(cache_file, 'w')) return ans2label
def create_train_val_test_imagefolder_dict_joint(dataset_root, img_paths, outfile, no_crop=True): """ For JOINT training: All 10 tasks in 1 data folder. Makes specific wrapper dictionary with the 3 ImageFolder objects we will use for training, validation and evaluation. """ # Data loading code if no_crop: out_dir = os.path.join(dataset_root, "no_crop") else: out_dir = dataset_root # Tiny Imgnet total values from pytorch normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) dsets = create_train_test_val_imagefolders(img_paths[1], dataset_root, normalize, True, no_crop=no_crop) ################ SAVE ################## utils.create_dir(out_dir) torch.save(dsets, os.path.join(out_dir, outfile)) print("JOINT SIZES: train={}, val={}, test={}".format(len(dsets['train']), len(dsets['val']), len(dsets['test']))) print("JOINT: Saved dictionary format of train/val/test dataset Imagefolders.")
def create_train_val_test_imagefolder_dict(dataset_root, img_paths, task_count, outfile, no_crop=True, transform=False): """ Makes specific wrapper dictionary with the 3 ImageFolder objects we will use for training, validation and evaluation. """ # Data loading code if no_crop: out_dir = os.path.join(dataset_root, "no_crop", "{}tasks".format(task_count)) else: out_dir = os.path.join(dataset_root, "{}tasks".format(task_count)) for task in range(1, task_count + 1): print("\nTASK ", task) # Tiny Imgnet total values from pytorch normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) dsets = create_train_test_val_imagefolders(img_paths[task], dataset_root, normalize, transform, no_crop) utils.create_dir(os.path.join(out_dir, str(task))) torch.save(dsets, os.path.join(out_dir, str(task), outfile)) print("SIZES: train={}, val={}, test={}".format(len(dsets['train']), len(dsets['val']), len(dsets['test']))) print("Saved dictionary format of train/val/test dataset Imagefolders.")
def lr_grid_single_task(args, manager, save_models_mode='keep_none'): """ Finetunes from starting model, acquire best lr and acc. LR gridsearch, with #finetune_iterations per LR. Makes symbolic link to the overall best iteration, corresponding with the obtained best lr. """ # Init manager.store_policy = StoragePolicy(save_models_mode) args.task_name = manager.dataset.get_taskname(args.task_counter) manager.ft_parent_exp_dir = os.path.join(manager.parent_exp_dir, 'task_' + str(args.task_counter), 'FT_LR_GRIDSEARCH') utils.create_dir(manager.ft_parent_exp_dir) print("FINETUNE LR GRIDSEARCH: Task ", args.task_name) # Logfile logfile_parent_dir = os.path.join(manager.ft_parent_exp_dir, 'log') utils.create_dir(logfile_parent_dir) logfile = os.path.join(logfile_parent_dir, utils.get_now() + '_finetune_grid.log') utils.append_to_file(logfile, "FINETUNE GRIDSEARCH LOG: Processed LRs") # Load Checkpoint processed_lrs = {} grid_checkpoint_file = os.path.join(manager.ft_parent_exp_dir, 'grid_checkpoint.pth') if os.path.exists(grid_checkpoint_file): checkpoint = torch.load(grid_checkpoint_file) processed_lrs = checkpoint['processed_lrs'] print("STARTING FROM CHECKPOINT: ", checkpoint) utils.append_to_file(logfile, "STARTING FROM CHECKPOINT") ######################################################## # PRESTEPS args.presteps_elapsed_time = 0 if hasattr(manager.method, 'grid_prestep'): manager.method.grid_prestep(args, manager) ######################################################## # LR GRIDSEARCH best_acc = 0 best_lr = None manager.best_exp_grid_node_dirname = None best_iteration_batch_dirs = [] for lr in args.lrs: print("\n", "<" * 20, "LR ", lr, ">" * 20) accum_acc = 0 best_iteration_dir = None best_iteration_acc = 0 iteration_batch_dirs = [] if lr not in processed_lrs: processed_lrs[lr] = {'acc': []} for finetune_iteration in range(0, args.finetune_iterations): print("\n", "-" * 20, "FT ITERATION ", finetune_iteration, "-" * 20) start_time = time.time() # Paths exp_grid_node_dirname = "lr=" + str( utils.float_to_scientific_str(lr)) if args.finetune_iterations > 1: exp_grid_node_dirname += "_it" + str(finetune_iteration) manager.gridsearch_exp_dir = os.path.join( manager.ft_parent_exp_dir, exp_grid_node_dirname) iteration_batch_dirs.append(manager.gridsearch_exp_dir) if finetune_iteration < len(processed_lrs[lr]['acc']): acc = processed_lrs[lr]['acc'][finetune_iteration] utils.set_random(finetune_iteration) print("RESTORING FROM CHECKPOINT: ITERATION = ", finetune_iteration, "ACC = ", acc) else: # Set new seed for reproducability utils.set_random(finetune_iteration) # Only actually saved when in save_model mode utils.create_dir(manager.gridsearch_exp_dir) # TRAIN model, acc = manager.method.grid_train(args, manager, lr) # Append results processed_lrs[lr]['acc'].append(acc) msg = "LR = {}, FT Iteration {}/{}, Acc = {}".format( lr, finetune_iteration + 1, args.finetune_iterations, acc) print(msg) utils.append_to_file(logfile, msg) # New best if acc > best_iteration_acc: if args.finetune_iterations > 1: msg = "=> NEW BEST FT ITERATION {}/{}: (Attempt '{}': Acc '{}' > best attempt Acc '{}')" \ .format(finetune_iteration + 1, args.finetune_iterations, finetune_iteration, acc, best_iteration_acc) print(msg) utils.append_to_file(logfile, msg) best_iteration_acc = acc best_iteration_dir = manager.gridsearch_exp_dir accum_acc = accum_acc + acc # update logfile/checkpoint torch.save({'processed_lrs': processed_lrs}, grid_checkpoint_file) # Save iteration hyperparams if hasattr(manager.method, "grid_chkpt") and manager.method.grid_chkpt: it_elapsed_time = time.time() - start_time hyperparams = { 'val_acc': acc, 'lr': lr, 'iteration_elapsed_time': it_elapsed_time, 'args': vars(args), 'manager': vars(manager) } utils.print_timing(it_elapsed_time, 'TRAIN') manager.save_hyperparams(manager.gridsearch_exp_dir, hyperparams) avg_acc = accum_acc / args.finetune_iterations print("Done FT iterations\n") print("LR AVG ACC = ", avg_acc, ", BEST OF LRs ACC = ", best_acc) # New it-avg best if avg_acc > best_acc: best_lr = lr best_acc = avg_acc manager.best_exp_grid_node_dirname = best_iteration_dir # Keep ref to best in all attempts print("UPDATE best lr = {}".format(best_lr)) print("UPDATE best lr acc= {}".format(best_acc)) utils.append_to_file(logfile, "UPDATE best lr = {}".format(best_lr)) utils.append_to_file(logfile, "UPDATE best lr acc= {}\n".format(best_acc)) # Clean all from previous best if manager.store_policy.only_keep_best: for out_dir in best_iteration_batch_dirs: if os.path.exists(out_dir): shutil.rmtree(out_dir, ignore_errors=True) print("[CLEANUP] removing {}".format(out_dir)) best_iteration_batch_dirs = iteration_batch_dirs else: if manager.store_policy.only_keep_best: for out_dir in iteration_batch_dirs: if os.path.exists(out_dir): shutil.rmtree(out_dir, ignore_errors=True) print("[CLEANUP] removing {}".format(out_dir)) if manager.store_policy.keep_none: for out_dir in iteration_batch_dirs: if os.path.exists(out_dir): shutil.rmtree(out_dir, ignore_errors=True) print("[CLEANUP] removing {}".format(out_dir)) print("FINETUNE DONE: best_lr={}, best_acc={}".format(best_lr, best_acc)) ######################################################## # POSTPROCESS if hasattr(manager.method, 'grid_poststep'): manager.method.grid_poststep(args, manager) return best_lr, best_acc
name = 'eval_results.json' else: name = 'eval_results_epoch{}.json'.format(epoch) results_path = os.path.join(output_path, name) with open(results_path, 'w') as fd: json.dump(results, fd) if __name__ == '__main__': args = parse_args() seed_torch(args.seed) os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.test_only: args.resume = True output_path = 'saved_models/{}/{}'.format(config.type + config.version, args.output) utils.create_dir(output_path) torch.backends.cudnn.benchmark = True ######################################### DATASET PREPARATION ####################################### if config.mode == 'train': train_loader = data.get_loader('train') val_loader = data.get_loader('val') elif args.test_only: train_loader = None val_loader = data.get_loader('test') else: train_loader = data.get_loader('trainval') val_loader = data.get_loader('test') ######################################### MODEL PREPARATION ####################################### embeddings = np.load(os.path.join(config.cache_root, 'glove6b_init_300d.npy'))
def main(overwrite_args, nc_per_task): """ For quick implementation: args are overwritten by overwrite_args (if specified). Additional params are passed in the main function. Do this task and return acc (task contains multiple classes = class-incremental in this setup) :param overwrite_args: :param nc_per_task: array with the amount of classes for each task :return: """ parser = argparse.ArgumentParser(description='Continuum learning') parser.add_argument('--task_name', type=str, help='name of the task') parser.add_argument('--task_count', type=int, help='count of the task, STARTING FROM 1') parser.add_argument('--prev_model_path', type=str, help='path to prev model where to start from') parser.add_argument('--save_path', type=str, default='results/', help='save models during and at the end of training') parser.add_argument('--n_outputs', type=int, default=200, help='total number of outputs for ALL tasks') parser.add_argument('--method', choices=[ 'gem', 'baseline_rehearsal_full_mem', 'icarl', 'baseline_rehearsal_partial_mem' ], type=str, default='gem', help='method to use for train') parser.add_argument( '--postprocess', action="store_true", help='Do datamanagement (e.g. update buffers) after task is learned') # implemented in separate step to only perform once on THE best model in pipeline parser.add_argument('--debug', action="store_true", help='Debug mode') # model parameters parser.add_argument('--n_hiddens', type=int, default=100, help='number of hidden neurons at each layer') parser.add_argument('--n_layers', type=int, default=2, help='number of hidden layers') parser.add_argument('--n_inputs', type=int, default=-1, help='number of hidden layers') parser.add_argument('--weight_decay', type=float, default=0, help='weight_decay') parser.add_argument('--is_scratch_model', action='store_true', help='is this the first task you train for?') # memory parameters parser.add_argument('--n_memories', type=int, default=0, help='number of memories per task') parser.add_argument('--memory_strength', default=0, type=float, help='memory strength (meaning depends on memory)') parser.add_argument('--finetune', action="store_true", help='whether to initialize nets in indep. nets') # optimizer parameters parser.add_argument('--n_epochs', type=int, default=1, help='Number of epochs per task') parser.add_argument('--batch_size', type=int, default=70, help='batch size') parser.add_argument('--lr', type=float, default=1e-3, help='SGD learning rate') # experiment parameters parser.add_argument('--cuda', action="store_true", help='Use GPU?') parser.add_argument('--log_every', type=int, default=100, help='frequency of logs, in minibatches') # data parameters parser.add_argument('dataset_path', type=str, help='path to imgfolders of datasets') parser.add_argument('--shuffle_tasks', type=str, default='no', help='present tasks in order') parser.add_argument('--n_tasks', type=int, default=10, help='number of hidden layers') ##################################### # ARGS PARSING ##################################### args = parser.parse_known_args()[0] # Overwrite with specified method args args.nc_per_task = nc_per_task # Array with nr outputs per class for key_arg, val_arg in overwrite_args.items(): setattr(args, key_arg, val_arg) # Index starting from 0 (for array) args.task_idx = args.task_count - 1 args.n_exemplars_to_append_per_batch = 0 if 'baseline_rehearsal' in args.method: args.finetune = True if args.method == 'baseline_rehearsal_full_mem': args.full_mem_mode = True elif args.method == 'baseline_rehearsal_partial_mem': args.full_mem_mode = False else: raise Exception("UNKNOWN BASELINE METHOD:", args.method) # Input checks assert args.n_outputs == sum(args.nc_per_task) assert args.n_tasks == len(nc_per_task) # Baselines from scratch, others from SI model if args.task_count == 1 and 'baseline' not in args.method: assert 'SI' in args.prev_model_path, "FIRST TASK NOT STARTING FROM SCRATCH, BUT FROM SI: " \ "ONLY STORING WRAPPER WITH EXEMPLARS, path = {}" \ .format(args.prev_model_path) assert args.postprocess, "FIRST TASK WE DO ONLY POSTPROCESSING" assert os.path.isfile( args.prev_model_path ), "Must specify existing prev_model_path, got: " + args.prev_model_path print("RUNNING WITH ARGS: ", overwrite_args) ##################################### # DATASET ##################################### # load data: We consider 1 task, not class-incremental dsets = torch.load(args.dataset_path) args.task_imgfolders = dsets args.dset_loaders = { x: torch.utils.data.DataLoader(ImageFolder_Subset_PathRetriever(dsets[x]), batch_size=args.batch_size, shuffle=True, num_workers=8, pin_memory=True) for x in ['train', 'val'] } dset_sizes = {x: len(dsets[x]) for x in ['train', 'val']} # Assign random part of batch to exemplars if 'baseline' in args.method or args.method == 'icarl': if args.method == 'baseline_rehearsal_partial_mem': # Only trains on exemplar sets, no validation # Based on lengths of tr datasets: random sampling, but guaranteed at each batch from all exemplar sets n_mem_samples = args.n_memories * args.task_idx elif args.method == 'baseline_rehearsal_full_mem' or args.method == 'icarl': n_mem_samples = args.n_memories * args.n_tasks n_total_samples = float(dset_sizes['train']) + n_mem_samples ratio = float(n_mem_samples) / n_total_samples if not args.debug: args.n_exemplars_to_append_per_batch = int( np.ceil(args.batch_size * ratio)) # Ceil: at least 1 per task else: args.n_exemplars_to_append_per_batch = int(args.batch_size / 2 + 17) args.total_batch_size = args.batch_size args.batch_size = args.batch_size - args.n_exemplars_to_append_per_batch print("BATCH CONSISTS OF: {} new samples, {} exemplars".format( args.batch_size, args.n_exemplars_to_append_per_batch)) print("mem length = {}, tr_dset_size={}, ratio={}".format( float(n_mem_samples), float(dset_sizes['train']), ratio)) ##################################### # LOADING MODEL ##################################### if args.is_scratch_model: # make model with self.net assert args.task_idx == 0 # Has to start from 0 Model = importlib.import_module('model.' + args.method) model = Model.Net(args.n_inputs, args.n_outputs, args.n_tasks, args) else: # load prev saved model print("Loading prev model from path: ", args.prev_model_path) model = torch.load(args.prev_model_path) print("MODEL LOADED") model.init_setup(args) # Checks assert model.n_tasks == args.n_tasks, "model tasks={}, args tasks={}".format( model.n_tasks, args.n_tasks) assert model.n_outputs == args.n_outputs print("MODEL SETUP FOR CURRENT TASK") if args.postprocess: ##################################### # POSTPROCESS ##################################### """ iCARL each task, GEM only first task model SI (during training no exemplars where gathered) """ print("POSTPROCESSING") model.manage_memory(args.task_idx, args) # if args.debug: # model.check_exemplars(args.task_idx) utils.create_dir(os.path.dirname(args.save_path), "Postprocessed model dir") torch.save(model, args.save_path) print("SAVED POSTPROCESSED MODEL TO: {}".format(args.save_path)) return None, None else: ##################################### # TRAIN ##################################### # if there is a checkpoint to be resumed, in case where the training has stopped before on a given task resume = os.path.join(args.save_path, 'epoch.pth.tar') model, best_val_acc = methods.rehearsal.train_rehearsal.train_model( model, args, dset_sizes, resume=resume) print("FINISHED TRAINING") return model, best_val_acc