def get_gen_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run, and if not do so eval_tag = "" if args.eval_tag == "none" else "-{}".format(args.eval_tag) if not os.path.isfile("{}/prec-{}.txt".format(args.r_dir, param_stamp)): print("{}: ...running...".format(param_stamp)) args.train = True main_cl.run(args) elif (os.path.isfile("{}/ll-{}.txt".format(args.r_dir, param_stamp)) and os.path.isfile("{}/is{}-{}.txt".format(args.r_dir, eval_tag, param_stamp))): print("{}: already run".format(param_stamp)) else: print("{}: ...running evaluation only...".format(param_stamp)) args.train = False main_cl.run(args) # -get average precisions fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() # -get log-likelihoods fileName = '{}/ll-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ll = float(file.readline()) file.close() # -get reconstruction error (per input unit) fileName = '{}/re-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) re = float(file.readline()) file.close() # -get inception score fileName = '{}/is{}-{}.txt'.format(args.r_dir, eval_tag, param_stamp) file = open(fileName) IS = float(file.readline()) file.close() # -get Frechet inception distance fileName = '{}/fid{}-{}.txt'.format(args.r_dir, eval_tag, param_stamp) file = open(fileName) FID = float(file.readline()) file.close() # -get precision and recall curve file_name = '{}/precision{}-{}.txt'.format(args.r_dir, eval_tag, param_stamp) precision = [] with open(file_name, 'r') as f: for line in f: precision.append(float(line[:-1])) file_name = '{}/recall{}-{}.txt'.format(args.r_dir, eval_tag, param_stamp) recall = [] with open(file_name, 'r') as f: for line in f: recall.append(float(line[:-1])) # -return tuple with the results return (ave, ll, re, IS, FID, precision, recall)
def get_results(args, model_name, shift, slot): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so if os.path.isfile('{}/dict-{}-{}-{}.pkl'.format(args.r_dir, param_stamp, args.slot, args.shift)): print("{}: already run".format(param_stamp)) else: print("{}: ...running...".format(param_stamp)) args.metrics = True main_cl.run(args, model_name=model_name, shift=shift, slot=slot) '''# -get average precision
def get_result(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run, and if not do so if os.path.isfile('{}/prec-{}.txt'.format(args.r_dir, param_stamp)): print("{}: already run".format(param_stamp)) else: print("{}: ...running...".format(param_stamp)) main_cl.run(args) # -get average precision fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() # -return it return ave
def get_prec(args, ext=""): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so if not os.path.isfile('{}/prec{}-{}.txt'.format(args.r_dir, ext, param_stamp)): print(" ...running: ... ") main.run(args) # -get average precision fileName = '{}/prec{}-{}.txt'.format(args.r_dir, ext, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() # -print average precision on screen print("--> average precision: {}".format(ave)) # -return average precision return ave
def get_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run, and if not do so if os.path.isfile("{}/dict-{}.pkl".format(args.r_dir, param_stamp)): print("{}: already run".format(param_stamp)) else: print("{}: ...running...".format(param_stamp)) main_cl.run(args) # -get average precisions fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() # -results-dict dict = utils.load_object("{}/dict-{}".format(args.r_dir, param_stamp)) # -return tuple with the results return (dict, ave)
def get_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so if os.path.isfile("{}/dict-{}.pkl".format(args.r_dir, param_stamp)): print("{}: already run".format(param_stamp)) else: print("{}: ...missing...".format(param_stamp)) raise Exception('Missing results !!!!') # -get results-dict dict = utils.load_object("{}/dict-{}".format(args.r_dir, param_stamp)) # -get average precision fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() # -print average precision on screen print("--> average precision: {}".format(ave)) # -return average precision return (dict, ave)
def get_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so # if os.path.isfile("{}/dict-{}.pkl".format(args.r_dir, param_stamp)): # print("{}: already run".format(param_stamp)) # else: # print("{}: ...missing...".format(param_stamp)) # raise Exception('Missing results !!!!') # -get results-dict result_dict = None # -get training time fileName = '{}/time-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) time = float(file.readline()) file.close() # -print average precision on screen print("--> average training time: {}".format(time)) # -return average precision return result_dict, time
def get_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so if not os.path.isfile('{}/prec-{}.txt'.format(args.r_dir, param_stamp)): print(" ... running ... ") main.run(args) # -get results-dict dict = utils.load_object("{}/dict-{}".format(args.r_dir, param_stamp)) # -get average precisions & trainig-times fileName = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) ave = float(file.readline()) file.close() fileName = '{}/time-{}.txt'.format(args.r_dir, param_stamp) file = open(fileName) training_time = float(file.readline()) file.close() # -print average precision on screen print("--> average precision: {}".format(ave)) # -return tuple with the results return (dict, ave, training_time)
def get_results(args): # -get param-stamp param_stamp = get_param_stamp_from_args(args) # -check whether already run; if not do so if os.path.isfile('{}/dict-{}.pkl'.format(args.r_dir, param_stamp)): print("{}: already run".format(param_stamp)) else: print("{}: ...running...".format(param_stamp)) args.metrics = True main_cl.run(args) # -get average precision file_name = '{}/prec-{}.txt'.format(args.r_dir, param_stamp) file = open(file_name) ave = float(file.readline()) file.close() # -get metrics-dict file_name = '{}/dict-{}'.format(args.r_dir, param_stamp) metrics_dict = utils.load_object(file_name) # -print average precision on screen print("--> average precision: {}".format(ave)) # -return average precision & metrics-dict return (ave, metrics_dict)
def run(args): # Set default arguments & check for incompatible options args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True args.bce = True args.bce_distill = True # -if XdG is selected but not the Task-IL scenario, give error if (not args.scenario == "task") and args.xdg: raise ValueError("'XdG' is only compatible with the Task-IL scenario.") # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.xdg or args.icarl): raise NotImplementedError( "EWC, SI, XdG and iCaRL are not supported with feedback connections." ) # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if args.xdg and (not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError( "XdG is not supported with both '{}' replay and EWC / SI.".format( args.replay)) #--> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -if 'BCEdistill' is selected for other than scenario=="class", give error if args.bce_distill and not args.scenario == "class": raise ValueError( "BCE-distill can only be used for class-incremental learning.") # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) scenario = args.scenario # If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain" # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) if args.singlehead and args.scenario == "task": scenario = "domain" # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) #print(train_datasets, test_datasets) #a = input() #-------------------------------------------------------------------------------------------------# #------------------------------# #----- MODEL (CLASSIFIER) -----# #------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. #--> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.xdg and args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=args.bce_distill, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- CL-STRATEGY: ALLOCATION -----# #-----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and (args.xdg and args.gating_prop > 0): mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.g_z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen # -main model print("\n") utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for plotting in visdom # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict( args.tasks) if args.use_exemplars else None # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format( fb="1M-" if args.feedback else "", replay="{}{}".format(args.replay, "D" if args.distill else ""), syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", xdg="" if (not args.xdg) or args.gating_prop == 0 else "-XdG{}".format(args.gating_prop), icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars and args.bce and args.bce_distill) else "", bud="-bud{}".format(args.budget) if (args.use_exemplars or args.add_exemplars or args.replay == "exemplars") else "", ) visdom = {'env': env_name, 'graph': graph_name} if args.use_exemplars: visdom_exemplars = { 'env': env_name, 'graph': "{}-EX".format(graph_name) } else: visdom = visdom_exemplars = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=scenario, ) # -with exemplars (both for visdom & reporting / pdf) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, visdom=visdom_exemplars, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, scenario=scenario, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# print("--> Training:" + args.name) print("Total tasks:" + str(args.tasks_to_complete)) # Keep track of training-time start = time.time() # Train model train_cl( args.tasks_to_complete, args.name, model, train_datasets, test_datasets, replay_mode=args.replay, scenario=scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, ) # Get total training-time in seconds, and write to file training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs_ex = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs_ex)) print("\n") #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -precision-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object( precision_dict_exemplars if args.use_exemplars else precision_dict, file_name) # Average precision on full test set not evaluated using exemplars (i.e., using softmax on final layer) if args.use_exemplars: output_file = open( "{}/prec_noex-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # -precision-dict: file_name = "{}/dict_noex-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf pp = visual_plt.open_pdf("{}/{}.pdf".format(args.p_dir, param_stamp)) # -show samples and reconstructions (either from main model or from separate generator) if args.feedback or args.replay == "generative": evaluate.show_samples(model if args.feedback else generator, config, size=args.sample_n, pdf=pp) for i in range(args.tasks): evaluate.show_reconstruction( model if args.feedback else generator, test_datasets[i], config, pdf=pp, task=i + 1) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) if args.use_exemplars: figure = visual_plt.plot_lines( precision_dict_exemplars["all_tasks"], x_axes=precision_dict_exemplars["x_task"], line_names=[ 'task {}'.format(i + 1) for i in range(args.tasks) ]) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close()
def run(args, model_name, shift, slot, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, tasks=args.tasks, slot=args.slot, shift=args.shift, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train), max_samples=args.max_samples) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and utils.checkattr( args, "pre_convE") and (hasattr(args, "depth") and args.depth > 0): print("\nDefining the model...") model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.herding = args.herding model.norm_exemplars = args.herding #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp, reinit_param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, no_task_mask=False, classes_per_task=classes_per_task, test_size=None) else: metrics_dict = None # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}-{tasks}".format(exp=args.experiment, tasks=args.tasks) replay_statement = "{mode}{b}".format( mode=args.replay, b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), ) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=False) ] if (not args.use_exemplars) else [None] #--> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=True) if args.use_exemplars else None ] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, model_name=model_name, shift=shift, slot=slot, replay_mode=args.replay if hasattr(args, 'replay') else "none", classes_per_task=classes_per_task, iters=args.iters, args=args, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last'), metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, ) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) # Load previously created metrics-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) metrics_dict = utils.load_object(file_name) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) # If requested, compute metrics '''if args.metrics:
def run(args): result_path = os.path.join('./benchmarks/results', args.savepath) savepath = result_path + '/' + str( datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv' if not os.path.exists(result_path): print('no exist the path and create one ...') os.makedirs(result_path, exist_ok=True) # Set default arguments args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True # -if EWC or SI is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.icarl): raise NotImplementedError( "EWC, SI and iCaRL are not supported with feedback connections.") # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) if not os.path.isdir(RESULT_DIR): os.mkdir(RESULT_DIR) # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = "cuda" if cuda else "cpu" gpu_devices = None if args.gpuID == None: if torch.cuda.device_count() > 1: gpu_devices = ','.join( [str(id) for id in range(torch.cuda.device_count())]) print('==> training with CUDA (GPU id: ' + gpu_devices + ') ... <==') else: gpu_devices = ','.join([str(id) for id in args.gpuID]) os.environ['CUDA_VISIBLE_DEVICES'] = gpu_devices print('==> training with CUDA (GPU id: ' + str(args.gpuID) + ') ... <==') # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) if args.factor == 'sequence': args.tasks = 12 # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for OpenLORIS-Object if args.dataset == 'OpenLORIS-Object': with open('./benchmarks/data/OpenLORIS-Object/' + args.factor + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) else: with open( './benchmarks/data/' + args.dataset + '/' + args.dataset + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) if args.cul == 1: for i in range(1, len(train_datasets)): train_datasets[i].imgs.extend(train_datasets[i - 1].imgs) train_datasets[i].labels.extend(train_datasets[i - 1].labels) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier(image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=False, binaryCE=args.bce).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict( args.tasks) if args.use_exemplars else None # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy eval_cb = cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb(log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] # --------------------# # ----- TRAINING -----# # --------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, test_datasets, replay_mode=args.replay, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, savepath=savepath, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, ) # -------------------------------------------------------------------------------------------------# # --------------------# # -- VISUALIZATION ---# # --------------------# matrices_names = args.matrices method_names = [] if args.cul == 1: method_names.append('Cumulative') elif args.cul == 0: method_names.append('Naive') if args.replay == 'current': method_names.append('LwF') if args.online and args.ewc: method_names.append('Online EWC') if args.si: method_names.append('SI') if args.replay == "generative" and not args.feedback and not args.distill: method_names.append('DGR') if args.replay == "generative" and not args.feedback and args.distill: method_names.append('DGR with distillation') if args.replay == "generative" and args.feedback and args.distill: method_names.append('DGR with feedback') if args.ewc and not args.online: method_names.append('EWC') print('The selected methods are:', method_names) print('The selected performance matrices are:', matrices_names) if args.cross_methods: print('==> Drawing results for cross selected-methods ... <==') if 'spider' in args.cross_methods_type: spider = True if 'bar' in args.cross_methods_type: bar = True if args.cross_tasks: print('==> Drawing results for cross tasks ... <==')
def run(args, verbose=False): # Set default arguments & check for incompatible options args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True args.bce = True args.bce_distill = True # -if XdG is selected but not the Task-IL scenario, give error if (not args.scenario == "task") and args.xdg: raise ValueError("'XdG' is only compatible with the Task-IL scenario.") # -if EWC, SI, XdG, A-GEM or iCaRL is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.xdg or args.icarl or args.agem): raise NotImplementedError( "EWC, SI, XdG, A-GEM and iCaRL are not supported with feedback connections." ) # -if A-GEM is selected without any replay, give warning if args.agem and args.replay == "none": raise Warning( "The '--agem' flag is selected, but without any type of replay. " "For the original A-GEM method, also select --replay='exemplars'.") # -if EWC, SI, XdG, A-GEM or iCaRL is selected together with offline-replay, give error if args.replay == "offline" and (args.ewc or args.si or args.xdg or args.icarl or args.agem): raise NotImplementedError( "Offline replay cannot be combined with EWC, SI, XdG, A-GEM or iCaRL." ) # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if (args.xdg and args.gating_prop > 0) and ( not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError( "XdG is not supported with both '{}' replay and EWC / SI.".format( args.replay)) # --> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -if 'BCEdistill' is selected for other than scenario=="class", give error if args.bce_distill and not args.scenario == "class": raise ValueError( "BCE-distill can only be used for class-incremental learning.") # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) scenario = args.scenario # If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain" # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) if args.singlehead and args.scenario == "task": scenario = "domain" # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=verbose, exception=True if args.seed == 0 else False, ) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.xdg and args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=args.bce_distill, AGEM=args.agem, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # -------------------------------------------------------------------------------------------------# # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -------------------------------------------------------------------------------------------------# # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and (args.xdg and args.gating_prop > 0): mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.g_z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # -------------------------------------------------------------------------------------------------# # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks, scenario=args.scenario) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, classes_per_task=classes_per_task, scenario=scenario, test_size=None, no_task_mask=False) else: metrics_dict = None # Prepare for plotting in visdom # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format( fb="1M-" if args.feedback else "", replay="{}{}{}".format(args.replay, "D" if args.distill else "", "-aGEM" if args.agem else ""), syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", xdg="" if (not args.xdg) or args.gating_prop == 0 else "-XdG{}".format(args.gating_prop), icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars and args.bce and args.bce_distill) else "", bud="-bud{}".format(args.budget) if (args.use_exemplars or args.add_exemplars or args.replay == "exemplars") else "", ) visdom = {'env': env_name, 'graph': graph_name} else: visdom = None # -------------------------------------------------------------------------------------------------# # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, with_exemplars=False) ] if (not args.use_exemplars) else [None] # --> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, scenario=scenario, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, with_exemplars=True) if args.use_exemplars else None ] # -------------------------------------------------------------------------------------------------# # --------------------# # ----- TRAINING -----# # --------------------# if verbose: print("\nTraining...") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, replay_mode=args.replay, scenario=scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, param_stamp=param_stamp, ) # Get total training-time in seconds, and write to file if args.time: training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() # -------------------------------------------------------------------------------------------------# # ----------------------# # ----- EVALUATION -----# # ----------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {} [{}-{}]: {:.4f}".format( i + 1, classes_per_task * i, classes_per_task * (i + 1) - 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) if args.metrics: # Accuracy matrix if args.scenario in ('task', 'domain'): R = pd.DataFrame(data=metrics_dict['acc per task'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict['initial acc per task'] if ( not args.use_exemplars) else ['NA' for _ in range(args.tasks)] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -print on screen if verbose: print("Accuracy matrix") print(R) print("\nFWT = {:.4f}".format(FWT)) print("BWT = {:.4f}".format(BWT)) print(" F = {:.4f}\n\n".format(F)) else: if verbose: # Accuracy matrix based only on classes in that task (i.e., evaluation as if Task-IL scenario) R = pd.DataFrame( data=metrics_dict['acc per task (only classes in task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) print( "Accuracy matrix, based on only classes in that task ('as if Task-IL scenario')" ) print(R) # Accuracy matrix, always based on all classes R = pd.DataFrame( data=metrics_dict['acc per task (all classes)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) print("\nAccuracy matrix, always based on all classes") print(R) # Accuracy matrix, based on all classes thus far R = pd.DataFrame(data=metrics_dict[ 'acc per task (all classes up to trained task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) print( "\nAccuracy matrix, based on all classes up to the trained task" ) print(R) # Accuracy matrix, based on all classes up to the task being evaluated # (this is the accuracy-matrix used for calculating the metrics in the Class-IL scenario) R = pd.DataFrame(data=metrics_dict[ 'acc per task (all classes up to evaluated task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -print on screen if verbose: print( "\nAccuracy matrix, based on all classes up to the evaluated task" ) print(R) print("\n=> FWT = {:.4f}".format(FWT)) print("=> BWT = {:.4f}".format(BWT)) print("=> F = {:.4f}\n".format(F)) if verbose and args.time: print( "=> Total training time = {:.1f} seconds\n".format(training_time)) # -------------------------------------------------------------------------------------------------# # ------------------# # ----- OUTPUT -----# # ------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -metrics-dict if args.metrics: file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(metrics_dict, file_name) # -------------------------------------------------------------------------------------------------# # --------------------# # ----- PLOTTING -----# # --------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = visual_plt.open_pdf(plot_name) # -show samples and reconstructions (either from main model or from separate generator) if args.feedback or args.replay == "generative": evaluate.show_samples(model if args.feedback else generator, config, size=args.sample_n, pdf=pp) for i in range(args.tasks): evaluate.show_reconstruction( model if args.feedback else generator, test_datasets[i], config, pdf=pp, task=i + 1) # -show metrics reflecting progression during training figure_list = [] # -> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) key = "acc per task ({} task)".format( "all classes up to trained" if scenario == 'class' else "only classes in") plot_list = [] for i in range(args.tasks): plot_list.append(metrics_dict[key]["task {}".format(i + 1)]) figure = visual_plt.plot_lines( plot_list, x_axes=metrics_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([metrics_dict["average"]], x_axes=metrics_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))
def run(args): result_path = os.path.join('./precision_onEachTask', args.savepath) savepath = result_path + '/' + str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv' os.makedirs(result_path, exist_ok=True) # Set default arguments args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.gating_prop > 0 or args.icarl): raise NotImplementedError("EWC, SI, XdG and iCaRL are not supported with feedback connections.") # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError("Binary classification loss not supported with feedback connections.") # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if args.gating_prop > 0 and (not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError("XdG is not supported with both '{}' replay and EWC / SI.".format(args.replay)) # --> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -create plots- and results-directories if needed if not os.path.isdir(RESULT_DIR): os.mkdir(RESULT_DIR) scenario = SCENARIO # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(SEED) torch.manual_seed(SEED) if cuda: torch.cuda.manual_seed(SEED) if args.factor == 'sequence': args.tasks = 12 # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for chosen experiment with open(args.factor + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) if args.cul == 1: for i in range(1, len(train_datasets)): train_datasets[i].imgs.extend(train_datasets[i - 1].imgs) train_datasets[i].labels.extend(train_datasets[i - 1].labels) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.g_fc_uni, z_dim=Z_DIM, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=True, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr}] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError("Unrecognized optimizer, '{}' is not currently a valid option".format(args.optimizer)) # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and args.gating_prop > 0: mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=100, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [ {'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen}] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Prepare for plotting in visdom # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict(args.tasks) if args.use_exemplars else None # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb(log=args.loss_log, visdom=VISDOM, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=VISDOM, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=VISDOM, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=VISDOM, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=SCENARIO, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=SCENARIO, ) # -with exemplars (both for visdom & reporting / pdf) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, visdom=VISDOM_EXEMPLARS, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, scenario=SCENARIO, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] # -------------------------------------------------------------------------------------------------# # --------------------# # ----- TRAINING -----# # --------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, test_datasets, replay_mode=args.replay, scenario=SCENARIO, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, savepath=savepath, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, )
def run(args, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, tasks=args.tasks, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train), max_samples=args.max_samples) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and utils.checkattr( args, "pre_convE") and (hasattr(args, "depth") and args.depth > 0): print("\nDefining the model...") model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.herding = args.herding model.norm_exemplars = args.herding #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp, reinit_param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, no_task_mask=False, classes_per_task=classes_per_task, test_size=None) else: metrics_dict = None # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}-{tasks}".format(exp=args.experiment, tasks=args.tasks) replay_statement = "{mode}{b}".format( mode=args.replay, b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), ) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=False) ] if (not args.use_exemplars) else [None] #--> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=True) if args.use_exemplars else None ] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, replay_mode=args.replay if hasattr(args, 'replay') else "none", classes_per_task=classes_per_task, iters=args.iters, args=args, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last'), metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, ) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) # Load previously created metrics-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) metrics_dict = utils.load_object(file_name) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) # If requested, compute metrics if args.metrics: # Load accuracy matrix of "reinit"-experiment (i.e., each task's accuracy when only trained on that task) if not utils.checkattr(args, 'reinit'): file_name = "{}/dict-{}".format(args.r_dir, reinit_param_stamp) if not os.path.isfile("{}.pkl".format(file_name)): raise FileNotFoundError( "Need to run the correct 'reinit' experiment (with --metrics) first!!" ) reinit_metrics_dict = utils.load_object(file_name) # Accuracy matrix R = pd.DataFrame( data=metrics_dict['acc per task'], index=['after task {}'.format(i + 1) for i in range(args.tasks)]) R = R[["task {}".format(task_id + 1) for task_id in range(args.tasks)]] R.loc['at start'] = metrics_dict['initial acc per task'] if ( not args.use_exemplars) else ['NA' for _ in range(args.tasks)] if not utils.checkattr(args, 'reinit'): R.loc['only trained on itself'] = [ reinit_metrics_dict['acc per task']['task {}'.format( task_id + 1)][task_id] for task_id in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)] + ['only trained on itself']) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -Vogelstein et al's measures of transfer efficiency if not utils.checkattr(args, 'reinit'): TEs = [((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(args.tasks), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] BTEs = [((1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(args.tasks), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] FTEs = [((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] # -TEs and BTEs after each task TEs_all = [] BTEs_all = [] for after_task_id in range(args.tasks): TEs_all.append([ ((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(after_task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(after_task_id + 1) ]) BTEs_all.append([ ((1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(after_task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(after_task_id + 1) ]) R.loc['TEs (per task, after all 10 tasks)'] = TEs for after_task_id in range(args.tasks): R.loc['TEs (per task, after {} tasks)'.format( after_task_id + 1)] = TEs_all[after_task_id] + ['NA'] * (args.tasks - after_task_id - 1) R.loc['BTEs (per task, after all 10 tasks)'] = BTEs for after_task_id in range(args.tasks): R.loc['BTEs (per task, after {} tasks)'.format( after_task_id + 1)] = BTEs_all[after_task_id] + ['NA'] * ( args.tasks - after_task_id - 1) R.loc['FTEs (per task)'] = FTEs metrics_dict['R'] = R # -print on screen if verbose: print("Accuracy matrix") print(R) print("\nFWT = {:.4f}".format(FWT)) print("BWT = {:.4f}".format(BWT)) print(" F = {:.4f}\n\n".format(F)) #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -metrics-dict if args.metrics: file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(metrics_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = evaluate.visual.plt.open_pdf(plot_name) # -plot TEs if not utils.checkattr(args, 'reinit'): BTEs = [] for task_id in range(args.tasks): BTEs.append([ R.loc['BTEs (per task, after {} tasks)'. format(after_task_id + 1), 'task {}'.format(task_id + 1)] for after_task_id in range(task_id, args.tasks) ]) figure = visual_plt.plot_TEs([FTEs], [BTEs], [TEs], ["test"]) pp.savefig(figure) # -show metrics reflecting progression during training if args.train and (not utils.checkattr(args, 'only_last')): # -create list to store all figures to be plotted. figure_list = [] # -generate all figures (and store them in [figure_list]) key = "acc per task" plot_list = [] for i in range(args.tasks): plot_list.append(metrics_dict[key]["task {}".format(i + 1)]) figure = visual_plt.plot_lines(plot_list, x_axes=metrics_dict["x_task"], line_names=[ 'task {}'.format(i + 1) for i in range(args.tasks) ]) figure_list.append(figure) figure = visual_plt.plot_lines( [metrics_dict["average"]], x_axes=metrics_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf for figure in figure_list: pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))
def run(args, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=args.scenario, tasks=args.tasks, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train)) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and (utils.checkattr(args, "pre_convE") or utils.checkattr(args, "pre_convD")) and \ (hasattr(args, "depth") and args.depth>0): print("\nDefining the model...") if utils.checkattr(args, 'feedback'): model = define.define_autoencoder(args=args, config=config, device=device) else: model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False if utils.checkattr(args, 'feedback') and utils.checkattr( args, "freeze_convD"): for param in model.convD.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = (hasattr(args, 'replay') and args.replay == "generative" and not utils.checkattr(args, 'feedback')) if train_gen: # Specify architecture generator = define.define_autoencoder(args, config, device, generator=True) # Initialize parameters generator = define.init_params(generator, args) # -freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in generator.convE.parameters(): param.requires_grad = False if utils.checkattr(args, "freeze_convD"): for param in generator.convD.parameters(): param.requires_grad = False # Set optimizer(s) generator.optim_list = [ { 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen if hasattr(args, 'lr_gen') else args.lr }, ] generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, replay_model_name=generator.name if (hasattr(args, 'replay') and args.replay in ("generative") and not utils.checkattr(args, 'feedback')) else None, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Define [progress_dicts] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) replay_statement = "{mode}{fb}{con}{gat}{int}{dis}{b}{u}".format( mode=args.replay, fb="Rtf" if utils.checkattr(args, "feedback") else "", con="Con" if (hasattr(args, "prior") and args.prior == "GMM" and utils.checkattr(args, "per_class")) else "", gat="Gat{}".format(args.dg_prop) if (utils.checkattr(args, "dg_gates") and hasattr(args, "dg_prop") and args.dg_prop > 0) else "", int="Int" if utils.checkattr(args, "hidden") else "", dis="Dis" if args.replay == "generative" and args.distill else "", b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), u="" if args.g_fc_uni == args.fc_units else "-gu{}".format( args.g_fc_uni)) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# g_iters = args.g_iters if hasattr(args, 'g_iters') else args.iters # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, replay=(hasattr(args, "replay") and not args.replay == "none"), model=model if utils.checkattr(args, 'feedback') else generator, tasks=args.tasks, iters_per_task=args.iters if utils.checkattr(args, 'feedback') else g_iters) ] if (train_gen or utils.checkattr(args, 'feedback')) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] if (not utils.checkattr(args, 'feedback')) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples no_samples = (utils.checkattr(args, "no_samples") or (utils.checkattr(args, "hidden") and hasattr(args, 'depth') and args.depth > 0)) sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=g_iters) ] if ((train_gen or utils.checkattr(args, 'feedback')) and not no_samples) else [None] # Callbacks for reporting and visualizing accuracy, and visualizing representation extracted by main model # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=args.scenario, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=args.scenario, ) # -visualize feature space latent_space_cb = cb._latent_space_cb( log=args.iters, datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, sample_size=400, ) # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full, latent_space_cb] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, replay_mode=args.replay if hasattr(args, 'replay') else "none", scenario=args.scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, generator=generator, gen_iters=g_iters, gen_loss_cbs=generator_loss_cbs, feedback=utils.checkattr(args, 'feedback'), sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if utils.checkattr(args, 'feedback') else solver_loss_cbs, args=args, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last')) # Save evaluation metrics measured throughout training file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) if generator is not None: save_name = "gM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( generator.name, args.full_stag) utils.save_checkpoint(generator, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) if generator is not None: load_name = "gM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( generator.name, args.full_ltag) utils.load_checkpoint(generator, args.m_dir, name=load_name, verbose=verbose) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if args.scenario == "task" else None) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Accuracy of final model on test-set:") for i in range(args.tasks): print(" - {} {}: {:.4f}".format( "For classes from task" if args.scenario == "class" else "Task", i + 1, precs[i])) print('=> Average accuracy over all {} {}: {:.4f}\n'.format( args.tasks * classes_per_task if args.scenario == "class" else args.tasks, "classes" if args.scenario == "class" else "tasks", average_precs)) # -write out to text file output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of GENERATOR -----# #-----------------------------------# if (utils.checkattr(args, 'feedback') or train_gen ) and args.experiment == "CIFAR100" and args.scenario == "class": # Dataset and model to be used test_set = ConcatDataset(test_datasets) gen_model = model if utils.checkattr(args, 'feedback') else generator gen_model.eval() # Evaluate log-likelihood of generative model on combined test-set (with S=100 importance samples per datapoint) ll_per_datapoint = gen_model.estimate_loglikelihood( test_set, S=100, batch_size=args.batch) if verbose: print('=> Log-likelihood on test set: {:.4f} +/- {:.4f}\n'.format( np.mean(ll_per_datapoint), np.sqrt(np.var(ll_per_datapoint)))) # -write out to text file output_file = open("{}/ll-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(np.mean(ll_per_datapoint))) output_file.close() # Evaluate reconstruction error (averaged over number of input units) re_per_datapoint = gen_model.calculate_recon_error( test_set, batch_size=args.batch, average=True) if verbose: print( '=> Reconstruction error (per input unit) on test set: {:.4f} +/- {:.4f}\n' .format(np.mean(re_per_datapoint), np.sqrt(np.var(re_per_datapoint)))) # -write out to text file output_file = open("{}/re-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(np.mean(re_per_datapoint))) output_file.close() # Try loading the classifier (our substitute for InceptionNet) for calculating IS, FID and Recall & Precision # -define model config['classes'] = 100 pretrained_classifier = define.define_classifier(args=args, config=config, device=device) pretrained_classifier.hidden = False # -load pretrained weights eval_tag = "" if args.eval_tag == "none" else "-{}".format( args.eval_tag) try: utils.load_checkpoint(pretrained_classifier, args.m_dir, verbose=True, name="{}{}".format( pretrained_classifier.name, eval_tag)) FileFound = True except FileNotFoundError: if verbose: print("= Could not find model {}{} in {}".format( pretrained_classifier.name, eval_tag, args.m_dir)) print("= IS, FID and Precision & Recall not computed!") FileFound = False pretrained_classifier.eval() # Only continue with computing these measures if the requested classifier network (using --eval-tag) was found if FileFound: # Preparations total_n = len(test_set) n_repeats = int(np.ceil(total_n / args.batch)) # -sample data from generator (for IS, FID and Precision & Recall) gen_x = gen_model.sample(size=total_n, only_x=True) # -generate predictions for generated data (for IS) gen_pred = [] for i in range(n_repeats): x = gen_x[(i * args.batch):int(min(((i + 1) * args.batch), total_n))] with torch.no_grad(): gen_pred.append( F.softmax(pretrained_classifier.hidden_to_output(x) if args.hidden else pretrained_classifier(x), dim=1).cpu().numpy()) gen_pred = np.concatenate(gen_pred) # -generate embeddings for generated data (for FID and Precision & Recall) gen_emb = [] for i in range(n_repeats): with torch.no_grad(): gen_emb.append( pretrained_classifier.feature_extractor( gen_x[(i * args.batch ):int(min(((i + 1) * args.batch), total_n))], from_hidden=args.hidden).cpu().numpy()) gen_emb = np.concatenate(gen_emb) # -generate embeddings for test data (for FID and Precision & Recall) data_loader = utils.get_data_loader(test_set, batch_size=args.batch, cuda=cuda) real_emb = [] for real_x, _ in data_loader: with torch.no_grad(): real_emb.append( pretrained_classifier.feature_extractor( real_x.to(device)).cpu().numpy()) real_emb = np.concatenate(real_emb) # Calculate "Inception Score" (IS) py = gen_pred.mean(axis=0) is_per_datapoint = [] for i in range(len(gen_pred)): pyx = gen_pred[i, :] is_per_datapoint.append(entropy(pyx, py)) IS = np.exp(np.mean(is_per_datapoint)) if verbose: print('=> Inception Score = {:.4f}\n'.format(IS)) # -write out to text file output_file = open( "{}/is{}-{}.txt".format(args.r_dir, eval_tag, param_stamp), 'w') output_file.write('{}\n'.format(IS)) output_file.close() ## Calculate "Frechet Inception Distance" (FID) FID = fid.calculate_fid_from_embedding(gen_emb, real_emb) if verbose: print('=> Frechet Inception Distance = {:.4f}\n'.format(FID)) # -write out to text file output_file = open( "{}/fid{}-{}.txt".format(args.r_dir, eval_tag, param_stamp), 'w') output_file.write('{}\n'.format(FID)) output_file.close() # Calculate "Precision & Recall"-curves precision, recall = pr.compute_prd_from_embedding( gen_emb, real_emb) # -write out to text files file_name = "{}/precision{}-{}.txt".format(args.r_dir, eval_tag, param_stamp) with open(file_name, 'w') as f: for item in precision: f.write("%s\n" % item) file_name = "{}/recall{}-{}.txt".format(args.r_dir, eval_tag, param_stamp) with open(file_name, 'w') as f: for item in recall: f.write("%s\n" % item) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = evaluate.visual.plt.open_pdf(plot_name) # -show metrics reflecting progression during training if args.train and (not utils.checkattr(args, 'only_last')): # -create list to store all figures to be plotted. figure_list = [] # -generate figures (and store them in [figure_list]) figure = evaluate.visual.plt.plot_lines( precision_dict["all_tasks"], x_axes=[ i * classes_per_task for i in precision_dict["x_task"] ] if args.scenario == "class" else precision_dict["x_task"], line_names=[ '{} {}'.format( "episode / task" if args.scenario == "class" else "task", i + 1) for i in range(args.tasks) ], xlabel="# of {}s so far".format("classe" if args.scenario == "class" else "task"), ylabel="Test accuracy") figure_list.append(figure) figure = evaluate.visual.plt.plot_lines( [precision_dict["average"]], x_axes=[ i * classes_per_task for i in precision_dict["x_task"] ] if args.scenario == "class" else precision_dict["x_task"], line_names=[ 'Average based on all {}s so far'.format(( "digit" if args.experiment == "splitMNIST" else "classe") if args.scenario else "task") ], xlabel="# of {}s so far".format("classe" if args.scenario == "class" else "task"), ylabel="Test accuracy") figure_list.append(figure) # -add figures to pdf for figure in figure_list: pp.savefig(figure) gen_eval = (utils.checkattr(args, 'feedback') or train_gen) # -show samples (from main model or separate generator) if gen_eval and not no_samples: evaluate.show_samples( model if utils.checkattr(args, 'feedback') else generator, config, size=args.sample_n, pdf=pp, title="Generated samples (by final model)") # -plot "Precision & Recall"-curve if gen_eval and args.experiment == "CIFAR100" and args.scenario == "class" and FileFound: figure = evaluate.visual.plt.plot_pr_curves([[precision]], [[recall]]) pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))
def run(args, verbose=False): # Create plots- and results-directories, if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it printed to screen and exit if utils.checkattr(args, "get_stamp"): print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #-----------------------# #----- DATA-STREAM -----# #-----------------------# # Find number of classes per task if args.experiment=="splitMNIST" and args.tasks>10: raise ValueError("Experiment 'splitMNIST' cannot have more than 10 tasks!") classes_per_task = 10 if args.experiment=="permMNIST" else int(np.floor(10/args.tasks)) # Print information on data-stream to screen if verbose: print("\nPreparing the data-stream...") print(" --> {}-incremental learning".format(args.scenario)) ti = "{} classes".format(args.tasks*classes_per_task) if args.stream=="random" and args.scenario=="class" else ( "{} tasks, with {} classes each".format(args.tasks, classes_per_task) ) print(" --> {} data stream: {}\n".format(args.stream, ti)) # Set up the stream of labels (i.e., classes, domain or tasks) to use if args.stream=="task-based": labels_per_batch = True if ((not args.scenario=="class") or classes_per_task==1) else False # -in Task- & Domain-IL scenario, each label is always for entire batch # -in Class-IL scenario, each label is always just for single observation # (but if there is just 1 class per task, setting `label-per_batch` to ``True`` is more efficient) label_stream = TaskBasedStream( n_tasks=args.tasks, iters_per_task=args.iters if labels_per_batch else args.iters*args.batch, labels_per_task=classes_per_task if args.scenario=="class" else 1 ) elif args.stream=="random": label_stream = RandomStream(labels=args.tasks*classes_per_task if args.scenario=="class" else args.tasks) else: raise NotImplementedError("Stream type '{}' not currently implemented.".format(args.stream)) # Load the data-sets (train_datasets, test_datasets), config, labels_per_task = prepare_datasets( name=args.experiment, n_labels=label_stream.n_labels, classes=(args.scenario=="class"), classes_per_task=classes_per_task, dir=args.d_dir, exception=(args.seed<10) ) # Set up the data-stream to be presented to the network data_stream = DataStream( train_datasets, label_stream, batch=args.batch, return_task=(args.scenario=="task"), per_batch=labels_per_batch if (args.stream=="task-based") else args.labels_per_batch, ) #-------------------------------------------------------------------------------------------------# #-----------------# #----- MODEL -----# #-----------------# # Define model # -how many units in the softmax output layer? (e.g., multi-headed or not?) softmax_classes = label_stream.n_labels if args.scenario=="class" else ( classes_per_task if (args.scenario=="domain" or args.singlehead) else classes_per_task*label_stream.n_labels ) # -set up model and move to correct device model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=softmax_classes, fc_layers=args.fc_lay, fc_units=args.fc_units, ).to(device) # -if using a multi-headed output layer, set the "label-per-task"-list as attribute of the model model.multi_head = labels_per_task if (args.scenario=="task" and not args.singlehead) else None #-------------------------------------------------------------------------------------------------# #---------------------# #----- OPTIMIZER -----# #---------------------# # Define optimizer (only include parameters that "requires_grad") optim_list = [{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr}] if not args.cs: # Use the chosen 'standard' optimizer if args.optimizer == "sgd": model.optimizer = optim.SGD(optim_list, weight_decay=args.decay) elif args.optimizer=="adam": model.optimizer = optim.Adam(optim_list, betas=(0.9, 0.999), weight_decay=args.decay) else: # Use the "complex synapse"-version of the chosen optimizer if args.optimizer=="sgd": model.optimizer = cs.ComplexSynapse(optim_list, n_beakers=args.beakers, alpha=args.alpha, beta=args.beta, verbose=verbose) elif args.optimizer=="adam": model.optimizer = cs.AdamComplexSynapse(optim_list, betas=(0.9, 0.999), n_beakers=args.beakers, alpha=args.alpha, beta=args.beta, verbose=verbose) #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp = get_param_stamp(args, model.name, verbose=verbose) # Print some model-characteristics on the screen if verbose: utils.print_model_info(model, title="MAIN MODEL") # Prepare for keeping track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_labels=label_stream.n_labels, classes=(args.scenario == "class")) # Prepare for plotting in visdom if args.visdom: env_name = "{exp}-{scenario}".format(exp=args.experiment, scenario=args.scenario) graph_name = "CS" if args.cs else "Normal" visdom = {'env': env_name, 'graph': graph_name} else: visdom = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss loss_cbs = [ cb.def_loss_cb( log=args.loss_log, visdom=visdom, tasks=label_stream.n_tasks, iters_per_task=args.iters if args.stream=="task-based" else None, task_name="Episode" if args.scenario=="class" else ("Task" if args.scenario=="task" else "Domain") ) ] # Callbacks for reporting and visualizing accuracy eval_cbs = [ cb.def_eval_cb(log=args.eval_log, test_datasets=test_datasets, scenario=args.scenario, iters_per_task=args.iters if args.stream=="task-based" else None, classes_per_task=classes_per_task, metrics_dict=metrics_dict, test_size=args.eval_n, visdom=visdom, provide_task_info=(args.scenario=="task")) ] # -evaluate accuracy before any training for eval_cb in eval_cbs: if eval_cb is not None: eval_cb(model, 0) #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# # Keep track of training-time if args.time: start = time.time() # Train model if verbose: print("\nTraining...") train_stream(model, data_stream, iters=args.iters*args.tasks if args.stream=="task-based" else args.iters, eval_cbs=eval_cbs, loss_cbs=loss_cbs) # Get total training-time in seconds, and write to file and screen if args.time: training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() if verbose: print("=> Total training time = {:.1f} seconds\n".format(training_time)) #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i+1 if args.scenario=="task" else None, ) for i in range(len(test_datasets))] average_precs = sum(precs) / len(test_datasets) # -print to screen if verbose: print("\n Precision on test-set:") for i in range(len(test_datasets)): print(" - {} {}: {:.4f}".format(args.scenario, i + 1, precs[i])) print('=> Average precision over all {} {}{}s: {:.4f}\n'.format( len(test_datasets), args.scenario, "e" if args.scenario=="class" else "", average_precs )) #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # -metrics-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(metrics_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = visual_plt.open_pdf(plot_name) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) key = "class" if args.scenario=='class' else "task" plot_list = [] for i in range(label_stream.n_labels): plot_list.append(metrics_dict["acc_per_{}".format(key)]["{}_{}".format(key, i+1)]) figure = visual_plt.plot_lines( plot_list, x_axes=metrics_dict["iters"], xlabel="Iterations", ylabel="Accuracy", line_names=['{} {}'.format(args.scenario, i+1) for i in range(label_stream.n_labels)] ) figure_list.append(figure) figure = visual_plt.plot_lines( [metrics_dict["ave_acc"]], x_axes=metrics_dict["iters"], xlabel="Iterations", ylabel="Accuracy", line_names=['average (over all {}{}s)'.format(args.scenario, "e" if args.scenario=="class" else "")], ylim=(0,1) ) figure_list.append(figure) figure = visual_plt.plot_lines( [metrics_dict["ave_acc_so_far"]], x_axes=metrics_dict["iters"], xlabel="Iterations", ylabel="Accuracy", line_names=['average (over all {}{}s so far)'.format(args.scenario, "e" if args.scenario=="class" else "")], ylim=(0,1) ) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))