def run(args, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=args.scenario, tasks=args.tasks, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train)) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and (utils.checkattr(args, "pre_convE") or utils.checkattr(args, "pre_convD")) and \ (hasattr(args, "depth") and args.depth>0): print("\nDefining the model...") if utils.checkattr(args, 'feedback'): model = define.define_autoencoder(args=args, config=config, device=device) else: model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False if utils.checkattr(args, 'feedback') and utils.checkattr( args, "freeze_convD"): for param in model.convD.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = (hasattr(args, 'replay') and args.replay == "generative" and not utils.checkattr(args, 'feedback')) if train_gen: # Specify architecture generator = define.define_autoencoder(args, config, device, generator=True) # Initialize parameters generator = define.init_params(generator, args) # -freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in generator.convE.parameters(): param.requires_grad = False if utils.checkattr(args, "freeze_convD"): for param in generator.convD.parameters(): param.requires_grad = False # Set optimizer(s) generator.optim_list = [ { 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen if hasattr(args, 'lr_gen') else args.lr }, ] generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, replay_model_name=generator.name if (hasattr(args, 'replay') and args.replay in ("generative") and not utils.checkattr(args, 'feedback')) else None, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Define [progress_dicts] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) replay_statement = "{mode}{fb}{con}{gat}{int}{dis}{b}{u}".format( mode=args.replay, fb="Rtf" if utils.checkattr(args, "feedback") else "", con="Con" if (hasattr(args, "prior") and args.prior == "GMM" and utils.checkattr(args, "per_class")) else "", gat="Gat{}".format(args.dg_prop) if (utils.checkattr(args, "dg_gates") and hasattr(args, "dg_prop") and args.dg_prop > 0) else "", int="Int" if utils.checkattr(args, "hidden") else "", dis="Dis" if args.replay == "generative" and args.distill else "", b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), u="" if args.g_fc_uni == args.fc_units else "-gu{}".format( args.g_fc_uni)) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# g_iters = args.g_iters if hasattr(args, 'g_iters') else args.iters # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, replay=(hasattr(args, "replay") and not args.replay == "none"), model=model if utils.checkattr(args, 'feedback') else generator, tasks=args.tasks, iters_per_task=args.iters if utils.checkattr(args, 'feedback') else g_iters) ] if (train_gen or utils.checkattr(args, 'feedback')) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] if (not utils.checkattr(args, 'feedback')) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples no_samples = (utils.checkattr(args, "no_samples") or (utils.checkattr(args, "hidden") and hasattr(args, 'depth') and args.depth > 0)) sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=g_iters) ] if ((train_gen or utils.checkattr(args, 'feedback')) and not no_samples) else [None] # Callbacks for reporting and visualizing accuracy, and visualizing representation extracted by main model # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=args.scenario, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=args.scenario, ) # -visualize feature space latent_space_cb = cb._latent_space_cb( log=args.iters, datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, sample_size=400, ) # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full, latent_space_cb] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, replay_mode=args.replay if hasattr(args, 'replay') else "none", scenario=args.scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, generator=generator, gen_iters=g_iters, gen_loss_cbs=generator_loss_cbs, feedback=utils.checkattr(args, 'feedback'), sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if utils.checkattr(args, 'feedback') else solver_loss_cbs, args=args, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last')) # Save evaluation metrics measured throughout training file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) if generator is not None: save_name = "gM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( generator.name, args.full_stag) utils.save_checkpoint(generator, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) if generator is not None: load_name = "gM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( generator.name, args.full_ltag) utils.load_checkpoint(generator, args.m_dir, name=load_name, verbose=verbose) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if args.scenario == "task" else None) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Accuracy of final model on test-set:") for i in range(args.tasks): print(" - {} {}: {:.4f}".format( "For classes from task" if args.scenario == "class" else "Task", i + 1, precs[i])) print('=> Average accuracy over all {} {}: {:.4f}\n'.format( args.tasks * classes_per_task if args.scenario == "class" else args.tasks, "classes" if args.scenario == "class" else "tasks", average_precs)) # -write out to text file output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of GENERATOR -----# #-----------------------------------# if (utils.checkattr(args, 'feedback') or train_gen ) and args.experiment == "CIFAR100" and args.scenario == "class": # Dataset and model to be used test_set = ConcatDataset(test_datasets) gen_model = model if utils.checkattr(args, 'feedback') else generator gen_model.eval() # Evaluate log-likelihood of generative model on combined test-set (with S=100 importance samples per datapoint) ll_per_datapoint = gen_model.estimate_loglikelihood( test_set, S=100, batch_size=args.batch) if verbose: print('=> Log-likelihood on test set: {:.4f} +/- {:.4f}\n'.format( np.mean(ll_per_datapoint), np.sqrt(np.var(ll_per_datapoint)))) # -write out to text file output_file = open("{}/ll-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(np.mean(ll_per_datapoint))) output_file.close() # Evaluate reconstruction error (averaged over number of input units) re_per_datapoint = gen_model.calculate_recon_error( test_set, batch_size=args.batch, average=True) if verbose: print( '=> Reconstruction error (per input unit) on test set: {:.4f} +/- {:.4f}\n' .format(np.mean(re_per_datapoint), np.sqrt(np.var(re_per_datapoint)))) # -write out to text file output_file = open("{}/re-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(np.mean(re_per_datapoint))) output_file.close() # Try loading the classifier (our substitute for InceptionNet) for calculating IS, FID and Recall & Precision # -define model config['classes'] = 100 pretrained_classifier = define.define_classifier(args=args, config=config, device=device) pretrained_classifier.hidden = False # -load pretrained weights eval_tag = "" if args.eval_tag == "none" else "-{}".format( args.eval_tag) try: utils.load_checkpoint(pretrained_classifier, args.m_dir, verbose=True, name="{}{}".format( pretrained_classifier.name, eval_tag)) FileFound = True except FileNotFoundError: if verbose: print("= Could not find model {}{} in {}".format( pretrained_classifier.name, eval_tag, args.m_dir)) print("= IS, FID and Precision & Recall not computed!") FileFound = False pretrained_classifier.eval() # Only continue with computing these measures if the requested classifier network (using --eval-tag) was found if FileFound: # Preparations total_n = len(test_set) n_repeats = int(np.ceil(total_n / args.batch)) # -sample data from generator (for IS, FID and Precision & Recall) gen_x = gen_model.sample(size=total_n, only_x=True) # -generate predictions for generated data (for IS) gen_pred = [] for i in range(n_repeats): x = gen_x[(i * args.batch):int(min(((i + 1) * args.batch), total_n))] with torch.no_grad(): gen_pred.append( F.softmax(pretrained_classifier.hidden_to_output(x) if args.hidden else pretrained_classifier(x), dim=1).cpu().numpy()) gen_pred = np.concatenate(gen_pred) # -generate embeddings for generated data (for FID and Precision & Recall) gen_emb = [] for i in range(n_repeats): with torch.no_grad(): gen_emb.append( pretrained_classifier.feature_extractor( gen_x[(i * args.batch ):int(min(((i + 1) * args.batch), total_n))], from_hidden=args.hidden).cpu().numpy()) gen_emb = np.concatenate(gen_emb) # -generate embeddings for test data (for FID and Precision & Recall) data_loader = utils.get_data_loader(test_set, batch_size=args.batch, cuda=cuda) real_emb = [] for real_x, _ in data_loader: with torch.no_grad(): real_emb.append( pretrained_classifier.feature_extractor( real_x.to(device)).cpu().numpy()) real_emb = np.concatenate(real_emb) # Calculate "Inception Score" (IS) py = gen_pred.mean(axis=0) is_per_datapoint = [] for i in range(len(gen_pred)): pyx = gen_pred[i, :] is_per_datapoint.append(entropy(pyx, py)) IS = np.exp(np.mean(is_per_datapoint)) if verbose: print('=> Inception Score = {:.4f}\n'.format(IS)) # -write out to text file output_file = open( "{}/is{}-{}.txt".format(args.r_dir, eval_tag, param_stamp), 'w') output_file.write('{}\n'.format(IS)) output_file.close() ## Calculate "Frechet Inception Distance" (FID) FID = fid.calculate_fid_from_embedding(gen_emb, real_emb) if verbose: print('=> Frechet Inception Distance = {:.4f}\n'.format(FID)) # -write out to text file output_file = open( "{}/fid{}-{}.txt".format(args.r_dir, eval_tag, param_stamp), 'w') output_file.write('{}\n'.format(FID)) output_file.close() # Calculate "Precision & Recall"-curves precision, recall = pr.compute_prd_from_embedding( gen_emb, real_emb) # -write out to text files file_name = "{}/precision{}-{}.txt".format(args.r_dir, eval_tag, param_stamp) with open(file_name, 'w') as f: for item in precision: f.write("%s\n" % item) file_name = "{}/recall{}-{}.txt".format(args.r_dir, eval_tag, param_stamp) with open(file_name, 'w') as f: for item in recall: f.write("%s\n" % item) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = evaluate.visual.plt.open_pdf(plot_name) # -show metrics reflecting progression during training if args.train and (not utils.checkattr(args, 'only_last')): # -create list to store all figures to be plotted. figure_list = [] # -generate figures (and store them in [figure_list]) figure = evaluate.visual.plt.plot_lines( precision_dict["all_tasks"], x_axes=[ i * classes_per_task for i in precision_dict["x_task"] ] if args.scenario == "class" else precision_dict["x_task"], line_names=[ '{} {}'.format( "episode / task" if args.scenario == "class" else "task", i + 1) for i in range(args.tasks) ], xlabel="# of {}s so far".format("classe" if args.scenario == "class" else "task"), ylabel="Test accuracy") figure_list.append(figure) figure = evaluate.visual.plt.plot_lines( [precision_dict["average"]], x_axes=[ i * classes_per_task for i in precision_dict["x_task"] ] if args.scenario == "class" else precision_dict["x_task"], line_names=[ 'Average based on all {}s so far'.format(( "digit" if args.experiment == "splitMNIST" else "classe") if args.scenario else "task") ], xlabel="# of {}s so far".format("classe" if args.scenario == "class" else "task"), ylabel="Test accuracy") figure_list.append(figure) # -add figures to pdf for figure in figure_list: pp.savefig(figure) gen_eval = (utils.checkattr(args, 'feedback') or train_gen) # -show samples (from main model or separate generator) if gen_eval and not no_samples: evaluate.show_samples( model if utils.checkattr(args, 'feedback') else generator, config, size=args.sample_n, pdf=pp, title="Generated samples (by final model)") # -plot "Precision & Recall"-curve if gen_eval and args.experiment == "CIFAR100" and args.scenario == "class" and FileFound: figure = evaluate.visual.plt.plot_pr_curves([[precision]], [[recall]]) pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))
def run(args): # Set default arguments args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni args.g_iters = args.iters if args.g_iters is None else args.g_iters # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if XdG is selected but not the incremental task learning scenario, give error if (not args.scenario == "task") and args.gating_prop > 0: raise ValueError( "'XdG' only works for the incremental task learning scenario.") # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.gating_prop > 0): raise NotImplementedError( "EWC, SI and XdG are not supported with feedback connections.") # -if XdG is selected together with replay of any kind, give error if args.gating_prop > 0 and (not args.replay == "none"): raise NotImplementedError( "XdG is not supported with '{}' replay.".format(args.replay)) # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=args.scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) #-------------------------------------------------------------------------------------------------# #------------------------------# #----- MODEL (CLASSIFIER) -----# #------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. #--> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.gating_prop > 0 else False, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # Set loss-function for reconstruction if args.feedback: model.recon_criterion = nn.BCELoss(size_average=True) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- CL-STRATEGY: ALLOCATION -----# #-----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and args.gating_prop > 0: mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) # -set reconstruction criterion generator.recon_criterion = nn.BCELoss(size_average=True) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) param_stamp = utils.get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen # -main model print("\n") utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for plotting # -open pdf pp = visual_plt.open_pdf("{}/{}.pdf".format( args.p_dir, param_stamp)) if args.pdf else None # -define [precision_dict] to keep track of performance during training for later plotting precision_dict = evaluate.initiate_precision_dict(args.tasks) # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{mode}{syn}{ewc}{XdG}".format( fb="1M-" if args.feedback else "", mode=args.replay, syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", XdG="" if args.gating_prop == 0 else "-XdG{}".format(args.gating_prop)) visdom = {'env': env_name, 'graph': graph_name} else: visdom = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb(log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log]) eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, scenario=args.scenario, collate_fn=utils.label_squeezing_collate_fn, test_size=args.prec_n, classes_per_task=classes_per_task, task_mask=True if isinstance(model, ContinualLearner) and (args.gating_prop > 0) else False) # -pdf: for summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, scenario=args.scenario, collate_fn=utils.label_squeezing_collate_fn, iters_per_task=args.iters, classes_per_task=classes_per_task, task_mask=True if isinstance(model, ContinualLearner) and (args.gating_prop > 0) else False) # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, replay_mode=args.replay, scenario=args.scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, collate_fn=utils.label_squeezing_collate_fn, visualize=True if args.visdom else False, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, ) # Get total training-time in seconds, and write to file training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# print('\n\n--> Evaluation ("incremental {} learning scenario"):'.format( args.scenario)) # Generation (plot in pdf) if (pp is not None) and train_gen: evaluate.show_samples(generator, config, size=args.sample_n, pdf=pp) if (pp is not None) and args.feedback: evaluate.show_samples(model, config, size=args.sample_n, pdf=pp) # Reconstruction (plot in pdf) if (pp is not None) and (train_gen or args.feedback): for i in range(args.tasks): if args.feedback: evaluate.show_reconstruction(model, test_datasets[i], config, pdf=pp, task=i + 1) else: evaluate.show_reconstruction(generator, test_datasets[i], config, pdf=pp, task=i + 1) # Classifier (print on screen & write to file) if args.scenario == "task": precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task_mask=True if isinstance(model, ContinualLearner) and args.gating_prop > 0 else False, task=i + 1, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] else: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1) for i in range(args.tasks) ] print("\n Precision on test-set:") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set (no restrictions on which nodes can be predicted: "incremental" / "singlehead") output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # Precision-dictionary file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if pp is not None: # -create list to store all figures to be plotted. figure_list = [] # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # Close pdf if pp is not None: pp.close()
def run(args): # Set default arguments & check for incompatible options args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True args.bce = True args.bce_distill = True # -if XdG is selected but not the Task-IL scenario, give error if (not args.scenario == "task") and args.xdg: raise ValueError("'XdG' is only compatible with the Task-IL scenario.") # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.xdg or args.icarl): raise NotImplementedError( "EWC, SI, XdG and iCaRL are not supported with feedback connections." ) # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if args.xdg and (not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError( "XdG is not supported with both '{}' replay and EWC / SI.".format( args.replay)) #--> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -if 'BCEdistill' is selected for other than scenario=="class", give error if args.bce_distill and not args.scenario == "class": raise ValueError( "BCE-distill can only be used for class-incremental learning.") # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) scenario = args.scenario # If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain" # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) if args.singlehead and args.scenario == "task": scenario = "domain" # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) #print(train_datasets, test_datasets) #a = input() #-------------------------------------------------------------------------------------------------# #------------------------------# #----- MODEL (CLASSIFIER) -----# #------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. #--> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.xdg and args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=args.bce_distill, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- CL-STRATEGY: ALLOCATION -----# #-----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and (args.xdg and args.gating_prop > 0): mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.g_z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen # -main model print("\n") utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for plotting in visdom # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict( args.tasks) if args.use_exemplars else None # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format( fb="1M-" if args.feedback else "", replay="{}{}".format(args.replay, "D" if args.distill else ""), syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", xdg="" if (not args.xdg) or args.gating_prop == 0 else "-XdG{}".format(args.gating_prop), icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars and args.bce and args.bce_distill) else "", bud="-bud{}".format(args.budget) if (args.use_exemplars or args.add_exemplars or args.replay == "exemplars") else "", ) visdom = {'env': env_name, 'graph': graph_name} if args.use_exemplars: visdom_exemplars = { 'env': env_name, 'graph': "{}-EX".format(graph_name) } else: visdom = visdom_exemplars = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=scenario, ) # -with exemplars (both for visdom & reporting / pdf) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, visdom=visdom_exemplars, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, scenario=scenario, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# print("--> Training:" + args.name) print("Total tasks:" + str(args.tasks_to_complete)) # Keep track of training-time start = time.time() # Train model train_cl( args.tasks_to_complete, args.name, model, train_datasets, test_datasets, replay_mode=args.replay, scenario=scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, ) # Get total training-time in seconds, and write to file training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs_ex = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs_ex)) print("\n") #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -precision-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object( precision_dict_exemplars if args.use_exemplars else precision_dict, file_name) # Average precision on full test set not evaluated using exemplars (i.e., using softmax on final layer) if args.use_exemplars: output_file = open( "{}/prec_noex-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # -precision-dict: file_name = "{}/dict_noex-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf pp = visual_plt.open_pdf("{}/{}.pdf".format(args.p_dir, param_stamp)) # -show samples and reconstructions (either from main model or from separate generator) if args.feedback or args.replay == "generative": evaluate.show_samples(model if args.feedback else generator, config, size=args.sample_n, pdf=pp) for i in range(args.tasks): evaluate.show_reconstruction( model if args.feedback else generator, test_datasets[i], config, pdf=pp, task=i + 1) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) if args.use_exemplars: figure = visual_plt.plot_lines( precision_dict_exemplars["all_tasks"], x_axes=precision_dict_exemplars["x_task"], line_names=[ 'task {}'.format(i + 1) for i in range(args.tasks) ]) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close()
def run(args): if not args.single_test: import pidfile resfile = pidfile.exclusive_dirfn( os.path.join(args.r_dir, args.save_dir)) if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # set cuda cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # set random seeds random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) scenario = args.scenario #------------------------------------------------------------------------------------------------- # DATA #------------------------------------------------------------------------------------------------- (train_datasets, test_datasets), config = get_multitask_experiment( args, name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) args.tasks = len(config['labels_per_task']) args.labels_per_task = config['labels_per_task'] if not args.task_boundary: args.iterations_per_virtual_epc = config['iterations_per_virtual_epc'] args.task_dict = config['task_dict'] #------------------------------------------------------------------------------------------------- # MODEL #------------------------------------------------------------------------------------------------- if args.ebm: model = EBM(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) else: model = Classifier(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) if args.experiment == 'cifar100': model = utils.init_params(model, args) for param in model.convE.parameters(): param.requires_grad = False if args.pretrain: checkpoint = torch.load(args.pretrain) best_acc = checkpoint['best_acc'] checkpoint_state = checkpoint['state_dict'] print( '-----------------------------------------------------------------------------' ) print('load pretrained model %s' % args.pretrain) print('best_acc', best_acc) print( '-----------------------------------------------------------------------------' ) model_dict = model.fcE.state_dict() checkpoint_state = { k[7:]: v for k, v in checkpoint_state.items() if k[7:] in model_dict } ## remove module. del checkpoint_state['classifier.weight'] del checkpoint_state['classifier.bias'] if 'y_ebm.weight' in checkpoint_state: del checkpoint_state['y_ebm.weight'] model_dict.update(checkpoint_state) model.fcE.load_state_dict(model_dict) for param in model.fcE.model.parameters(): param.requires_grad = False model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #------------------------------------------------------------------------------------------------- # CL-STRATEGY: ALLOCATION #------------------------------------------------------------------------------------------------- # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon #------------------------------------------------------------------------------------------------- # Get parameter-stamp (and print on screen) #------------------------------------------------------------------------------------------------- param_stamp = get_param_stamp(args, model.name, verbose=True) param_stamp = param_stamp + '--' + args.model_name # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, model=model, tasks=args.tasks, iters_per_task=args.iters) ] eval_cb = cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=args.visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cb_full = cb._eval_cb(log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cbs = [eval_cb, eval_cb_full] #------------------------------------------------------------------------------------------------- # TRAINING #------------------------------------------------------------------------------------------------- print("--> Training:") start = time.time() if args.task_boundary: train_cl(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) else: train_cl_noboundary(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) training_time = time.time() - start #------------------------------------------------------------------------------------------------- # EVALUATION #------------------------------------------------------------------------------------------------- print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) if args.ebm: precs = [ evaluate.validate_ebm(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] else: precs = [ evaluate.validate(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) #------------------------------------------------------------------------------------------------- # OUTPUT #------------------------------------------------------------------------------------------------- if not os.path.exists(os.path.join(args.r_dir, args.save_dir)): os.makedirs(os.path.join(args.r_dir, args.save_dir)) output_file = open( "{}/{}/{}.txt".format(args.r_dir, args.save_dir, param_stamp), 'w') output_file.write("Training time {} \n".format(training_time)) for i in range(args.tasks): output_file.write(" - Task {}: {:.4f}".format(i + 1, precs[i])) output_file.write("\n") output_file.write(' - Average {}\n'.format(average_precs)) output_file.close() file_name = "{}/{}/{}".format(args.r_dir, args.save_dir, param_stamp) utils.save_object(precision_dict, file_name) if args.pdf: pp = visual_plt.open_pdf("{}/{}/{}.pdf".format(args.r_dir, args.save_dir, param_stamp)) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) pp.close() if not args.single_test: resfile.done()
def run(args, model_name, shift, slot, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, tasks=args.tasks, slot=args.slot, shift=args.shift, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train), max_samples=args.max_samples) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and utils.checkattr( args, "pre_convE") and (hasattr(args, "depth") and args.depth > 0): print("\nDefining the model...") model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.herding = args.herding model.norm_exemplars = args.herding #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp, reinit_param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, no_task_mask=False, classes_per_task=classes_per_task, test_size=None) else: metrics_dict = None # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}-{tasks}".format(exp=args.experiment, tasks=args.tasks) replay_statement = "{mode}{b}".format( mode=args.replay, b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), ) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=False) ] if (not args.use_exemplars) else [None] #--> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=True) if args.use_exemplars else None ] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, model_name=model_name, shift=shift, slot=slot, replay_mode=args.replay if hasattr(args, 'replay') else "none", classes_per_task=classes_per_task, iters=args.iters, args=args, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last'), metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, ) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) # Load previously created metrics-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) metrics_dict = utils.load_object(file_name) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) # If requested, compute metrics '''if args.metrics:
def run(args, verbose=False): # Set default arguments & check for incompatible options args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True args.bce = True args.bce_distill = True # -if XdG is selected but not the Task-IL scenario, give error if (not args.scenario == "task") and args.xdg: raise ValueError("'XdG' is only compatible with the Task-IL scenario.") # -if EWC, SI, XdG, A-GEM or iCaRL is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.xdg or args.icarl or args.agem): raise NotImplementedError( "EWC, SI, XdG, A-GEM and iCaRL are not supported with feedback connections." ) # -if A-GEM is selected without any replay, give warning if args.agem and args.replay == "none": raise Warning( "The '--agem' flag is selected, but without any type of replay. " "For the original A-GEM method, also select --replay='exemplars'.") # -if EWC, SI, XdG, A-GEM or iCaRL is selected together with offline-replay, give error if args.replay == "offline" and (args.ewc or args.si or args.xdg or args.icarl or args.agem): raise NotImplementedError( "Offline replay cannot be combined with EWC, SI, XdG, A-GEM or iCaRL." ) # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if (args.xdg and args.gating_prop > 0) and ( not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError( "XdG is not supported with both '{}' replay and EWC / SI.".format( args.replay)) # --> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -if 'BCEdistill' is selected for other than scenario=="class", give error if args.bce_distill and not args.scenario == "class": raise ValueError( "BCE-distill can only be used for class-incremental learning.") # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) scenario = args.scenario # If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain" # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) if args.singlehead and args.scenario == "task": scenario = "domain" # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=verbose, exception=True if args.seed == 0 else False, ) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.xdg and args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=args.bce_distill, AGEM=args.agem, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # -------------------------------------------------------------------------------------------------# # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -------------------------------------------------------------------------------------------------# # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and (args.xdg and args.gating_prop > 0): mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.g_z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # -------------------------------------------------------------------------------------------------# # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks, scenario=args.scenario) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, classes_per_task=classes_per_task, scenario=scenario, test_size=None, no_task_mask=False) else: metrics_dict = None # Prepare for plotting in visdom # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format( fb="1M-" if args.feedback else "", replay="{}{}{}".format(args.replay, "D" if args.distill else "", "-aGEM" if args.agem else ""), syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", xdg="" if (not args.xdg) or args.gating_prop == 0 else "-XdG{}".format(args.gating_prop), icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars and args.bce and args.bce_distill) else "", bud="-bud{}".format(args.budget) if (args.use_exemplars or args.add_exemplars or args.replay == "exemplars") else "", ) visdom = {'env': env_name, 'graph': graph_name} else: visdom = None # -------------------------------------------------------------------------------------------------# # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, with_exemplars=False) ] if (not args.use_exemplars) else [None] # --> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, scenario=scenario, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, with_exemplars=True) if args.use_exemplars else None ] # -------------------------------------------------------------------------------------------------# # --------------------# # ----- TRAINING -----# # --------------------# if verbose: print("\nTraining...") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, replay_mode=args.replay, scenario=scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, param_stamp=param_stamp, ) # Get total training-time in seconds, and write to file if args.time: training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() # -------------------------------------------------------------------------------------------------# # ----------------------# # ----- EVALUATION -----# # ----------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {} [{}-{}]: {:.4f}".format( i + 1, classes_per_task * i, classes_per_task * (i + 1) - 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) if args.metrics: # Accuracy matrix if args.scenario in ('task', 'domain'): R = pd.DataFrame(data=metrics_dict['acc per task'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict['initial acc per task'] if ( not args.use_exemplars) else ['NA' for _ in range(args.tasks)] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -print on screen if verbose: print("Accuracy matrix") print(R) print("\nFWT = {:.4f}".format(FWT)) print("BWT = {:.4f}".format(BWT)) print(" F = {:.4f}\n\n".format(F)) else: if verbose: # Accuracy matrix based only on classes in that task (i.e., evaluation as if Task-IL scenario) R = pd.DataFrame( data=metrics_dict['acc per task (only classes in task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) print( "Accuracy matrix, based on only classes in that task ('as if Task-IL scenario')" ) print(R) # Accuracy matrix, always based on all classes R = pd.DataFrame( data=metrics_dict['acc per task (all classes)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) print("\nAccuracy matrix, always based on all classes") print(R) # Accuracy matrix, based on all classes thus far R = pd.DataFrame(data=metrics_dict[ 'acc per task (all classes up to trained task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) print( "\nAccuracy matrix, based on all classes up to the trained task" ) print(R) # Accuracy matrix, based on all classes up to the task being evaluated # (this is the accuracy-matrix used for calculating the metrics in the Class-IL scenario) R = pd.DataFrame(data=metrics_dict[ 'acc per task (all classes up to evaluated task)'], index=[ 'after task {}'.format(i + 1) for i in range(args.tasks) ]) R.loc['at start'] = metrics_dict[ 'initial acc per task (only classes in task)'] if not args.use_exemplars else [ 'NA' for _ in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)]) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -print on screen if verbose: print( "\nAccuracy matrix, based on all classes up to the evaluated task" ) print(R) print("\n=> FWT = {:.4f}".format(FWT)) print("=> BWT = {:.4f}".format(BWT)) print("=> F = {:.4f}\n".format(F)) if verbose and args.time: print( "=> Total training time = {:.1f} seconds\n".format(training_time)) # -------------------------------------------------------------------------------------------------# # ------------------# # ----- OUTPUT -----# # ------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -metrics-dict if args.metrics: file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(metrics_dict, file_name) # -------------------------------------------------------------------------------------------------# # --------------------# # ----- PLOTTING -----# # --------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = visual_plt.open_pdf(plot_name) # -show samples and reconstructions (either from main model or from separate generator) if args.feedback or args.replay == "generative": evaluate.show_samples(model if args.feedback else generator, config, size=args.sample_n, pdf=pp) for i in range(args.tasks): evaluate.show_reconstruction( model if args.feedback else generator, test_datasets[i], config, pdf=pp, task=i + 1) # -show metrics reflecting progression during training figure_list = [] # -> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) key = "acc per task ({} task)".format( "all classes up to trained" if scenario == 'class' else "only classes in") plot_list = [] for i in range(args.tasks): plot_list.append(metrics_dict[key]["task {}".format(i + 1)]) figure = visual_plt.plot_lines( plot_list, x_axes=metrics_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([metrics_dict["average"]], x_axes=metrics_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))
def run(args, verbose=False): # Create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # If only want param-stamp, get it and exit if args.get_stamp: from param_stamp import get_param_stamp_from_args print(get_param_stamp_from_args(args=args)) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Report whether cuda is used if verbose: print("CUDA is {}used".format("" if cuda else "NOT(!!) ")) # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment if verbose: print("\nPreparing the data...") (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, tasks=args.tasks, data_dir=args.d_dir, normalize=True if utils.checkattr(args, "normalize") else False, augment=True if utils.checkattr(args, "augment") else False, verbose=verbose, exception=True if args.seed < 10 else False, only_test=(not args.train), max_samples=args.max_samples) #-------------------------------------------------------------------------------------------------# #----------------------# #----- MAIN MODEL -----# #----------------------# # Define main model (i.e., classifier, if requested with feedback connections) if verbose and utils.checkattr( args, "pre_convE") and (hasattr(args, "depth") and args.depth > 0): print("\nDefining the model...") model = define.define_classifier(args=args, config=config, device=device) # Initialize / use pre-trained / freeze model-parameters # - initialize (pre-trained) parameters model = define.init_params(model, args) # - freeze weights of conv-layers? if utils.checkattr(args, "freeze_convE"): for param in model.convE.parameters(): param.requires_grad = False # Define optimizer (only optimize parameters that "requires_grad") model.optim_list = [ { 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }, ] model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.herding = args.herding model.norm_exemplars = args.herding #-------------------------------------------------------------------------------------------------# #----------------------------------------------------# #----- CL-STRATEGY: REGULARIZATION / ALLOCATION -----# #----------------------------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'ewc'): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.online = utils.checkattr(args, 'online') if model.online: model.gamma = args.gamma # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and utils.checkattr( args, 'xdg') and args.xdg_prop > 0: model.define_XdGmask(gating_prop=args.xdg_prop, n_tasks=args.tasks) #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, ContinualLearner) and hasattr( args, 'replay') and not args.replay == "none": model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) if verbose: print("\nParameter-stamp...") param_stamp, reinit_param_stamp = get_param_stamp( args, model.name, verbose=verbose, replay=True if (hasattr(args, 'replay') and not args.replay == "none") else False, ) # Print some model-characteristics on the screen if verbose: # -main model utils.print_model_info(model, title="MAIN MODEL") # Prepare for keeping track of statistics required for metrics (also used for plotting in pdf) if args.pdf or args.metrics: # -define [metrics_dict] to keep track of performance during training for storing & for later plotting in pdf metrics_dict = evaluate.initiate_metrics_dict(n_tasks=args.tasks) # -evaluate randomly initiated model on all tasks & store accuracies in [metrics_dict] (for calculating metrics) if not args.use_exemplars: metrics_dict = evaluate.intial_accuracy( model, test_datasets, metrics_dict, no_task_mask=False, classes_per_task=classes_per_task, test_size=None) else: metrics_dict = None # Prepare for plotting in visdom visdom = None if args.visdom: env_name = "{exp}-{tasks}".format(exp=args.experiment, tasks=args.tasks) replay_statement = "{mode}{b}".format( mode=args.replay, b="" if (args.batch_replay is None or args.batch_replay == args.batch) else "-br{}".format(args.batch_replay), ) if (hasattr(args, "replay") and not args.replay == "none") else "NR" graph_name = "{replay}{syn}{ewc}{xdg}".format( replay=replay_statement, syn="-si{}".format(args.si_c) if utils.checkattr(args, 'si') else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if utils.checkattr(args, "online") else "") if utils.checkattr( args, 'ewc') else "", xdg="" if (not utils.checkattr(args, 'xdg')) or args.xdg_prop == 0 else "-XdG{}".format(args.xdg_prop), ) visdom = {'env': env_name, 'graph': graph_name} #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, iters_per_task=args.iters, tasks=args.tasks, replay=(hasattr(args, "replay") and not args.replay == "none")) ] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cbs = [ cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=False) ] if (not args.use_exemplars) else [None] #--> during training on a task, evaluation cannot be with exemplars as those are only selected after training # (instead, evaluation for visdom is only done after each task, by including callback-function into [metric_cbs]) # Callbacks for calculating statists required for metrics # -pdf / reporting: summary plots (i.e, only after each task) (when using exemplars, also for visdom) metric_cbs = [ cb._metric_cb(log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, metrics_dict=metrics_dict, iters_per_task=args.iters, with_exemplars=args.use_exemplars), cb._eval_cb(log=args.iters, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, with_exemplars=True) if args.use_exemplars else None ] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# if args.train: if verbose: print("\nTraining...") # Train model train_cl( model, train_datasets, replay_mode=args.replay if hasattr(args, 'replay') else "none", classes_per_task=classes_per_task, iters=args.iters, args=args, batch_size=args.batch, batch_size_replay=args.batch_replay if hasattr( args, 'batch_replay') else None, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs, reinit=utils.checkattr(args, 'reinit'), only_last=utils.checkattr(args, 'only_last'), metric_cbs=metric_cbs, use_exemplars=args.use_exemplars, ) # Save trained model(s), if requested if args.save: save_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_stag') or args.full_stag == "none") else "{}-{}".format( model.name, args.full_stag) utils.save_checkpoint(model, args.m_dir, name=save_name, verbose=verbose) else: # Load previously trained model(s) (if goal is to only evaluate previously trained model) if verbose: print("\nLoading parameters of the previously trained models...") load_name = "mM-{}".format(param_stamp) if ( not hasattr(args, 'full_ltag') or args.full_ltag == "none") else "{}-{}".format( model.name, args.full_ltag) utils.load_checkpoint( model, args.m_dir, name=load_name, verbose=verbose, add_si_buffers=(isinstance(model, ContinualLearner) and utils.checkattr(args, 'si'))) # Load previously created metrics-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) metrics_dict = utils.load_object(file_name) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- EVALUATION of CLASSIFIER-----# #-----------------------------------# if verbose: print("\n\nEVALUATION RESULTS:") # Evaluate precision of final model on full test-set precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs = sum(precs) / args.tasks # -print on screen if verbose: print("\n Precision on test-set{}:".format( " (softmax classification)" if args.use_exemplars else "")) for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] average_precs_ex = sum(precs) / args.tasks # -print on screen if verbose: print(" Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) print('=> Average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs_ex)) # If requested, compute metrics if args.metrics: # Load accuracy matrix of "reinit"-experiment (i.e., each task's accuracy when only trained on that task) if not utils.checkattr(args, 'reinit'): file_name = "{}/dict-{}".format(args.r_dir, reinit_param_stamp) if not os.path.isfile("{}.pkl".format(file_name)): raise FileNotFoundError( "Need to run the correct 'reinit' experiment (with --metrics) first!!" ) reinit_metrics_dict = utils.load_object(file_name) # Accuracy matrix R = pd.DataFrame( data=metrics_dict['acc per task'], index=['after task {}'.format(i + 1) for i in range(args.tasks)]) R = R[["task {}".format(task_id + 1) for task_id in range(args.tasks)]] R.loc['at start'] = metrics_dict['initial acc per task'] if ( not args.use_exemplars) else ['NA' for _ in range(args.tasks)] if not utils.checkattr(args, 'reinit'): R.loc['only trained on itself'] = [ reinit_metrics_dict['acc per task']['task {}'.format( task_id + 1)][task_id] for task_id in range(args.tasks) ] R = R.reindex( ['at start'] + ['after task {}'.format(i + 1) for i in range(args.tasks)] + ['only trained on itself']) BWTs = [(R.loc['after task {}'.format(args.tasks), 'task {}'.format(i + 1)] - \ R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 1)]) for i in range(args.tasks - 1)] FWTs = [ 0. if args.use_exemplars else (R.loc['after task {}'.format(i + 1), 'task {}'.format(i + 2)] - R.loc['at start', 'task {}'.format(i + 2)]) for i in range(args.tasks - 1) ] forgetting = [] for i in range(args.tasks - 1): forgetting.append( max(R.iloc[1:args.tasks, i]) - R.iloc[args.tasks, i]) R.loc['FWT (per task)'] = ['NA'] + FWTs R.loc['BWT (per task)'] = BWTs + ['NA'] R.loc['F (per task)'] = forgetting + ['NA'] BWT = sum(BWTs) / (args.tasks - 1) F = sum(forgetting) / (args.tasks - 1) FWT = sum(FWTs) / (args.tasks - 1) metrics_dict['BWT'] = BWT metrics_dict['F'] = F metrics_dict['FWT'] = FWT # -Vogelstein et al's measures of transfer efficiency if not utils.checkattr(args, 'reinit'): TEs = [((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(args.tasks), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] BTEs = [((1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(args.tasks), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] FTEs = [((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(args.tasks)] # -TEs and BTEs after each task TEs_all = [] BTEs_all = [] for after_task_id in range(args.tasks): TEs_all.append([ ((1 - R.loc['only trained on itself', 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(after_task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(after_task_id + 1) ]) BTEs_all.append([ ((1 - R.loc['after task {}'.format(task_id + 1), 'task {}'.format(task_id + 1)]) / (1 - R.loc['after task {}'.format(after_task_id + 1), 'task {}'.format(task_id + 1)])) for task_id in range(after_task_id + 1) ]) R.loc['TEs (per task, after all 10 tasks)'] = TEs for after_task_id in range(args.tasks): R.loc['TEs (per task, after {} tasks)'.format( after_task_id + 1)] = TEs_all[after_task_id] + ['NA'] * (args.tasks - after_task_id - 1) R.loc['BTEs (per task, after all 10 tasks)'] = BTEs for after_task_id in range(args.tasks): R.loc['BTEs (per task, after {} tasks)'.format( after_task_id + 1)] = BTEs_all[after_task_id] + ['NA'] * ( args.tasks - after_task_id - 1) R.loc['FTEs (per task)'] = FTEs metrics_dict['R'] = R # -print on screen if verbose: print("Accuracy matrix") print(R) print("\nFWT = {:.4f}".format(FWT)) print("BWT = {:.4f}".format(BWT)) print(" F = {:.4f}\n\n".format(F)) #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -metrics-dict if args.metrics: file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(metrics_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf plot_name = "{}/{}.pdf".format(args.p_dir, param_stamp) pp = evaluate.visual.plt.open_pdf(plot_name) # -plot TEs if not utils.checkattr(args, 'reinit'): BTEs = [] for task_id in range(args.tasks): BTEs.append([ R.loc['BTEs (per task, after {} tasks)'. format(after_task_id + 1), 'task {}'.format(task_id + 1)] for after_task_id in range(task_id, args.tasks) ]) figure = visual_plt.plot_TEs([FTEs], [BTEs], [TEs], ["test"]) pp.savefig(figure) # -show metrics reflecting progression during training if args.train and (not utils.checkattr(args, 'only_last')): # -create list to store all figures to be plotted. figure_list = [] # -generate all figures (and store them in [figure_list]) key = "acc per task" plot_list = [] for i in range(args.tasks): plot_list.append(metrics_dict[key]["task {}".format(i + 1)]) figure = visual_plt.plot_lines(plot_list, x_axes=metrics_dict["x_task"], line_names=[ 'task {}'.format(i + 1) for i in range(args.tasks) ]) figure_list.append(figure) figure = visual_plt.plot_lines( [metrics_dict["average"]], x_axes=metrics_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf for figure in figure_list: pp.savefig(figure) # -close pdf pp.close() # -print name of generated plot on screen if verbose: print("\nGenerated plot: {}\n".format(plot_name))