def run(args): # Set default arguments args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni args.g_iters = args.iters if args.g_iters is None else args.g_iters # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if XdG is selected but not the incremental task learning scenario, give error if (not args.scenario == "task") and args.gating_prop > 0: raise ValueError( "'XdG' only works for the incremental task learning scenario.") # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.gating_prop > 0): raise NotImplementedError( "EWC, SI and XdG are not supported with feedback connections.") # -if XdG is selected together with replay of any kind, give error if args.gating_prop > 0 and (not args.replay == "none"): raise NotImplementedError( "XdG is not supported with '{}' replay.".format(args.replay)) # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=args.scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) #-------------------------------------------------------------------------------------------------# #------------------------------# #----- MODEL (CLASSIFIER) -----# #------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. #--> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.gating_prop > 0 else False, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # Set loss-function for reconstruction if args.feedback: model.recon_criterion = nn.BCELoss(size_average=True) #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- CL-STRATEGY: ALLOCATION -----# #-----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and args.gating_prop > 0: mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) # -set reconstruction criterion generator.recon_criterion = nn.BCELoss(size_average=True) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) param_stamp = utils.get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen # -main model print("\n") utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for plotting # -open pdf pp = visual_plt.open_pdf("{}/{}.pdf".format( args.p_dir, param_stamp)) if args.pdf else None # -define [precision_dict] to keep track of performance during training for later plotting precision_dict = evaluate.initiate_precision_dict(args.tasks) # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{mode}{syn}{ewc}{XdG}".format( fb="1M-" if args.feedback else "", mode=args.replay, syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", XdG="" if args.gating_prop == 0 else "-XdG{}".format(args.gating_prop)) visdom = {'env': env_name, 'graph': graph_name} else: visdom = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb(log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log]) eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, iters_per_task=args.iters, scenario=args.scenario, collate_fn=utils.label_squeezing_collate_fn, test_size=args.prec_n, classes_per_task=classes_per_task, task_mask=True if isinstance(model, ContinualLearner) and (args.gating_prop > 0) else False) # -pdf: for summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, scenario=args.scenario, collate_fn=utils.label_squeezing_collate_fn, iters_per_task=args.iters, classes_per_task=classes_per_task, task_mask=True if isinstance(model, ContinualLearner) and (args.gating_prop > 0) else False) # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, replay_mode=args.replay, scenario=args.scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, collate_fn=utils.label_squeezing_collate_fn, visualize=True if args.visdom else False, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, ) # Get total training-time in seconds, and write to file training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# print('\n\n--> Evaluation ("incremental {} learning scenario"):'.format( args.scenario)) # Generation (plot in pdf) if (pp is not None) and train_gen: evaluate.show_samples(generator, config, size=args.sample_n, pdf=pp) if (pp is not None) and args.feedback: evaluate.show_samples(model, config, size=args.sample_n, pdf=pp) # Reconstruction (plot in pdf) if (pp is not None) and (train_gen or args.feedback): for i in range(args.tasks): if args.feedback: evaluate.show_reconstruction(model, test_datasets[i], config, pdf=pp, task=i + 1) else: evaluate.show_reconstruction(generator, test_datasets[i], config, pdf=pp, task=i + 1) # Classifier (print on screen & write to file) if args.scenario == "task": precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task_mask=True if isinstance(model, ContinualLearner) and args.gating_prop > 0 else False, task=i + 1, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1)))) for i in range(args.tasks) ] else: precs = [ evaluate.validate(model, test_datasets[i], verbose=False, test_size=None, task=i + 1) for i in range(args.tasks) ] print("\n Precision on test-set:") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}\n'.format( args.tasks, average_precs)) #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set (no restrictions on which nodes can be predicted: "incremental" / "singlehead") output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # Precision-dictionary file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if pp is not None: # -create list to store all figures to be plotted. figure_list = [] # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # Close pdf if pp is not None: pp.close()
def run(args): # Set default arguments & check for incompatible options args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True args.bce = True args.bce_distill = True # -if XdG is selected but not the Task-IL scenario, give error if (not args.scenario == "task") and args.xdg: raise ValueError("'XdG' is only compatible with the Task-IL scenario.") # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.xdg or args.icarl): raise NotImplementedError( "EWC, SI, XdG and iCaRL are not supported with feedback connections." ) # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if args.xdg and (not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError( "XdG is not supported with both '{}' replay and EWC / SI.".format( args.replay)) #--> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -if 'BCEdistill' is selected for other than scenario=="class", give error if args.bce_distill and not args.scenario == "class": raise ValueError( "BCE-distill can only be used for class-incremental learning.") # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) scenario = args.scenario # If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain" # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) if args.singlehead and args.scenario == "task": scenario = "domain" # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) #-------------------------------------------------------------------------------------------------# #----------------# #----- DATA -----# #----------------# # Prepare data for chosen experiment (train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment( name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) #print(train_datasets, test_datasets) #a = input() #-------------------------------------------------------------------------------------------------# #------------------------------# #----- MODEL (CLASSIFIER) -----# #------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. #--> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.xdg and args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=args.bce_distill, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #-------------------------------------------------------------------------------------------------# #----------------------------------# #----- CL-STRATEGY: EXEMPLARS -----# #----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding #-------------------------------------------------------------------------------------------------# #-----------------------------------# #----- CL-STRATEGY: ALLOCATION -----# #-----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and (args.xdg and args.gating_prop > 0): mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list #-------------------------------------------------------------------------------------------------# #-------------------------------# #----- CL-STRATEGY: REPLAY -----# #-------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.g_z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- REPORTING -----# #---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Print some model-characteristics on the screen # -main model print("\n") utils.print_model_info(model, title="MAIN MODEL") # -generator if generator is not None: utils.print_model_info(generator, title="GENERATOR") # Prepare for plotting in visdom # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict( args.tasks) if args.use_exemplars else None # -visdom-settings if args.visdom: env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment, tasks=args.tasks, scenario=args.scenario) graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format( fb="1M-" if args.feedback else "", replay="{}{}".format(args.replay, "D" if args.distill else ""), syn="-si{}".format(args.si_c) if args.si else "", ewc="-ewc{}{}".format( args.ewc_lambda, "-O{}".format(args.gamma) if args.online else "") if args.ewc else "", xdg="" if (not args.xdg) or args.gating_prop == 0 else "-XdG{}".format(args.gating_prop), icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars and args.bce and args.bce_distill) else "", bud="-bud{}".format(args.budget) if (args.use_exemplars or args.add_exemplars or args.replay == "exemplars") else "", ) visdom = {'env': env_name, 'graph': graph_name} if args.use_exemplars: visdom_exemplars = { 'env': env_name, 'graph': "{}-EX".format(graph_name) } else: visdom = visdom_exemplars = None #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, visdom=visdom, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=visdom, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, visdom=visdom, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=scenario, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=scenario, ) # -with exemplars (both for visdom & reporting / pdf) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, visdom=visdom_exemplars, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, scenario=scenario, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] #-------------------------------------------------------------------------------------------------# #--------------------# #----- TRAINING -----# #--------------------# print("--> Training:" + args.name) print("Total tasks:" + str(args.tasks_to_complete)) # Keep track of training-time start = time.time() # Train model train_cl( args.tasks_to_complete, args.name, model, train_datasets, test_datasets, replay_mode=args.replay, scenario=scenario, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, ) # Get total training-time in seconds, and write to file training_time = time.time() - start time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w') time_file.write('{}\n'.format(training_time)) time_file.close() #-------------------------------------------------------------------------------------------------# #----------------------# #----- EVALUATION -----# #----------------------# print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) # Evaluate precision of final model on full test-set precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) # -with exemplars if args.use_exemplars: precs = [ evaluate.validate( model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=True, allowed_classes=list( range(classes_per_task * i, classes_per_task * (i + 1))) if scenario == "task" else None) for i in range(args.tasks) ] print("\n Precision on test-set (classification using exemplars):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs_ex = sum(precs) / args.tasks print('=> average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs_ex)) print("\n") #-------------------------------------------------------------------------------------------------# #------------------# #----- OUTPUT -----# #------------------# # Average precision on full test set output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format( average_precs_ex if args.use_exemplars else average_precs)) output_file.close() # -precision-dict file_name = "{}/dict-{}".format(args.r_dir, param_stamp) utils.save_object( precision_dict_exemplars if args.use_exemplars else precision_dict, file_name) # Average precision on full test set not evaluated using exemplars (i.e., using softmax on final layer) if args.use_exemplars: output_file = open( "{}/prec_noex-{}.txt".format(args.r_dir, param_stamp), 'w') output_file.write('{}\n'.format(average_precs)) output_file.close() # -precision-dict: file_name = "{}/dict_noex-{}".format(args.r_dir, param_stamp) utils.save_object(precision_dict, file_name) #-------------------------------------------------------------------------------------------------# #--------------------# #----- PLOTTING -----# #--------------------# # If requested, generate pdf if args.pdf: # -open pdf pp = visual_plt.open_pdf("{}/{}.pdf".format(args.p_dir, param_stamp)) # -show samples and reconstructions (either from main model or from separate generator) if args.feedback or args.replay == "generative": evaluate.show_samples(model if args.feedback else generator, config, size=args.sample_n, pdf=pp) for i in range(args.tasks): evaluate.show_reconstruction( model if args.feedback else generator, test_datasets[i], config, pdf=pp, task=i + 1) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) if args.use_exemplars: figure = visual_plt.plot_lines( precision_dict_exemplars["all_tasks"], x_axes=precision_dict_exemplars["x_task"], line_names=[ 'task {}'.format(i + 1) for i in range(args.tasks) ]) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) # -close pdf pp.close()
def run(args): result_path = os.path.join('./precision_onEachTask', args.savepath) savepath = result_path + '/' + str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv' os.makedirs(result_path, exist_ok=True) # Set default arguments args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True # -if EWC, SI or XdG is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.gating_prop > 0 or args.icarl): raise NotImplementedError("EWC, SI, XdG and iCaRL are not supported with feedback connections.") # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError("Binary classification loss not supported with feedback connections.") # -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine) if args.gating_prop > 0 and (not args.replay == "none") and (args.ewc or args.si): raise NotImplementedError("XdG is not supported with both '{}' replay and EWC / SI.".format(args.replay)) # --> problem is that applying different task-masks interferes with gradient calculation # (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately) # -create plots- and results-directories if needed if not os.path.isdir(RESULT_DIR): os.mkdir(RESULT_DIR) scenario = SCENARIO # (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL) # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # Set random seeds np.random.seed(SEED) torch.manual_seed(SEED) if cuda: torch.cuda.manual_seed(SEED) if args.factor == 'sequence': args.tasks = 12 # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for chosen experiment with open(args.factor + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) if args.cul == 1: for i in range(1, len(train_datasets)): train_datasets[i].imgs.extend(train_datasets[i - 1].imgs) train_datasets[i].labels.extend(train_datasets[i - 1].labels) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.g_fc_uni, z_dim=Z_DIM, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.gating_prop > 0 else False, binaryCE=args.bce, binaryCE_distill=True, ).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr}] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError("Unrecognized optimizer, '{}' is not currently a valid option".format(args.optimizer)) # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # XdG: create for every task a "mask" for each hidden fully connected layer if isinstance(model, ContinualLearner) and args.gating_prop > 0: mask_dict = {} excit_buffer_list = [] for task_id in range(args.tasks): mask_dict[task_id + 1] = {} for i in range(model.fcE.layers): layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear if task_id == 0: excit_buffer_list.append(layer.excit_buffer) n_units = len(layer.excit_buffer) gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False) mask_dict[task_id + 1][i] = gated_units model.mask_dict = mask_dict model.excit_buffer_list = excit_buffer_list # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=100, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [ {'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen}] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # Prepare for plotting in visdom # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict(args.tasks) if args.use_exemplars else None # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb(log=args.loss_log, visdom=VISDOM, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, visdom=VISDOM, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb(log=args.sample_log, visdom=VISDOM, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy # -visdom (i.e., after each [prec_log] eval_cb = cb._eval_cb( log=args.prec_log, test_datasets=test_datasets, visdom=VISDOM, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task, scenario=SCENARIO, ) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb( log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=SCENARIO, ) # -with exemplars (both for visdom & reporting / pdf) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, visdom=VISDOM_EXEMPLARS, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, scenario=SCENARIO, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] # -------------------------------------------------------------------------------------------------# # --------------------# # ----- TRAINING -----# # --------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, test_datasets, replay_mode=args.replay, scenario=SCENARIO, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, savepath=savepath, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, )
def run(args): if not args.single_test: import pidfile resfile = pidfile.exclusive_dirfn( os.path.join(args.r_dir, args.save_dir)) if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # set cuda cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # set random seeds random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) scenario = args.scenario #------------------------------------------------------------------------------------------------- # DATA #------------------------------------------------------------------------------------------------- (train_datasets, test_datasets), config = get_multitask_experiment( args, name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) args.tasks = len(config['labels_per_task']) args.labels_per_task = config['labels_per_task'] if not args.task_boundary: args.iterations_per_virtual_epc = config['iterations_per_virtual_epc'] args.task_dict = config['task_dict'] #------------------------------------------------------------------------------------------------- # MODEL #------------------------------------------------------------------------------------------------- if args.ebm: model = EBM(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) else: model = Classifier(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) if args.experiment == 'cifar100': model = utils.init_params(model, args) for param in model.convE.parameters(): param.requires_grad = False if args.pretrain: checkpoint = torch.load(args.pretrain) best_acc = checkpoint['best_acc'] checkpoint_state = checkpoint['state_dict'] print( '-----------------------------------------------------------------------------' ) print('load pretrained model %s' % args.pretrain) print('best_acc', best_acc) print( '-----------------------------------------------------------------------------' ) model_dict = model.fcE.state_dict() checkpoint_state = { k[7:]: v for k, v in checkpoint_state.items() if k[7:] in model_dict } ## remove module. del checkpoint_state['classifier.weight'] del checkpoint_state['classifier.bias'] if 'y_ebm.weight' in checkpoint_state: del checkpoint_state['y_ebm.weight'] model_dict.update(checkpoint_state) model.fcE.load_state_dict(model_dict) for param in model.fcE.model.parameters(): param.requires_grad = False model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #------------------------------------------------------------------------------------------------- # CL-STRATEGY: ALLOCATION #------------------------------------------------------------------------------------------------- # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon #------------------------------------------------------------------------------------------------- # Get parameter-stamp (and print on screen) #------------------------------------------------------------------------------------------------- param_stamp = get_param_stamp(args, model.name, verbose=True) param_stamp = param_stamp + '--' + args.model_name # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, model=model, tasks=args.tasks, iters_per_task=args.iters) ] eval_cb = cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=args.visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cb_full = cb._eval_cb(log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cbs = [eval_cb, eval_cb_full] #------------------------------------------------------------------------------------------------- # TRAINING #------------------------------------------------------------------------------------------------- print("--> Training:") start = time.time() if args.task_boundary: train_cl(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) else: train_cl_noboundary(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) training_time = time.time() - start #------------------------------------------------------------------------------------------------- # EVALUATION #------------------------------------------------------------------------------------------------- print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) if args.ebm: precs = [ evaluate.validate_ebm(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] else: precs = [ evaluate.validate(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) #------------------------------------------------------------------------------------------------- # OUTPUT #------------------------------------------------------------------------------------------------- if not os.path.exists(os.path.join(args.r_dir, args.save_dir)): os.makedirs(os.path.join(args.r_dir, args.save_dir)) output_file = open( "{}/{}/{}.txt".format(args.r_dir, args.save_dir, param_stamp), 'w') output_file.write("Training time {} \n".format(training_time)) for i in range(args.tasks): output_file.write(" - Task {}: {:.4f}".format(i + 1, precs[i])) output_file.write("\n") output_file.write(' - Average {}\n'.format(average_precs)) output_file.close() file_name = "{}/{}/{}".format(args.r_dir, args.save_dir, param_stamp) utils.save_object(precision_dict, file_name) if args.pdf: pp = visual_plt.open_pdf("{}/{}/{}.pdf".format(args.r_dir, args.save_dir, param_stamp)) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) pp.close() if not args.single_test: resfile.done()
def run(args): result_path = os.path.join('./benchmarks/results', args.savepath) savepath = result_path + '/' + str( datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv' if not os.path.exists(result_path): print('no exist the path and create one ...') os.makedirs(result_path, exist_ok=True) # Set default arguments args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen args.g_iters = args.iters if args.g_iters is None else args.g_iters args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni # -if [log_per_task], reset all logs if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters args.sample_log = args.iters # -if [iCaRL] is selected, select all accompanying options if hasattr(args, "icarl") and args.icarl: args.use_exemplars = True args.add_exemplars = True # -if EWC or SI is selected together with 'feedback', give error if args.feedback and (args.ewc or args.si or args.icarl): raise NotImplementedError( "EWC, SI and iCaRL are not supported with feedback connections.") # -if binary classification loss is selected together with 'feedback', give error if args.feedback and args.bce: raise NotImplementedError( "Binary classification loss not supported with feedback connections." ) if not os.path.isdir(RESULT_DIR): os.mkdir(RESULT_DIR) # If only want param-stamp, get it printed to screen and exit if hasattr(args, "get_stamp") and args.get_stamp: _ = get_param_stamp_from_args(args=args) exit() # Use cuda? cuda = torch.cuda.is_available() and args.cuda device = "cuda" if cuda else "cpu" gpu_devices = None if args.gpuID == None: if torch.cuda.device_count() > 1: gpu_devices = ','.join( [str(id) for id in range(torch.cuda.device_count())]) print('==> training with CUDA (GPU id: ' + gpu_devices + ') ... <==') else: gpu_devices = ','.join([str(id) for id in args.gpuID]) os.environ['CUDA_VISIBLE_DEVICES'] = gpu_devices print('==> training with CUDA (GPU id: ' + str(args.gpuID) + ') ... <==') # Set random seeds np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) if args.factor == 'sequence': args.tasks = 12 # -------------------------------------------------------------------------------------------------# # ----------------# # ----- DATA -----# # ----------------# # Prepare data for OpenLORIS-Object if args.dataset == 'OpenLORIS-Object': with open('./benchmarks/data/OpenLORIS-Object/' + args.factor + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) else: with open( './benchmarks/data/' + args.dataset + '/' + args.dataset + '.pk', 'rb') as f: ((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f) if args.cul == 1: for i in range(1, len(train_datasets)): train_datasets[i].imgs.extend(train_datasets[i - 1].imgs) train_datasets[i].labels.extend(train_datasets[i - 1].labels) # -------------------------------------------------------------------------------------------------# # ------------------------------# # ----- MODEL (CLASSIFIER) -----# # ------------------------------# # Define main model (i.e., classifier, if requested with feedback connections) if args.feedback: model = AutoEncoder( image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) model.lamda_pl = 1. # --> to make that this VAE is also trained to classify else: model = Classifier(image_size=config['size'], image_channels=config['channels'], classes=config['classes'], fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl, fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=False, binaryCE=args.bce).to(device) # Define optimizer (only include parameters that "requires_grad") model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) # ----------------------------------# # ----- CL-STRATEGY: EXEMPLARS -----# # ----------------------------------# # Store in model whether, how many and in what way to store exemplars if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"): model.memory_budget = args.budget model.norm_exemplars = args.norm_exemplars model.herding = args.herding # -----------------------------------# # ----- CL-STRATEGY: ALLOCATION -----# # -----------------------------------# # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon # -------------------------------------------------------------------------------------------------# # -------------------------------# # ----- CL-STRATEGY: REPLAY -----# # -------------------------------# # Use distillation loss (i.e., soft targets) for replayed data? (and set temperature) if isinstance(model, Replayer): model.replay_targets = "soft" if args.distill else "hard" model.KD_temp = args.temp # If needed, specify separate model for the generator train_gen = True if (args.replay == "generative" and not args.feedback) else False if train_gen: # -specify architecture generator = AutoEncoder( image_size=config['size'], image_channels=config['channels'], fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=args.z_dim, classes=config['classes'], fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl, ).to(device) # -set optimizer(s) generator.optim_list = [{ 'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen }] generator.optim_type = args.optimizer if generator.optim_type in ("adam", "adam_reset"): generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999)) elif generator.optim_type == "sgd": generator.optimizer = optim.SGD(generator.optim_list) else: generator = None # ---------------------# # ----- REPORTING -----# # ---------------------# # Get parameter-stamp (and print on screen) param_stamp = get_param_stamp( args, model.name, verbose=True, replay=True if (not args.replay == "none") else False, replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None, ) # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) precision_dict_exemplars = evaluate.initiate_precision_dict( args.tasks) if args.use_exemplars else None # ---------------------# # ----- CALLBACKS -----# # ---------------------# # Callbacks for reporting on and visualizing loss generator_loss_cbs = [ cb._VAE_loss_cb( log=args.loss_log, model=model if args.feedback else generator, tasks=args.tasks, iters_per_task=args.iters if args.feedback else args.g_iters, replay=False if args.replay == "none" else True) ] if (train_gen or args.feedback) else [None] solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, model=model, tasks=args.tasks, iters_per_task=args.iters, replay=False if args.replay == "none" else True) ] if (not args.feedback) else [None] # Callbacks for evaluating and plotting generated / reconstructed samples sample_cbs = [ cb._sample_cb( log=args.sample_log, config=config, test_datasets=test_datasets, sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters) ] if (train_gen or args.feedback) else [None] # Callbacks for reporting and visualizing accuracy eval_cb = cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, classes_per_task=classes_per_task) # -pdf / reporting: summary plots (i.e, only after each task) eval_cb_full = cb._eval_cb(log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, classes_per_task=classes_per_task) eval_cb_exemplars = cb._eval_cb( log=args.iters, test_datasets=test_datasets, classes_per_task=classes_per_task, precision_dict=precision_dict_exemplars, iters_per_task=args.iters, with_exemplars=True, ) if args.use_exemplars else None # -collect them in <lists> eval_cbs = [eval_cb, eval_cb_full] eval_cbs_exemplars = [eval_cb_exemplars] # --------------------# # ----- TRAINING -----# # --------------------# print("--> Training:") # Keep track of training-time start = time.time() # Train model train_cl( model, train_datasets, test_datasets, replay_mode=args.replay, classes_per_task=classes_per_task, iters=args.iters, batch_size=args.batch, savepath=savepath, generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs, sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs, eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars, ) # -------------------------------------------------------------------------------------------------# # --------------------# # -- VISUALIZATION ---# # --------------------# matrices_names = args.matrices method_names = [] if args.cul == 1: method_names.append('Cumulative') elif args.cul == 0: method_names.append('Naive') if args.replay == 'current': method_names.append('LwF') if args.online and args.ewc: method_names.append('Online EWC') if args.si: method_names.append('SI') if args.replay == "generative" and not args.feedback and not args.distill: method_names.append('DGR') if args.replay == "generative" and not args.feedback and args.distill: method_names.append('DGR with distillation') if args.replay == "generative" and args.feedback and args.distill: method_names.append('DGR with feedback') if args.ewc and not args.online: method_names.append('EWC') print('The selected methods are:', method_names) print('The selected performance matrices are:', matrices_names) if args.cross_methods: print('==> Drawing results for cross selected-methods ... <==') if 'spider' in args.cross_methods_type: spider = True if 'bar' in args.cross_methods_type: bar = True if args.cross_tasks: print('==> Drawing results for cross tasks ... <==')