def save_results(args, result_val_t, result_val_a, result_test_t, result_test_a, model, spent_time): fname = os.path.join(args.log_dir, 'results') # save confusion matrix and print one line of stats val_stats = confusion_matrix(result_val_t, result_val_a, args.log_dir, 'results.txt') one_liner = str(vars(args)) + ' # val: ' one_liner += ' '.join(["%.3f" % stat for stat in val_stats]) test_stats = 0 if args.calc_test_accuracy: test_stats = confusion_matrix(result_test_t, result_test_a, args.log_dir, 'results.txt') one_liner += ' # test: ' + ' '.join(["%.3f" % stat for stat in test_stats]) print(fname + ': ' + one_liner + ' # ' + str(spent_time)) # save all results in binary file torch.save((result_val_t, result_val_a, model.state_dict(), val_stats, one_liner, args), fname + '.pt') return val_stats, test_stats
# set up continuum continuum = Continuum(x_tr, batch_size, shuffle_tasks, samples_per_task, epochs_per_task) # load model Model = importlib.import_module('model.' + model_name) model = Model.Net(n_inputs, n_outputs, n_tasks, num_hidden_layers, num_hidden_neurons, memory_strength, learning_rate, memories_per_task, cuda_enabled) if cuda_enabled: model.cuda() # run model on continuum result_t, result_a, spent_time = life_experience(model, continuum, x_te, log_frequency, cuda_enabled) # prepare saving path and file name if not os.path.exists(save_path): os.makedirs(save_path) fname = model_name + '_' + data_file + '_' fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") fname = os.path.join(save_path, fname) # save confusion matrix and print one line of stats stats = confusion_matrix(result_t, result_a, fname + '.txt') one_liner = ' '.join(["%.3f" % stat for stat in stats]) print("{} {}".format(model_name, "MNIST Rotation") + ': ' + one_liner + ' # Spent time:' + str(spent_time))
args.n_iter) + '_slack' + str( args.slack) + '_normalize' + str( args.normalize) + '_' + args.data_file + '_' model.fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") model.fname += '_' + uid model.fname = os.path.join(args.save_path, model.fname) if args.cuda: model.cuda() if args.shared_head: model.is_cifar = False model.nc_per_task = n_outputs # run model on continuum result_t, result_a, avg_accuracy, current_res_per_t, current_avg, accurcy_on_mem, spent_time = life_experience( model, continuum, x_te, args) # prepare saving path and file name if not os.path.exists(args.save_path): os.makedirs(args.save_path) # save confusion matrix and print one line of stats stats = confusion_matrix(result_t, result_a, avg_accuracy, accurcy_on_mem, args.tasks_to_preserve, model.fname + '.txt') one_liner = str(vars(args)) + ' # ' one_liner += ' '.join(["%.3f" % stat for stat in stats]) print(model.fname + ': ' + one_liner + ' # ' + str(spent_time)) # save all results in binary file torch.save((result_t, result_a, model.state_dict(), current_res_per_t, current_avg, stats, one_liner, args), model.fname + '.pt')
def main(overwrite_args=None): args = parser.parse_args() if overwrite_args is not None: for k, v in overwrite_args.items(): # Debugging setattr(args, k, v) args.dyn_mem = True if args.dyn_mem == 'yes' else False args.cuda = True if args.cuda == 'yes' else False args.finetune = True if args.finetune == 'yes' else False args.normalize = True if args.normalize == 'yes' else False args.shared_head = True if args.shared_head == 'yes' else False args.iid = True if args.iid == 'yes' else False if args.mini_batch_size == 0: args.mini_batch_size = args.batch_size # no mini iterations # unique identifier uid = uuid.uuid4().hex if args.uid is None else args.uid now = str(datetime.datetime.now().date()) + "_" + ':'.join( str(datetime.datetime.now().time()).split(':')[:-1]) runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume # Paths setupname = [args.exp_name, args.model, args.data_file.split('.')[0]] parentdir = os.path.join(args.save_path, '_'.join(setupname)) print("Init args={}".format(args)) stat_files = [] seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds)) for seed in seeds: # initialize seeds print("STARTING SEED {}/{}".format(seed, args.n_seeds - 1)) torch.backends.cudnn.deterministic = False torch.backends.cudnn.enabled = False torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) if args.cuda: torch.cuda.manual_seed_all(seed) # load data x_tr, x_te, n_inputs, n_classes, n_tasks = load_datasets(args) args.is_cifar = ('cifar10' in args.data_file) args.is_mnist = ('mnist' in args.data_file) assert not (args.is_cifar and args.is_mnist) args.input_shape = x_tr[0][1][0].shape if args.input_shape[-1] == 3072: # CIFAR assert args.is_cifar args.CHW = (3, 32, 32) elif args.input_shape[-1] == 784: # MNIST assert args.is_mnist args.CHW = (1, 28, 28) else: raise NotImplementedError() args.n_classes = n_classes n_outputs = args.n_classes if args.n_outputs is None else args.n_outputs # Embedding or Softmax # set up continuum continuum = Continuum(x_tr, args) # load model args.tracker = ResultTracker() args.net = get_model(args, n_inputs, n_outputs) Model = importlib.import_module('model.' + args.model) model = Model.Net(n_inputs, n_outputs, n_tasks, args) # set up file name for saving/chkpt if args.n_sampled_memories == 0: args.n_sampled_memories = args.n_memories if args.output_name: model.fname = args.output_name model.fname = os.path.join(parentdir, runname, 'seed={}'.format(seed)) args.imgname = os.path.join( './img', '_'.join(setupname), '{}_{}/'.format(runname, 'seed={}'.format(seed))) if os.path.isfile(model.fname + '.pt'): print( "[CHECKPOINT] Loading seed checkpoint: {}".format(model.fname + '.pt')) chkpt = torch.load(model.fname + '.pt') if hasattr(chkpt[-1], 'output_name'): # See if is args object args = chkpt[-1] stat_files.append(model.fname + '.pt') # For final accs print("Args overwritten by chkpt: {}".format(args)) continue print("Checkpoint not restored, continuing with args: {}".format( args)) createdirs(model.fname) createdirs(args.imgname) # prepare saving path and file name if not os.path.exists(args.save_path): os.makedirs(args.save_path) if args.cuda: model.cuda() # run model on continuum res, acc_on_mem, spent_time = life_experience(model, continuum, x_te, args) # save confusion matrix and print one line of stats stats = confusion_matrix(res.task_idxs, res.tot_res_seqs, res.tot_avg_accs, acc_on_mem, args.tasks_to_preserve, model.fname + '.txt') one_liner = str(vars(args)) + ' # ' one_liner += ' '.join(["%.3f" % stat for stat in stats]) print(model.fname + ': ' + one_liner + ' # ' + str(spent_time)) # save all results in binary file torch.save( (*res.get_all(), model.state_dict(), stats, one_liner, args), model.fname + '.pt') stat_files.append(model.fname + '.pt') mean, std = stat_summarize(stat_files) print("FINISHED SCRIPT")
model = Model.Net(n_inputs, n_outputs, n_tasks, args) if args.cuda: try: model.cuda() except: pass # run model on continuum result_t, result_a, model_capacities, training_times, spent_time = life_experience( model, continuum, x_te, args) # prepare saving path and file name if not os.path.exists(args.save_path): os.makedirs(args.save_path) fname = args.model + '_' + args.data_file + '_' fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") # fname += '_' + uid fname = os.path.join(args.save_path, fname) # save confusion matrix and print one line of stats stats = confusion_matrix(result_t, result_a, model_capacities, training_times, fname + '.txt') one_liner = str(vars(args)) + ' # ' one_liner += ' '.join(["%.3f" % stat for stat in stats]) print(fname + ': ' + one_liner + ' # ' + str(spent_time)) # save all results in binary file torch.save((result_t, result_a, model_capacities, training_times, stats, one_liner, args), fname + '.pt')
use=args.use) for epoch in range(args.n_epochs): losses = 0. lca_counter = 0. for x, y, idx in tqdm(taskLoader, ncols=69, desc=desc): model.train() if str(args.model) in ['lwf']: info = [current_task] info.append(idx) loss = model.observe( Variable(x).cuda(), info, Variable(y).cuda()) losses += loss lca_counter += 1 model.on_epoch_end() print('Task loss: {:.3f}'.format(losses / len(taskLoader))) result_a.append(eval_tasks(model, x_te, args)) result_t.append(current_task) #result_a.append(eval_tasks(model, x_te)) #result_t.append(current_task) time_spent = time.time() - start_time if args.lca < 0: lca = [0] stats = confusion_matrix(torch.Tensor(result_t), torch.Tensor(result_a), torch.Tensor(lca), fname + '.txt') one_liner = str(vars(args)) + ' # ' one_liner += ' '.join(["%.3f" % stat for stat in stats]) print(fname + ':' + one_liner + ' # ' + str(time_spent)) #model.on_train_end(stats[0].item(), stats[1].item())