def save_results(args, result_val_t, result_val_a, result_test_t, result_test_a, model, spent_time):
fname = os.path.join(args.log_dir, 'results')
# save confusion matrix and print one line of stats
val_stats = confusion_matrix(result_val_t, result_val_a, args.log_dir, 'results.txt')
one_liner = str(vars(args)) + ' # val: '
one_liner += ' '.join(["%.3f" % stat for stat in val_stats])
test_stats = 0
if args.calc_test_accuracy:
test_stats = confusion_matrix(result_test_t, result_test_a, args.log_dir, 'results.txt')
one_liner += ' # test: ' + ' '.join(["%.3f" % stat for stat in test_stats])
print(fname + ': ' + one_liner + ' # ' + str(spent_time))
# save all results in binary file
torch.save((result_val_t, result_val_a, model.state_dict(),
val_stats, one_liner, args), fname + '.pt')
return val_stats, test_stats
# set up continuum
continuum = Continuum(x_tr, batch_size, shuffle_tasks, samples_per_task,
epochs_per_task)
# load model
Model = importlib.import_module('model.' + model_name)
model = Model.Net(n_inputs, n_outputs, n_tasks, num_hidden_layers,
num_hidden_neurons, memory_strength, learning_rate,
memories_per_task, cuda_enabled)
if cuda_enabled:
model.cuda()
# run model on continuum
result_t, result_a, spent_time = life_experience(model, continuum, x_te,
log_frequency,
cuda_enabled)
# prepare saving path and file name
if not os.path.exists(save_path):
os.makedirs(save_path)
fname = model_name + '_' + data_file + '_'
fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
fname = os.path.join(save_path, fname)
# save confusion matrix and print one line of stats
stats = confusion_matrix(result_t, result_a, fname + '.txt')
one_liner = ' '.join(["%.3f" % stat for stat in stats])
print("{} {}".format(model_name, "MNIST Rotation") + ': ' + one_liner +
' # Spent time:' + str(spent_time))
args.n_iter) + '_slack' + str(
args.slack) + '_normalize' + str(
args.normalize) + '_' + args.data_file + '_'
model.fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
model.fname += '_' + uid
model.fname = os.path.join(args.save_path, model.fname)
if args.cuda:
model.cuda()
if args.shared_head:
model.is_cifar = False
model.nc_per_task = n_outputs
# run model on continuum
result_t, result_a, avg_accuracy, current_res_per_t, current_avg, accurcy_on_mem, spent_time = life_experience(
model, continuum, x_te, args)
# prepare saving path and file name
if not os.path.exists(args.save_path):
os.makedirs(args.save_path)
# save confusion matrix and print one line of stats
stats = confusion_matrix(result_t, result_a, avg_accuracy, accurcy_on_mem,
args.tasks_to_preserve, model.fname + '.txt')
one_liner = str(vars(args)) + ' # '
one_liner += ' '.join(["%.3f" % stat for stat in stats])
print(model.fname + ': ' + one_liner + ' # ' + str(spent_time))
# save all results in binary file
torch.save((result_t, result_a, model.state_dict(), current_res_per_t,
current_avg, stats, one_liner, args), model.fname + '.pt')
def main(overwrite_args=None):
args = parser.parse_args()
if overwrite_args is not None:
for k, v in overwrite_args.items(): # Debugging
setattr(args, k, v)
args.dyn_mem = True if args.dyn_mem == 'yes' else False
args.cuda = True if args.cuda == 'yes' else False
args.finetune = True if args.finetune == 'yes' else False
args.normalize = True if args.normalize == 'yes' else False
args.shared_head = True if args.shared_head == 'yes' else False
args.iid = True if args.iid == 'yes' else False
if args.mini_batch_size == 0:
args.mini_batch_size = args.batch_size # no mini iterations
# unique identifier
uid = uuid.uuid4().hex if args.uid is None else args.uid
now = str(datetime.datetime.now().date()) + "_" + ':'.join(
str(datetime.datetime.now().time()).split(':')[:-1])
runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume
# Paths
setupname = [args.exp_name, args.model, args.data_file.split('.')[0]]
parentdir = os.path.join(args.save_path, '_'.join(setupname))
print("Init args={}".format(args))
stat_files = []
seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds))
for seed in seeds:
# initialize seeds
print("STARTING SEED {}/{}".format(seed, args.n_seeds - 1))
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.enabled = False
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
if args.cuda:
torch.cuda.manual_seed_all(seed)
# load data
x_tr, x_te, n_inputs, n_classes, n_tasks = load_datasets(args)
args.is_cifar = ('cifar10' in args.data_file)
args.is_mnist = ('mnist' in args.data_file)
assert not (args.is_cifar and args.is_mnist)
args.input_shape = x_tr[0][1][0].shape
if args.input_shape[-1] == 3072: # CIFAR
assert args.is_cifar
args.CHW = (3, 32, 32)
elif args.input_shape[-1] == 784: # MNIST
assert args.is_mnist
args.CHW = (1, 28, 28)
else:
raise NotImplementedError()
args.n_classes = n_classes
n_outputs = args.n_classes if args.n_outputs is None else args.n_outputs # Embedding or Softmax
# set up continuum
continuum = Continuum(x_tr, args)
# load model
args.tracker = ResultTracker()
args.net = get_model(args, n_inputs, n_outputs)
Model = importlib.import_module('model.' + args.model)
model = Model.Net(n_inputs, n_outputs, n_tasks, args)
# set up file name for saving/chkpt
if args.n_sampled_memories == 0:
args.n_sampled_memories = args.n_memories
if args.output_name:
model.fname = args.output_name
model.fname = os.path.join(parentdir, runname, 'seed={}'.format(seed))
args.imgname = os.path.join(
'./img', '_'.join(setupname),
'{}_{}/'.format(runname, 'seed={}'.format(seed)))
if os.path.isfile(model.fname + '.pt'):
print(
"[CHECKPOINT] Loading seed checkpoint: {}".format(model.fname +
'.pt'))
chkpt = torch.load(model.fname + '.pt')
if hasattr(chkpt[-1], 'output_name'): # See if is args object
args = chkpt[-1]
stat_files.append(model.fname + '.pt') # For final accs
print("Args overwritten by chkpt: {}".format(args))
continue
print("Checkpoint not restored, continuing with args: {}".format(
args))
createdirs(model.fname)
createdirs(args.imgname)
# prepare saving path and file name
if not os.path.exists(args.save_path):
os.makedirs(args.save_path)
if args.cuda:
model.cuda()
# run model on continuum
res, acc_on_mem, spent_time = life_experience(model, continuum, x_te,
args)
# save confusion matrix and print one line of stats
stats = confusion_matrix(res.task_idxs, res.tot_res_seqs,
res.tot_avg_accs, acc_on_mem,
args.tasks_to_preserve, model.fname + '.txt')
one_liner = str(vars(args)) + ' # '
one_liner += ' '.join(["%.3f" % stat for stat in stats])
print(model.fname + ': ' + one_liner + ' # ' + str(spent_time))
# save all results in binary file
torch.save(
(*res.get_all(), model.state_dict(), stats, one_liner, args),
model.fname + '.pt')
stat_files.append(model.fname + '.pt')
mean, std = stat_summarize(stat_files)
print("FINISHED SCRIPT")
model = Model.Net(n_inputs, n_outputs, n_tasks, args)
if args.cuda:
try:
model.cuda()
except:
pass
# run model on continuum
result_t, result_a, model_capacities, training_times, spent_time = life_experience(
model, continuum, x_te, args)
# prepare saving path and file name
if not os.path.exists(args.save_path):
os.makedirs(args.save_path)
fname = args.model + '_' + args.data_file + '_'
fname += datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
# fname += '_' + uid
fname = os.path.join(args.save_path, fname)
# save confusion matrix and print one line of stats
stats = confusion_matrix(result_t, result_a, model_capacities,
training_times, fname + '.txt')
one_liner = str(vars(args)) + ' # '
one_liner += ' '.join(["%.3f" % stat for stat in stats])
print(fname + ': ' + one_liner + ' # ' + str(spent_time))
# save all results in binary file
torch.save((result_t, result_a, model_capacities, training_times, stats,
one_liner, args), fname + '.pt')
use=args.use)
for epoch in range(args.n_epochs):
losses = 0.
lca_counter = 0.
for x, y, idx in tqdm(taskLoader, ncols=69, desc=desc):
model.train()
if str(args.model) in ['lwf']:
info = [current_task]
info.append(idx)
loss = model.observe(
Variable(x).cuda(), info,
Variable(y).cuda())
losses += loss
lca_counter += 1
model.on_epoch_end()
print('Task loss: {:.3f}'.format(losses / len(taskLoader)))
result_a.append(eval_tasks(model, x_te, args))
result_t.append(current_task)
#result_a.append(eval_tasks(model, x_te))
#result_t.append(current_task)
time_spent = time.time() - start_time
if args.lca < 0:
lca = [0]
stats = confusion_matrix(torch.Tensor(result_t), torch.Tensor(result_a),
torch.Tensor(lca), fname + '.txt')
one_liner = str(vars(args)) + ' # '
one_liner += ' '.join(["%.3f" % stat for stat in stats])
print(fname + ':' + one_liner + ' # ' + str(time_spent))
#model.on_train_end(stats[0].item(), stats[1].item())
