def run(args):
result_path = os.path.join('./benchmarks/results', args.savepath)
savepath = result_path + '/' + str(
datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv'
if not os.path.exists(result_path):
print('no exist the path and create one ...')
os.makedirs(result_path, exist_ok=True)
# Set default arguments
args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen
args.g_iters = args.iters if args.g_iters is None else args.g_iters
args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay
args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni
# -if [log_per_task], reset all logs
if args.log_per_task:
args.prec_log = args.iters
args.loss_log = args.iters
args.sample_log = args.iters
# -if [iCaRL] is selected, select all accompanying options
if hasattr(args, "icarl") and args.icarl:
args.use_exemplars = True
args.add_exemplars = True
# -if EWC or SI is selected together with 'feedback', give error
if args.feedback and (args.ewc or args.si or args.icarl):
raise NotImplementedError(
"EWC, SI and iCaRL are not supported with feedback connections.")
# -if binary classification loss is selected together with 'feedback', give error
if args.feedback and args.bce:
raise NotImplementedError(
"Binary classification loss not supported with feedback connections."
)
if not os.path.isdir(RESULT_DIR):
os.mkdir(RESULT_DIR)
# If only want param-stamp, get it printed to screen and exit
if hasattr(args, "get_stamp") and args.get_stamp:
_ = get_param_stamp_from_args(args=args)
exit()
# Use cuda?
cuda = torch.cuda.is_available() and args.cuda
device = "cuda" if cuda else "cpu"
gpu_devices = None
if args.gpuID == None:
if torch.cuda.device_count() > 1:
gpu_devices = ','.join(
[str(id) for id in range(torch.cuda.device_count())])
print('==> training with CUDA (GPU id: ' + gpu_devices +
') ... <==')
else:
gpu_devices = ','.join([str(id) for id in args.gpuID])
os.environ['CUDA_VISIBLE_DEVICES'] = gpu_devices
print('==> training with CUDA (GPU id: ' + str(args.gpuID) +
') ... <==')
# Set random seeds
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed(args.seed)
if args.factor == 'sequence':
args.tasks = 12
# -------------------------------------------------------------------------------------------------#
# ----------------#
# ----- DATA -----#
# ----------------#
# Prepare data for OpenLORIS-Object
if args.dataset == 'OpenLORIS-Object':
with open('./benchmarks/data/OpenLORIS-Object/' + args.factor + '.pk',
'rb') as f:
((train_datasets, test_datasets), config,
classes_per_task) = pickle.load(f)
else:
with open(
'./benchmarks/data/' + args.dataset + '/' + args.dataset +
'.pk', 'rb') as f:
((train_datasets, test_datasets), config,
classes_per_task) = pickle.load(f)
if args.cul == 1:
for i in range(1, len(train_datasets)):
train_datasets[i].imgs.extend(train_datasets[i - 1].imgs)
train_datasets[i].labels.extend(train_datasets[i - 1].labels)
# -------------------------------------------------------------------------------------------------#
# ------------------------------#
# ----- MODEL (CLASSIFIER) -----#
# ------------------------------#
# Define main model (i.e., classifier, if requested with feedback connections)
if args.feedback:
model = AutoEncoder(
image_size=config['size'],
image_channels=config['channels'],
classes=config['classes'],
fc_layers=args.fc_lay,
fc_units=args.g_fc_uni,
z_dim=args.z_dim,
fc_drop=args.fc_drop,
fc_bn=True if args.fc_bn == "yes" else False,
fc_nl=args.fc_nl,
).to(device)
model.lamda_pl = 1. # --> to make that this VAE is also trained to classify
else:
model = Classifier(image_size=config['size'],
image_channels=config['channels'],
classes=config['classes'],
fc_layers=args.fc_lay,
fc_units=args.fc_units,
fc_drop=args.fc_drop,
fc_nl=args.fc_nl,
fc_bn=True if args.fc_bn == "yes" else False,
excit_buffer=False,
binaryCE=args.bce).to(device)
# Define optimizer (only include parameters that "requires_grad")
model.optim_list = [{
'params':
filter(lambda p: p.requires_grad, model.parameters()),
'lr':
args.lr
}]
model.optim_type = args.optimizer
if model.optim_type in ("adam", "adam_reset"):
model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999))
elif model.optim_type == "sgd":
model.optimizer = optim.SGD(model.optim_list)
else:
raise ValueError(
"Unrecognized optimizer, '{}' is not currently a valid option".
format(args.optimizer))
# ----------------------------------#
# ----- CL-STRATEGY: EXEMPLARS -----#
# ----------------------------------#
# Store in model whether, how many and in what way to store exemplars
if isinstance(model, ExemplarHandler) and (args.use_exemplars
or args.add_exemplars
or args.replay == "exemplars"):
model.memory_budget = args.budget
model.norm_exemplars = args.norm_exemplars
model.herding = args.herding
# -----------------------------------#
# ----- CL-STRATEGY: ALLOCATION -----#
# -----------------------------------#
# Elastic Weight Consolidation (EWC)
if isinstance(model, ContinualLearner):
model.ewc_lambda = args.ewc_lambda if args.ewc else 0
if args.ewc:
model.fisher_n = args.fisher_n
model.gamma = args.gamma
model.online = args.online
model.emp_FI = args.emp_fi
# Synpatic Intelligence (SI)
if isinstance(model, ContinualLearner):
model.si_c = args.si_c if args.si else 0
if args.si:
model.epsilon = args.epsilon
# -------------------------------------------------------------------------------------------------#
# -------------------------------#
# ----- CL-STRATEGY: REPLAY -----#
# -------------------------------#
# Use distillation loss (i.e., soft targets) for replayed data? (and set temperature)
if isinstance(model, Replayer):
model.replay_targets = "soft" if args.distill else "hard"
model.KD_temp = args.temp
# If needed, specify separate model for the generator
train_gen = True if (args.replay == "generative"
and not args.feedback) else False
if train_gen:
# -specify architecture
generator = AutoEncoder(
image_size=config['size'],
image_channels=config['channels'],
fc_layers=args.g_fc_lay,
fc_units=args.g_fc_uni,
z_dim=args.z_dim,
classes=config['classes'],
fc_drop=args.fc_drop,
fc_bn=True if args.fc_bn == "yes" else False,
fc_nl=args.fc_nl,
).to(device)
# -set optimizer(s)
generator.optim_list = [{
'params':
filter(lambda p: p.requires_grad, generator.parameters()),
'lr':
args.lr_gen
}]
generator.optim_type = args.optimizer
if generator.optim_type in ("adam", "adam_reset"):
generator.optimizer = optim.Adam(generator.optim_list,
betas=(0.9, 0.999))
elif generator.optim_type == "sgd":
generator.optimizer = optim.SGD(generator.optim_list)
else:
generator = None
# ---------------------#
# ----- REPORTING -----#
# ---------------------#
# Get parameter-stamp (and print on screen)
param_stamp = get_param_stamp(
args,
model.name,
verbose=True,
replay=True if (not args.replay == "none") else False,
replay_model_name=generator.name if
(args.replay == "generative" and not args.feedback) else None,
)
# -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf
precision_dict = evaluate.initiate_precision_dict(args.tasks)
precision_dict_exemplars = evaluate.initiate_precision_dict(
args.tasks) if args.use_exemplars else None
# ---------------------#
# ----- CALLBACKS -----#
# ---------------------#
# Callbacks for reporting on and visualizing loss
generator_loss_cbs = [
cb._VAE_loss_cb(
log=args.loss_log,
model=model if args.feedback else generator,
tasks=args.tasks,
iters_per_task=args.iters if args.feedback else args.g_iters,
replay=False if args.replay == "none" else True)
] if (train_gen or args.feedback) else [None]
solver_loss_cbs = [
cb._solver_loss_cb(log=args.loss_log,
model=model,
tasks=args.tasks,
iters_per_task=args.iters,
replay=False if args.replay == "none" else True)
] if (not args.feedback) else [None]
# Callbacks for evaluating and plotting generated / reconstructed samples
sample_cbs = [
cb._sample_cb(
log=args.sample_log,
config=config,
test_datasets=test_datasets,
sample_size=args.sample_n,
iters_per_task=args.iters if args.feedback else args.g_iters)
] if (train_gen or args.feedback) else [None]
# Callbacks for reporting and visualizing accuracy
eval_cb = cb._eval_cb(log=args.prec_log,
test_datasets=test_datasets,
precision_dict=None,
iters_per_task=args.iters,
test_size=args.prec_n,
classes_per_task=classes_per_task)
# -pdf / reporting: summary plots (i.e, only after each task)
eval_cb_full = cb._eval_cb(log=args.iters,
test_datasets=test_datasets,
precision_dict=precision_dict,
iters_per_task=args.iters,
classes_per_task=classes_per_task)
eval_cb_exemplars = cb._eval_cb(
log=args.iters,
test_datasets=test_datasets,
classes_per_task=classes_per_task,
precision_dict=precision_dict_exemplars,
iters_per_task=args.iters,
with_exemplars=True,
) if args.use_exemplars else None
# -collect them in <lists>
eval_cbs = [eval_cb, eval_cb_full]
eval_cbs_exemplars = [eval_cb_exemplars]
# --------------------#
# ----- TRAINING -----#
# --------------------#
print("--> Training:")
# Keep track of training-time
start = time.time()
# Train model
train_cl(
model,
train_datasets,
test_datasets,
replay_mode=args.replay,
classes_per_task=classes_per_task,
iters=args.iters,
batch_size=args.batch,
savepath=savepath,
generator=generator,
gen_iters=args.g_iters,
gen_loss_cbs=generator_loss_cbs,
sample_cbs=sample_cbs,
eval_cbs=eval_cbs,
loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs,
eval_cbs_exemplars=eval_cbs_exemplars,
use_exemplars=args.use_exemplars,
add_exemplars=args.add_exemplars,
)
# -------------------------------------------------------------------------------------------------#
# --------------------#
# -- VISUALIZATION ---#
# --------------------#
matrices_names = args.matrices
method_names = []
if args.cul == 1:
method_names.append('Cumulative')
elif args.cul == 0:
method_names.append('Naive')
if args.replay == 'current':
method_names.append('LwF')
if args.online and args.ewc:
method_names.append('Online EWC')
if args.si:
method_names.append('SI')
if args.replay == "generative" and not args.feedback and not args.distill:
method_names.append('DGR')
if args.replay == "generative" and not args.feedback and args.distill:
method_names.append('DGR with distillation')
if args.replay == "generative" and args.feedback and args.distill:
method_names.append('DGR with feedback')
if args.ewc and not args.online:
method_names.append('EWC')
print('The selected methods are:', method_names)
print('The selected performance matrices are:', matrices_names)
if args.cross_methods:
print('==> Drawing results for cross selected-methods ... <==')
if 'spider' in args.cross_methods_type:
spider = True
if 'bar' in args.cross_methods_type:
bar = True
if args.cross_tasks:
print('==> Drawing results for cross tasks ... <==')
def run(args):
result_path = os.path.join('./precision_onEachTask', args.savepath)
savepath = result_path + '/' + str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv'
os.makedirs(result_path, exist_ok=True)
# Set default arguments
args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen
args.g_iters = args.iters if args.g_iters is None else args.g_iters
args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay
args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni
# -if [log_per_task], reset all logs
if args.log_per_task:
args.prec_log = args.iters
args.loss_log = args.iters
args.sample_log = args.iters
# -if [iCaRL] is selected, select all accompanying options
if hasattr(args, "icarl") and args.icarl:
args.use_exemplars = True
args.add_exemplars = True
# -if EWC, SI or XdG is selected together with 'feedback', give error
if args.feedback and (args.ewc or args.si or args.gating_prop > 0 or args.icarl):
raise NotImplementedError("EWC, SI, XdG and iCaRL are not supported with feedback connections.")
# -if binary classification loss is selected together with 'feedback', give error
if args.feedback and args.bce:
raise NotImplementedError("Binary classification loss not supported with feedback connections.")
# -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine)
if args.gating_prop > 0 and (not args.replay == "none") and (args.ewc or args.si):
raise NotImplementedError("XdG is not supported with both '{}' replay and EWC / SI.".format(args.replay))
# --> problem is that applying different task-masks interferes with gradient calculation
# (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately)
# -create plots- and results-directories if needed
if not os.path.isdir(RESULT_DIR):
os.mkdir(RESULT_DIR)
scenario = SCENARIO
# (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL)
# If only want param-stamp, get it printed to screen and exit
if hasattr(args, "get_stamp") and args.get_stamp:
_ = get_param_stamp_from_args(args=args)
exit()
# Use cuda?
cuda = torch.cuda.is_available() and args.cuda
device = torch.device("cuda" if cuda else "cpu")
# Set random seeds
np.random.seed(SEED)
torch.manual_seed(SEED)
if cuda:
torch.cuda.manual_seed(SEED)
if args.factor == 'sequence':
args.tasks = 12
# -------------------------------------------------------------------------------------------------#
# ----------------#
# ----- DATA -----#
# ----------------#
# Prepare data for chosen experiment
with open(args.factor + '.pk', 'rb') as f:
((train_datasets, test_datasets), config, classes_per_task) = pickle.load(f)
if args.cul == 1:
for i in range(1, len(train_datasets)):
train_datasets[i].imgs.extend(train_datasets[i - 1].imgs)
train_datasets[i].labels.extend(train_datasets[i - 1].labels)
# -------------------------------------------------------------------------------------------------#
# ------------------------------#
# ----- MODEL (CLASSIFIER) -----#
# ------------------------------#
# Define main model (i.e., classifier, if requested with feedback connections)
if args.feedback:
model = AutoEncoder(
image_size=config['size'], image_channels=config['channels'], classes=config['classes'],
fc_layers=args.fc_lay, fc_units=args.g_fc_uni, z_dim=Z_DIM,
fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl,
).to(device)
model.lamda_pl = 1. # --> to make that this VAE is also trained to classify
else:
model = Classifier(
image_size=config['size'], image_channels=config['channels'], classes=config['classes'],
fc_layers=args.fc_lay, fc_units=args.fc_units, fc_drop=args.fc_drop, fc_nl=args.fc_nl,
fc_bn=True if args.fc_bn == "yes" else False, excit_buffer=True if args.gating_prop > 0 else False,
binaryCE=args.bce, binaryCE_distill=True,
).to(device)
# Define optimizer (only include parameters that "requires_grad")
model.optim_list = [{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr}]
model.optim_type = args.optimizer
if model.optim_type in ("adam", "adam_reset"):
model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999))
elif model.optim_type == "sgd":
model.optimizer = optim.SGD(model.optim_list)
else:
raise ValueError("Unrecognized optimizer, '{}' is not currently a valid option".format(args.optimizer))
# ----------------------------------#
# ----- CL-STRATEGY: EXEMPLARS -----#
# ----------------------------------#
# Store in model whether, how many and in what way to store exemplars
if isinstance(model, ExemplarHandler) and (args.use_exemplars or args.add_exemplars or args.replay == "exemplars"):
model.memory_budget = args.budget
model.norm_exemplars = args.norm_exemplars
model.herding = args.herding
# -----------------------------------#
# ----- CL-STRATEGY: ALLOCATION -----#
# -----------------------------------#
# Elastic Weight Consolidation (EWC)
if isinstance(model, ContinualLearner):
model.ewc_lambda = args.ewc_lambda if args.ewc else 0
if args.ewc:
model.fisher_n = args.fisher_n
model.gamma = args.gamma
model.online = args.online
model.emp_FI = args.emp_fi
# Synpatic Intelligence (SI)
if isinstance(model, ContinualLearner):
model.si_c = args.si_c if args.si else 0
if args.si:
model.epsilon = args.epsilon
# XdG: create for every task a "mask" for each hidden fully connected layer
if isinstance(model, ContinualLearner) and args.gating_prop > 0:
mask_dict = {}
excit_buffer_list = []
for task_id in range(args.tasks):
mask_dict[task_id + 1] = {}
for i in range(model.fcE.layers):
layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear
if task_id == 0:
excit_buffer_list.append(layer.excit_buffer)
n_units = len(layer.excit_buffer)
gated_units = np.random.choice(n_units, size=int(args.gating_prop * n_units), replace=False)
mask_dict[task_id + 1][i] = gated_units
model.mask_dict = mask_dict
model.excit_buffer_list = excit_buffer_list
# -------------------------------------------------------------------------------------------------#
# -------------------------------#
# ----- CL-STRATEGY: REPLAY -----#
# -------------------------------#
# Use distillation loss (i.e., soft targets) for replayed data? (and set temperature)
if isinstance(model, Replayer):
model.replay_targets = "soft" if args.distill else "hard"
model.KD_temp = args.temp
# If needed, specify separate model for the generator
train_gen = True if (args.replay == "generative" and not args.feedback) else False
if train_gen:
# -specify architecture
generator = AutoEncoder(
image_size=config['size'], image_channels=config['channels'],
fc_layers=args.g_fc_lay, fc_units=args.g_fc_uni, z_dim=100, classes=config['classes'],
fc_drop=args.fc_drop, fc_bn=True if args.fc_bn == "yes" else False, fc_nl=args.fc_nl,
).to(device)
# -set optimizer(s)
generator.optim_list = [
{'params': filter(lambda p: p.requires_grad, generator.parameters()), 'lr': args.lr_gen}]
generator.optim_type = args.optimizer
if generator.optim_type in ("adam", "adam_reset"):
generator.optimizer = optim.Adam(generator.optim_list, betas=(0.9, 0.999))
elif generator.optim_type == "sgd":
generator.optimizer = optim.SGD(generator.optim_list)
else:
generator = None
# ---------------------#
# ----- REPORTING -----#
# ---------------------#
# Get parameter-stamp (and print on screen)
param_stamp = get_param_stamp(
args, model.name, verbose=True, replay=True if (not args.replay == "none") else False,
replay_model_name=generator.name if (args.replay == "generative" and not args.feedback) else None,
)
# Prepare for plotting in visdom
# -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf
precision_dict = evaluate.initiate_precision_dict(args.tasks)
precision_dict_exemplars = evaluate.initiate_precision_dict(args.tasks) if args.use_exemplars else None
# ---------------------#
# ----- CALLBACKS -----#
# ---------------------#
# Callbacks for reporting on and visualizing loss
generator_loss_cbs = [
cb._VAE_loss_cb(log=args.loss_log, visdom=VISDOM, model=model if args.feedback else generator, tasks=args.tasks,
iters_per_task=args.iters if args.feedback else args.g_iters,
replay=False if args.replay == "none" else True)
] if (train_gen or args.feedback) else [None]
solver_loss_cbs = [
cb._solver_loss_cb(log=args.loss_log, visdom=VISDOM, model=model, tasks=args.tasks,
iters_per_task=args.iters, replay=False if args.replay == "none" else True)
] if (not args.feedback) else [None]
# Callbacks for evaluating and plotting generated / reconstructed samples
sample_cbs = [
cb._sample_cb(log=args.sample_log, visdom=VISDOM, config=config, test_datasets=test_datasets,
sample_size=args.sample_n, iters_per_task=args.iters if args.feedback else args.g_iters)
] if (train_gen or args.feedback) else [None]
# Callbacks for reporting and visualizing accuracy
# -visdom (i.e., after each [prec_log]
eval_cb = cb._eval_cb(
log=args.prec_log, test_datasets=test_datasets, visdom=VISDOM, precision_dict=None, iters_per_task=args.iters,
test_size=args.prec_n, classes_per_task=classes_per_task, scenario=SCENARIO,
)
# -pdf / reporting: summary plots (i.e, only after each task)
eval_cb_full = cb._eval_cb(
log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict,
iters_per_task=args.iters, classes_per_task=classes_per_task, scenario=SCENARIO,
)
# -with exemplars (both for visdom & reporting / pdf)
eval_cb_exemplars = cb._eval_cb(
log=args.iters, test_datasets=test_datasets, visdom=VISDOM_EXEMPLARS, classes_per_task=classes_per_task,
precision_dict=precision_dict_exemplars, scenario=SCENARIO, iters_per_task=args.iters,
with_exemplars=True,
) if args.use_exemplars else None
# -collect them in <lists>
eval_cbs = [eval_cb, eval_cb_full]
eval_cbs_exemplars = [eval_cb_exemplars]
# -------------------------------------------------------------------------------------------------#
# --------------------#
# ----- TRAINING -----#
# --------------------#
print("--> Training:")
# Keep track of training-time
start = time.time()
# Train model
train_cl(
model, train_datasets, test_datasets, replay_mode=args.replay, scenario=SCENARIO,
classes_per_task=classes_per_task,
iters=args.iters, batch_size=args.batch, savepath=savepath,
generator=generator, gen_iters=args.g_iters, gen_loss_cbs=generator_loss_cbs,
sample_cbs=sample_cbs, eval_cbs=eval_cbs, loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs,
eval_cbs_exemplars=eval_cbs_exemplars, use_exemplars=args.use_exemplars, add_exemplars=args.add_exemplars,
)
def run(args):
result_path = os.path.join('./precision_onEachTask', args.scenario,
args.savepath)
savepath = result_path + '/' + str(
datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) + '.csv'
os.makedirs(result_path, exist_ok=True)
# Set default arguments
args.lr_gen = args.lr if args.lr_gen is None else args.lr_gen
args.g_iters = args.iters if args.g_iters is None else args.g_iters
args.g_fc_lay = args.fc_lay if args.g_fc_lay is None else args.g_fc_lay
args.g_fc_uni = args.fc_units if args.g_fc_uni is None else args.g_fc_uni
# -if [log_per_task], reset all logs
if args.log_per_task:
args.prec_log = args.iters
args.loss_log = args.iters
args.sample_log = args.iters
# -if [iCaRL] is selected, select all accompanying options
if hasattr(args, "icarl") and args.icarl:
args.use_exemplars = True
args.add_exemplars = True
args.bce = True
args.bce_distill = True
# -if XdG is selected but not the Task-IL scenario, give error
if (not args.scenario == "task") and args.gating_prop > 0:
raise ValueError("'XdG' is only compatible with the Task-IL scenario.")
# -if EWC, SI or XdG is selected together with 'feedback', give error
if args.feedback and (args.ewc or args.si or args.gating_prop > 0
or args.icarl):
raise NotImplementedError(
"EWC, SI, XdG and iCaRL are not supported with feedback connections."
)
# -if binary classification loss is selected together with 'feedback', give error
if args.feedback and args.bce:
raise NotImplementedError(
"Binary classification loss not supported with feedback connections."
)
# -if XdG is selected together with both replay and EWC, give error (either one of them alone with XdG is fine)
if args.gating_prop > 0 and (not args.replay == "none") and (args.ewc
or args.si):
raise NotImplementedError(
"XdG is not supported with both '{}' replay and EWC / SI.".format(
args.replay))
#--> problem is that applying different task-masks interferes with gradient calculation
# (should be possible to overcome by calculating backward step on EWC/SI-loss also for each mask separately)
# -if 'BCEdistill' is selected for other than scenario=="class", give error
if args.bce_distill and not args.scenario == "class":
raise ValueError(
"BCE-distill can only be used for class-incremental learning.")
# -create plots- and results-directories if needed
if not os.path.isdir(args.r_dir):
os.mkdir(args.r_dir)
if args.pdf and not os.path.isdir(args.p_dir):
os.mkdir(args.p_dir)
scenario = args.scenario
# If Task-IL scenario is chosen with single-headed output layer, set args.scenario to "domain"
# (but note that when XdG is used, task-identity information is being used so the actual scenario is still Task-IL)
if args.singlehead and args.scenario == "task":
scenario = "domain"
# If only want param-stamp, get it printed to screen and exit
if hasattr(args, "get_stamp") and args.get_stamp:
_ = get_param_stamp_from_args(args=args)
exit()
# Use cuda?
cuda = torch.cuda.is_available() and args.cuda
device = torch.device("cuda" if cuda else "cpu")
# Set random seeds
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed(args.seed)
#-------------------------------------------------------------------------------------------------#
#----------------#
#----- DATA -----#
#----------------#
# Prepare data for chosen experiment
'''(train_datasets, test_datasets), config, classes_per_task = get_multitask_experiment(
name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir,
verbose=True, exception=True if args.seed==0 else False,
)
print(train_datasets[0])
with open('mydata.pk','wb') as f:
pickle.dump(((train_datasets, test_datasets), config, classes_per_task),f)
return
'''
with open('loadData.pk', 'rb') as f:
((train_datasets, test_datasets), config,
classes_per_task) = pickle.load(f)
classes_per_task = 20
config[
'classes'] = classes_per_task if scenario == 'domain' else classes_per_task * args.tasks
'''
for i in range(len(train_datasets)):
for j in range(len(train_datasets[i])):
#print(train_datasets[i][j][1])
train_datasets[i][j][1]=train_datasets[i][j][1]+i*len(train_datasets)
for j in range(len(test_datasets[i])):
test_datasets[i][j][1]+=i*len(train_datasets)
for data in train_datasets:
random.shuffle(data)
for data in test_datasets:
random.shuffle(data)
print(train_datasets[0][1])
'''
#print('<-------------------------------------->')
#print(train_datasets[0][0][0].size())
#-------------------------------------------------------------------------------------------------#
#------------------------------#
#----- MODEL (CLASSIFIER) -----#
#------------------------------#
# Define main model (i.e., classifier, if requested with feedback connections)
if args.feedback:
model = AutoEncoder(
image_size=config['size'],
image_channels=config['channels'],
classes=config['classes'],
fc_layers=args.fc_lay,
fc_units=args.fc_units,
z_dim=args.z_dim,
fc_drop=args.fc_drop,
fc_bn=True if args.fc_bn == "yes" else False,
fc_nl=args.fc_nl,
).to(device)
model.lamda_pl = 1. #--> to make that this VAE is also trained to classify
else:
model = Classifier(
image_size=config['size'],
image_channels=config['channels'],
classes=config['classes'],
fc_layers=args.fc_lay,
fc_units=args.fc_units,
fc_drop=args.fc_drop,
fc_nl=args.fc_nl,
fc_bn=True if args.fc_bn == "yes" else False,
excit_buffer=True if args.gating_prop > 0 else False,
binaryCE=args.bce,
binaryCE_distill=args.bce_distill,
).to(device)
# Define optimizer (only include parameters that "requires_grad")
model.optim_list = [{
'params':
filter(lambda p: p.requires_grad, model.parameters()),
'lr':
args.lr
}]
model.optim_type = args.optimizer
if model.optim_type in ("adam", "adam_reset"):
model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999))
elif model.optim_type == "sgd":
model.optimizer = optim.SGD(model.optim_list)
else:
raise ValueError(
"Unrecognized optimizer, '{}' is not currently a valid option".
format(args.optimizer))
#-------------------------------------------------------------------------------------------------#
#----------------------------------#
#----- CL-STRATEGY: EXEMPLARS -----#
#----------------------------------#
# Store in model whether, how many and in what way to store exemplars
if isinstance(model, ExemplarHandler) and (args.use_exemplars
or args.add_exemplars
or args.replay == "exemplars"):
model.memory_budget = args.budget
model.norm_exemplars = args.norm_exemplars
model.herding = args.herding
#-------------------------------------------------------------------------------------------------#
#-----------------------------------#
#----- CL-STRATEGY: ALLOCATION -----#
#-----------------------------------#
# Elastic Weight Consolidation (EWC)
if isinstance(model, ContinualLearner):
model.ewc_lambda = args.ewc_lambda if args.ewc else 0
if args.ewc:
model.fisher_n = args.fisher_n
model.gamma = args.gamma
model.online = args.online
model.emp_FI = args.emp_fi
# Synpatic Intelligence (SI)
if isinstance(model, ContinualLearner):
model.si_c = args.si_c if args.si else 0
if args.si:
model.epsilon = args.epsilon
# XdG: create for every task a "mask" for each hidden fully connected layer
if isinstance(model, ContinualLearner) and args.gating_prop > 0:
mask_dict = {}
excit_buffer_list = []
for task_id in range(args.tasks):
mask_dict[task_id + 1] = {}
for i in range(model.fcE.layers):
layer = getattr(model.fcE, "fcLayer{}".format(i + 1)).linear
if task_id == 0:
excit_buffer_list.append(layer.excit_buffer)
n_units = len(layer.excit_buffer)
gated_units = np.random.choice(n_units,
size=int(args.gating_prop *
n_units),
replace=False)
mask_dict[task_id + 1][i] = gated_units
model.mask_dict = mask_dict
model.excit_buffer_list = excit_buffer_list
#-------------------------------------------------------------------------------------------------#
#-------------------------------#
#----- CL-STRATEGY: REPLAY -----#
#-------------------------------#
# Use distillation loss (i.e., soft targets) for replayed data? (and set temperature)
if isinstance(model, Replayer):
model.replay_targets = "soft" if args.distill else "hard"
model.KD_temp = args.temp
# If needed, specify separate model for the generator
train_gen = True if (args.replay == "generative"
and not args.feedback) else False
if train_gen:
# -specify architecture
generator = AutoEncoder(
image_size=config['size'],
image_channels=config['channels'],
fc_layers=args.g_fc_lay,
fc_units=args.g_fc_uni,
z_dim=args.g_z_dim,
classes=config['classes'],
fc_drop=args.fc_drop,
fc_bn=True if args.fc_bn == "yes" else False,
fc_nl=args.fc_nl,
).to(device)
# -set optimizer(s)
generator.optim_list = [{
'params':
filter(lambda p: p.requires_grad, generator.parameters()),
'lr':
args.lr_gen
}]
generator.optim_type = args.optimizer
if generator.optim_type in ("adam", "adam_reset"):
generator.optimizer = optim.Adam(generator.optim_list,
betas=(0.9, 0.999))
elif generator.optim_type == "sgd":
generator.optimizer = optim.SGD(generator.optim_list)
else:
generator = None
#-------------------------------------------------------------------------------------------------#
#---------------------#
#----- REPORTING -----#
#---------------------#
# Get parameter-stamp (and print on screen)
param_stamp = get_param_stamp(
args,
model.name,
verbose=True,
replay=True if (not args.replay == "none") else False,
replay_model_name=generator.name if
(args.replay == "generative" and not args.feedback) else None,
)
# Print some model-characteristics on the screen
# -main model
print("\n")
#utils.print_model_info(model, title="MAIN MODEL")
# -generator
if generator is not None:
utils.print_model_info(generator, title="GENERATOR")
# Prepare for plotting in visdom
# -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf
precision_dict = evaluate.initiate_precision_dict(args.tasks)
precision_dict_exemplars = evaluate.initiate_precision_dict(
args.tasks) if args.use_exemplars else None
# -visdom-settings
if args.visdom:
env_name = "{exp}{tasks}-{scenario}".format(exp=args.experiment,
tasks=args.tasks,
scenario=args.scenario)
graph_name = "{fb}{replay}{syn}{ewc}{xdg}{icarl}{bud}".format(
fb="1M-" if args.feedback else "",
replay="{}{}".format(args.replay, "D" if args.distill else ""),
syn="-si{}".format(args.si_c) if args.si else "",
ewc="-ewc{}{}".format(
args.ewc_lambda, "-O{}".format(args.gamma)
if args.online else "") if args.ewc else "",
xdg=""
if args.gating_prop == 0 else "-XdG{}".format(args.gating_prop),
icarl="-iCaRL" if (args.use_exemplars and args.add_exemplars
and args.bce and args.bce_distill) else "",
bud="-bud{}".format(args.budget) if
(args.use_exemplars or args.add_exemplars
or args.replay == "exemplars") else "",
)
visdom = {'env': env_name, 'graph': graph_name}
if args.use_exemplars:
visdom_exemplars = {
'env': env_name,
'graph': "{}-EX".format(graph_name)
}
else:
visdom = visdom_exemplars = None
#-------------------------------------------------------------------------------------------------#
#---------------------#
#----- CALLBACKS -----#
#---------------------#
# Callbacks for reporting on and visualizing loss
generator_loss_cbs = [
cb._VAE_loss_cb(
log=args.loss_log,
visdom=visdom,
model=model if args.feedback else generator,
tasks=args.tasks,
iters_per_task=args.iters if args.feedback else args.g_iters,
replay=False if args.replay == "none" else True)
] if (train_gen or args.feedback) else [None]
solver_loss_cbs = [
cb._solver_loss_cb(log=args.loss_log,
visdom=visdom,
model=model,
tasks=args.tasks,
iters_per_task=args.iters,
replay=False if args.replay == "none" else True)
] if (not args.feedback) else [None]
# Callbacks for evaluating and plotting generated / reconstructed samples
sample_cbs = [
cb._sample_cb(
log=args.sample_log,
visdom=visdom,
config=config,
test_datasets=test_datasets,
sample_size=args.sample_n,
iters_per_task=args.iters if args.feedback else args.g_iters)
] if (train_gen or args.feedback) else [None]
# Callbacks for reporting and visualizing accuracy
# -visdom (i.e., after each [prec_log]
eval_cb = cb._eval_cb(
log=args.prec_log,
test_datasets=test_datasets,
visdom=visdom,
precision_dict=None,
iters_per_task=args.iters,
test_size=args.prec_n,
classes_per_task=classes_per_task,
scenario=scenario,
)
# -pdf / reporting: summary plots (i.e, only after each task)
eval_cb_full = cb._eval_cb(
log=args.iters,
test_datasets=test_datasets,
precision_dict=precision_dict,
iters_per_task=args.iters,
classes_per_task=classes_per_task,
scenario=scenario,
)
# -with exemplars (both for visdom & reporting / pdf)
eval_cb_exemplars = cb._eval_cb(
log=args.iters,
test_datasets=test_datasets,
visdom=visdom_exemplars,
classes_per_task=classes_per_task,
precision_dict=precision_dict_exemplars,
scenario=scenario,
iters_per_task=args.iters,
with_exemplars=True,
) if args.use_exemplars else None
# -collect them in <lists>
eval_cbs = [eval_cb, eval_cb_full]
eval_cbs_exemplars = [eval_cb_exemplars]
#-------------------------------------------------------------------------------------------------#
#--------------------#
#----- TRAINING -----#
#--------------------#
print("--> Training:")
# Keep track of training-time
start = time.time()
# Train model
train_cl(
model,
train_datasets,
test_datasets,
replay_mode=args.replay,
scenario=scenario,
classes_per_task=classes_per_task,
iters=args.iters,
batch_size=args.batch,
savepath=savepath,
generator=generator,
gen_iters=args.g_iters,
gen_loss_cbs=generator_loss_cbs,
sample_cbs=sample_cbs,
eval_cbs=eval_cbs,
loss_cbs=generator_loss_cbs if args.feedback else solver_loss_cbs,
eval_cbs_exemplars=eval_cbs_exemplars,
use_exemplars=args.use_exemplars,
add_exemplars=args.add_exemplars,
)
# Get total training-time in seconds, and write to file
training_time = time.time() - start
time_file = open("{}/time-{}.txt".format(args.r_dir, param_stamp), 'w')
time_file.write('{}\n'.format(training_time))
time_file.close()
#-------------------------------------------------------------------------------------------------#
#----------------------#
#----- EVALUATION -----#
#----------------------#
print("\n\n--> Evaluation ({}-incremental learning scenario):".format(
args.scenario))
# Evaluate precision of final model on full test-set
precs = [
evaluate.validate(
model,
test_datasets[i],
verbose=False,
test_size=None,
task=i + 1,
with_exemplars=False,
allowed_classes=list(
range(classes_per_task * i, classes_per_task *
(i + 1))) if scenario == "task" else None)
for i in range(args.tasks)
]
print("\n Precision on test-set (softmax classification):")
for i in range(args.tasks):
print(" - Task {}: {:.4f}".format(i + 1, precs[i]))
average_precs = sum(precs) / args.tasks
print('=> average precision over all {} tasks: {:.4f}'.format(
args.tasks, average_precs))
# -with exemplars
if args.use_exemplars:
precs = [
evaluate.validate(
model,
test_datasets[i],
verbose=False,
test_size=None,
task=i + 1,
with_exemplars=True,
allowed_classes=list(
range(classes_per_task * i, classes_per_task *
(i + 1))) if scenario == "task" else None)
for i in range(args.tasks)
]
print("\n Precision on test-set (classification using exemplars):")
for i in range(args.tasks):
print(" - Task {}: {:.4f}".format(i + 1, precs[i]))
average_precs_ex = sum(precs) / args.tasks
print('=> average precision over all {} tasks: {:.4f}'.format(
args.tasks, average_precs_ex))
print("\n")
#-------------------------------------------------------------------------------------------------#
#------------------#
#----- OUTPUT -----#
#------------------#
# Average precision on full test set
output_file = open("{}/prec-{}.txt".format(args.r_dir, param_stamp), 'w')
output_file.write('{}\n'.format(
average_precs_ex if args.use_exemplars else average_precs))
output_file.close()
# -precision-dict
file_name = "{}/dict-{}".format(args.r_dir, param_stamp)
utils.save_object(
precision_dict_exemplars if args.use_exemplars else precision_dict,
file_name)
# Average precision on full test set not evaluated using exemplars (i.e., using softmax on final layer)
if args.use_exemplars:
output_file = open(
"{}/prec_noex-{}.txt".format(args.r_dir, param_stamp), 'w')
output_file.write('{}\n'.format(average_precs))
output_file.close()
# -precision-dict:
file_name = "{}/dict_noex-{}".format(args.r_dir, param_stamp)
utils.save_object(precision_dict, file_name)
#-------------------------------------------------------------------------------------------------#
#--------------------#
#----- PLOTTING -----#
#--------------------#
# If requested, generate pdf
if args.pdf:
# -open pdf
pp = visual_plt.open_pdf("{}/{}.pdf".format(args.p_dir, param_stamp))
# -show samples and reconstructions (either from main model or from separate generator)
if args.feedback or args.replay == "generative":
evaluate.show_samples(model if args.feedback else generator,
config,
size=args.sample_n,
pdf=pp)
for i in range(args.tasks):
evaluate.show_reconstruction(
model if args.feedback else generator,
test_datasets[i],
config,
pdf=pp,
task=i + 1)
# -show metrics reflecting progression during training
figure_list = [] #-> create list to store all figures to be plotted
# -generate all figures (and store them in [figure_list])
figure = visual_plt.plot_lines(
precision_dict["all_tasks"],
x_axes=precision_dict["x_task"],
line_names=['task {}'.format(i + 1) for i in range(args.tasks)])
figure_list.append(figure)
figure = visual_plt.plot_lines([precision_dict["average"]],
x_axes=precision_dict["x_task"],
line_names=['average all tasks so far'])
figure_list.append(figure)
if args.use_exemplars:
figure = visual_plt.plot_lines(
precision_dict_exemplars["all_tasks"],
x_axes=precision_dict_exemplars["x_task"],
line_names=[
'task {}'.format(i + 1) for i in range(args.tasks)
])
figure_list.append(figure)
# -add figures to pdf (and close this pdf).
for figure in figure_list:
pp.savefig(figure)
# -close pdf
pp.close()