def main(args):
# Set random seeds
old_losses = [2.**30,2**30]
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
my_experiment = experiment(args.name, args, "./results/", commit_changes=args.commit)
writer = SummaryWriter(my_experiment.path + "tensorboard")
print(args)
tasks = list(range(400))
logger = logging.getLogger('experiment')
sampler = ts.SamplerFactory.get_sampler("Sin", tasks, None, capacity=args.capacity + 1)
config = mf.ModelFactory.get_model(args.model, "Sin", in_channels=args.capacity + 1, num_actions=1,
width=args.width)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
maml = MetaLearnerRegression(args, config).to(device)
for name, param in maml.named_parameters(): #get names of paramters and parameters themselves
param.learn = True #WHAT IS DIS
for name, param in maml.net.named_parameters(): #get names of paramters and parameters themselves
param.learn = True #WHAT IS DIS
#Get list of parameters that are NOT frozen
tmp = filter(lambda x: x.requires_grad, maml.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info(maml)
logger.info('Total trainable tensors: %d', num)
#
accuracy = 0
# FREZE LAYERS OF RLN
frozen_layers = []
for temp in range(args.rln * 2): #What's RLN
frozen_layers.append("net.vars." + str(temp)) #layer name is net.vars.1, net.vars.2, etc
logger.info("Frozen layers = %s", " ".join(frozen_layers))
for step in range(args.epoch): #one epoch
if step == 0: #if initial step, record frozen layers
for name, param in maml.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
for name, param in maml.net.named_parameters(): #what's dif from above?
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
# randomly seelect the specified numer of tasks from list of all tasks
t1 = np.random.choice(tasks, args.tasks, replace=False) #sample WITHOUT replacement
#for each task selected, get iterator info for the task
iterators = []
for t in t1:
# print(sampler.sample_task([t]))
iterators.append(sampler.sample_task([t]))
x_traj, y_traj, x_rand, y_rand = construct_set(iterators, sampler, steps=args.update_step)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.cuda(), y_traj.cuda(), x_rand.cuda(), y_rand.cuda()
#predict on trajectory AND random data stream
# calls forward() method in MetaLearnerRegresssion class!!!
# Updates the RLN RLN RLN in the process!!! (STEP 4)
accs = maml(x_traj, y_traj, x_rand, y_rand,old_losses,args)
# Compute gradients for this loss wrt initial parameters to update initial parameters
# Update initial paramteres theta, W?????
# Is tis the meta-update? HELP?!?!
# I THINK THIS IS THE UPDTE TO TLN TLN TLN IN STEP 4--> (STEP 4 END)
maml.meta_optim.step() # STEP FOUR??? HELP
# Monitoring
if step in [0, 2000, 3000, 4000]:
for param_group in maml.optimizer.param_groups:
logger.info("Learning Rate at step %d = %s", step, str(param_group['lr']))
accuracy = accuracy * 0.95 + 0.05 * accs[-1] #"averaging" the accuracy #WHY DO DIS
if step % 5 == 0:
writer.add_scalar('/metatrain/train/accuracy', accs[-1], step)
writer.add_scalar('/metatrain/train/runningaccuracy', accuracy, step)
logger.info("Running average of accuracy = %s", str(accuracy))
logger.info('step: %d \t training acc (first, last) %s', step, str(accs[0]) + "," + str(accs[-1]))
if step % 100 == 0: #if step is multiple of 100
counter = 0
for name, _ in maml.net.named_parameters():
counter += 1
for lrs in [args.update_lr]: #WhY HAVE MORE THAN ONE LERARNING RTE?? # USE 0.003, aka doing inner updates
lr_results = {}
lr_results[lrs] = []
for temp in range(0, 20):
t1 = np.random.choice(tasks, args.tasks, replace=False)
iterators = []
#
for t in t1:
iterators.append(sampler.sample_task([t]))
# Step 1 on flowchart: Sample trajectory (X_traj, Y_traj) and random batch D_rand = (X_rand, Y_rand)
x_traj, y_traj, x_rand, y_rand = construct_set(iterators, sampler, steps=40)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.cuda(), y_traj.cuda(), x_rand.cuda(), y_rand.cuda()
net = copy.deepcopy(maml.net) #copy the TLN??
net = net.to(device) #port copy to the GPU
for params_old, params_new in zip(maml.net.parameters(), net.parameters()):
params_new.learn = params_old.learn #set parameters of net (the copy) to have same 'learn' value as actual net
list_of_params = list(filter(lambda x: x.learn, net.parameters())) #get paramteres of copy fw 'learn' is on/True
optimizer = optim.SGD(list_of_params, lr=lrs)
#Step 2 in flowchart figure 6 page 12
#Do k gradient updates on the TLN (W's), using MSE loss and 0.003 LR
# This is the INNER UPDATE I THINK!!!
for k in range(len(x_traj)):
logits = net(x_traj[k], None, bn_training=False)
logits_select = []
for no, val in enumerate(y_traj[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss = F.mse_loss(logits, y_traj[k, :, 0].unsqueeze(1))
optimizer.zero_grad()
loss.backward()
optimizer.step() #UPDATE THE TLN
#STep 3 of flowchart
# Use updated network to compute loss on random batch, add to the list of losses/loss results
with torch.no_grad():
#Use the updated network to predict on the random batch of data
logits = net(x_rand[0], vars=None, bn_training=False)
logits_select = []
for no, val in enumerate(y_rand[0, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_q = F.mse_loss(logits, y_rand[0, :, 0].unsqueeze(1))
lr_results[lrs].append(loss_q.item())
logger.info("Avg MSE LOSS for lr %s = %s", str(lrs), str(np.mean(lr_results[lrs])))
if args.smart and args.use_mini:
torch.save(maml.net, my_experiment.path + "regression_smart_little.model")
elif args.smart and not args.use_mini:
torch.save(maml.net, my_experiment.path + "regression_smart_big.model")
else:
torch.save(maml.net, my_experiment.path + "regression_model.model")
def main(args):
# Seed random number generators
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
my_experiment = experiment(args.name, args, "../results/", commit_changes=args.commit)
writer = SummaryWriter(my_experiment.path + "tensorboard")
print(args)
# Initalize tasks; we sample 1000 tasks for evaluation
tasks = list(range(1000))
logger = logging.getLogger('experiment')
sampler = ts.SamplerFactory.get_sampler("Sin", tasks, None, None, capacity=args.capacity + 1)
config = mf.ModelFactory.get_model("na", "Sin", in_channels=args.capacity + 1, num_actions=args.tasks)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
# Load the model
maml = MetaLearnerRegression(args, config).to(device)
maml.net = torch.load(args.model, map_location='cpu').to(device)
for name, param in maml.named_parameters():
param.learn = True
for name, param in maml.net.named_parameters():
param.learn = True
tmp = filter(lambda x: x.requires_grad, maml.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info(maml)
logger.info('Total trainable tensors: %d', num)
##### Setting up parameters for freezing RLN layers
#### Also resets TLN layers with random initialization if args.reset is true
frozen_layers = []
for temp in range(args.rln * 2):
frozen_layers.append("net.vars." + str(temp))
for name, param in maml.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
else:
if args.reset:
w = nn.Parameter(torch.ones_like(param))
if len(w.shape) > 1:
logger.info("Resseting layer %s", str(name))
torch.nn.init.kaiming_normal_(w)
else:
w = nn.Parameter(torch.zeros_like(param))
param.data = w
param.learn = True
for name, param in maml.net.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
correct = 0
counter = 0
for name, _ in maml.net.named_parameters():
# logger.info("LRs of layer %s = %s", str(name), str(torch.mean(maml.lrs[counter])))
counter += 1
for lrs in [0.003]:
loss_vector = np.zeros(args.tasks)
loss_vector_results = []
lr_results = {}
incremental_results = {}
lr_results[lrs] = []
runs = args.runs
for temp in range(0, runs):
loss_vector = np.zeros(args.tasks)
t1 = np.random.choice(tasks, args.tasks, replace=False)
print(t1)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_spt, y_spt, x_qry, y_qry = construct_set(iterators, sampler, steps=args.update_step, iid=args.iid)
if torch.cuda.is_available():
x_spt, y_spt, x_qry, y_qry = x_spt.cuda(), y_spt.cuda(), x_qry.cuda(), y_qry.cuda()
net = copy.deepcopy(maml.net)
net = net.to(device)
for params_old, params_new in zip(maml.net.parameters(), net.parameters()):
params_new.learn = params_old.learn
list_of_params = list(filter(lambda x: x.learn, net.parameters()))
optimizer = optim.SGD(list_of_params, lr=lrs)
counter = 0
x_spt_test, y_spt_test, x_qry_test, y_qry_test = construct_set(iterators, sampler, steps=300)
if torch.cuda.is_available():
x_spt_test, y_spt_test, x_qry_test, y_qry_test = x_spt_test.cuda(), y_spt_test.cuda(), x_qry_test.cuda(), y_qry_test.cuda()
for k in range(len(x_spt)):
if k % args.update_step == 0 and k > 0:
counter += 1
loss_temp = 0
if not counter in incremental_results:
incremental_results[counter] = []
with torch.no_grad():
for update_upto in range(0, counter * 300):
logits = net(x_spt_test[update_upto], vars=None, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt_test[update_upto, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_temp += F.mse_loss(logits, y_spt_test[update_upto, :, 0].unsqueeze(1))
loss_temp = loss_temp / (counter * 300)
incremental_results[counter].append(loss_temp.item())
my_experiment.results["incremental"] = incremental_results
logits = net(x_spt[k], None, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss = F.mse_loss(logits, y_spt[k, :, 0].unsqueeze(1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
counter += 1
loss_temp = 0
if not counter in incremental_results:
incremental_results[counter] = []
with torch.no_grad():
for update_upto in range(0, counter * 300):
logits = net(x_spt_test[update_upto], vars=None, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt_test[update_upto, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_temp += F.mse_loss(logits, y_spt_test[update_upto, :, 0].unsqueeze(1))
# lr_results[lrs].append(loss_q.item())
loss_temp = loss_temp / (counter * 300)
incremental_results[counter].append(loss_temp.item())
my_experiment.results["incremental"] = incremental_results
#
x_spt, y_spt, x_qry, y_qry = x_spt_test, y_spt_test, x_qry_test, y_qry_test
if torch.cuda.is_available():
x_spt, y_spt, x_qry, y_qry = x_spt.cuda(), y_spt.cuda(), x_qry.cuda(), y_qry.cuda()
with torch.no_grad():
logits = net(x_qry[0], vars=None, bn_training=False)
logits_select = []
for no, val in enumerate(y_qry[0, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_q = F.mse_loss(logits, y_qry[0, :, 0].unsqueeze(1))
lr_results[lrs].append(loss_q.item())
counter = 0
loss = 0
for k in range(len(x_spt)):
logits = net(x_spt[k], None, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_vector[int(counter / (300))] += F.mse_loss(logits, y_spt[k, :, 0].unsqueeze(1)) / 300
counter += 1
loss_vector_results.append(loss_vector.tolist())
logger.info("Loss vector all %s", str(loss_vector_results))
logger.info("Avg MSE LOSS for lr %s = %s", str(lrs), str(np.mean(lr_results[lrs])))
logger.info("Std MSE LOSS for lr %s = %s", str(lrs), str(np.std(lr_results[lrs])))
loss_vector = loss_vector / runs
print("Loss vector = ", loss_vector)
my_experiment.results[str(lrs)] = str(loss_vector_results)
my_experiment.store_json()
torch.save(maml.net, my_experiment.path + "learner.model")
def main(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
my_experiment = experiment(args.name,
args,
"../results/",
commit_changes=args.commit)
writer = SummaryWriter(my_experiment.path + "tensorboard")
print(args)
tasks = list(range(400))
logger = logging.getLogger('experiment')
sampler = ts.SamplerFactory.get_sampler("Sin",
tasks,
None,
capacity=args.capacity + 1)
config = mf.ModelFactory.get_model("na",
"Sin",
in_channels=args.capacity + 1,
num_actions=1,
width=args.width)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
maml = MetaLearnerRegression(args, config).to(device)
for name, param in maml.named_parameters():
param.learn = True
for name, param in maml.net.named_parameters():
param.learn = True
tmp = filter(lambda x: x.requires_grad, maml.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info(maml)
logger.info('Total trainable tensors: %d', num)
#
accuracy = 0
frozen_layers = []
for temp in range(args.rln * 2):
frozen_layers.append("net.vars." + str(temp))
logger.info("Frozen layers = %s", " ".join(frozen_layers))
for step in range(args.epoch):
if step == 0 and not args.no_freeze:
for name, param in maml.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
for name, param in maml.net.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
t1 = np.random.choice(tasks, args.tasks, replace=False)
iterators = []
for t in t1:
# print(sampler.sample_task([t]))
iterators.append(sampler.sample_task([t]))
x_traj, y_traj, x_rand, y_rand = construct_set(iterators,
sampler,
steps=args.update_step)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.cuda(), y_traj.cuda(
), x_rand.cuda(), y_rand.cuda()
# print(x_spt, y_spt)
accs = maml(x_traj, y_traj, x_rand, y_rand)
maml.meta_optim.step()
if step in [0, 2000, 3000, 4000]:
for param_group in maml.optimizer.param_groups:
logger.info("Learning Rate at step %d = %s", step,
str(param_group['lr']))
accuracy = accuracy * 0.95 + 0.05 * accs[-1]
if step % 5 == 0:
writer.add_scalar('/metatrain/train/accuracy', accs[-1], step)
writer.add_scalar('/metatrain/train/runningaccuracy', accuracy,
step)
logger.info("Running average of accuracy = %s", str(accuracy))
logger.info('step: %d \t training acc (first, last) %s', step,
str(accs[0]) + "," + str(accs[-1]))
if step % 100 == 0:
counter = 0
for name, _ in maml.net.named_parameters():
counter += 1
for lrs in [args.update_lr]:
lr_results = {}
lr_results[lrs] = []
for temp in range(0, 20):
t1 = np.random.choice(tasks, args.tasks, replace=False)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_traj, y_traj, x_rand, y_rand = construct_set(
iterators, sampler, steps=40, no_rand=args.no_rand)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.cuda(
), y_traj.cuda(), x_rand.cuda(), y_rand.cuda()
net = copy.deepcopy(maml.net)
net = net.to(device)
for params_old, params_new in zip(maml.net.parameters(),
net.parameters()):
params_new.learn = params_old.learn
list_of_params = list(
filter(lambda x: x.learn, net.parameters()))
optimizer = optim.SGD(list_of_params, lr=lrs)
for k in range(len(x_traj)):
logits = net(x_traj[k], None, bn_training=False)
logits_select = []
for no, val in enumerate(y_traj[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss = F.mse_loss(logits, y_traj[k, :, 0].unsqueeze(1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
#
with torch.no_grad():
logits = net(x_rand[0], vars=None, bn_training=False)
logits_select = []
for no, val in enumerate(y_rand[0, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_q = F.mse_loss(logits, y_rand[0, :,
0].unsqueeze(1))
lr_results[lrs].append(loss_q.item())
logger.info("Avg MSE LOSS for lr %s = %s", str(lrs),
str(np.mean(lr_results[lrs])))
torch.save(maml.net, my_experiment.path + "learner.model")
def main(args):
# Seed random number generators
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
my_experiment = experiment(args.name,
args,
"/data5/jlindsey/continual/results",
commit_changes=args.commit)
writer = SummaryWriter(my_experiment.path + "tensorboard")
print(args)
# Initalize tasks; we sample 1000 tasks for evaluation
tasks = list(range(1000))
logger = logging.getLogger('experiment')
sampler = ts.SamplerFactory.get_sampler("Sin",
tasks,
None,
None,
capacity=args.capacity + 1)
#config = mf.ModelFactory.get_model("na", "Sin", in_channels=args.capacity + 1, num_actions=args.tasks)
config = mf.ModelFactory.get_model(args.modeltype,
"Sin",
in_channels=args.capacity + 1,
num_actions=1,
width=args.width)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
# Load the model
maml = MetaLearnerRegression(args, config).to(device)
maml.net = torch.load(args.model, map_location='cpu').to(device)
for name, param in maml.named_parameters():
param.learn = True
for name, param in maml.net.named_parameters():
param.learn = True
tmp = filter(lambda x: x.requires_grad, maml.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info(maml)
logger.info('Total trainable tensors: %d', num)
##### Setting up parameters for freezing RLN layers
#### Also resets TLN layers with random initialization if args.reset is true
frozen_layers = []
for temp in range(args.rln * 2):
frozen_layers.append("net.vars." + str(temp))
for name, param in maml.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
else:
if args.reset:
w = nn.Parameter(torch.ones_like(param))
if len(w.shape) > 1:
logger.info("Resseting layer %s", str(name))
torch.nn.init.kaiming_normal_(w)
else:
w = nn.Parameter(torch.zeros_like(param))
param.data = w
param.learn = True
for name, param in maml.net.named_parameters():
logger.info(name)
if name in frozen_layers:
logger.info("Freeezing name %s", str(name))
param.learn = False
logger.info(str(param.requires_grad))
correct = 0
counter = 0
for name, _ in maml.net.named_parameters():
# logger.info("LRs of layer %s = %s", str(name), str(torch.mean(maml.lrs[counter])))
counter += 1
for lrs in [0.003]:
loss_vector = np.zeros(args.tasks)
loss_vector_results = []
lr_results = {}
incremental_results = {}
lr_results[lrs] = []
runs = args.runs
loss_hist = []
for temp in range(0, runs):
loss_vector = np.zeros(args.tasks)
t1 = np.random.choice(tasks, args.tasks, replace=False)
print(t1)
loss_hist.append([])
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
if args.vary_length:
num_steps = np.random.randint(args.update_step // 10,
args.update_step + 1)
x_spt, y_spt, x_qry, y_qry = construct_set(iterators,
sampler,
steps=num_steps,
iid=args.iid)
else:
num_steps = args.update_step
x_spt, y_spt, x_qry, y_qry = construct_set(
iterators, sampler, steps=args.update_step, iid=args.iid)
if torch.cuda.is_available():
x_spt, y_spt, x_qry, y_qry = x_spt.cuda(), y_spt.cuda(
), x_qry.cuda(), y_qry.cuda()
net = copy.deepcopy(maml.net)
net = net.to(device)
for params_old, params_new in zip(maml.net.parameters(),
net.parameters()):
params_new.learn = params_old.learn
list_of_params = list(filter(lambda x: x.learn, net.parameters()))
optimizer = optim.SGD(list_of_params, lr=lrs)
counter = 0
x_spt_test, y_spt_test, x_qry_test, y_qry_test = construct_set(
iterators, sampler, steps=300)
if args.train_performance:
x_spt_test, y_spt_test, x_qry_test, y_qry_test = x_spt, y_spt, x_qry, y_qry
x_qry_test, y_qry_test = x_spt_test, y_spt_test
if torch.cuda.is_available():
x_spt_test, y_spt_test, x_qry_test, y_qry_test = x_spt_test.cuda(
), y_spt_test.cuda(), x_qry_test.cuda(), y_qry_test.cuda()
fast_weights = net.vars
if args.randomize_plastic_weights:
net.randomize_plastic_weights()
if args.zero_plastic_weights:
net.zero_plastic_weights()
for k in range(len(x_spt)):
#print('hey', k, torch.sum(fast_weights[0]), torch.sum(fast_weights[14]))
if k % num_steps == 0 and k > 0:
counter += 1
loss_temp = 0
if not counter in incremental_results:
incremental_results[counter] = []
with torch.no_grad():
if args.train_performance:
for update_upto in range(0, k):
logits = net(x_spt_test[update_upto],
vars=fast_weights,
bn_training=False)
logits_select = []
for no, val in enumerate(
y_spt_test[update_upto, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(
1)
loss_temp += F.mse_loss(
logits, y_spt_test[update_upto, :,
0].unsqueeze(1))
loss_temp = loss_temp / (k)
else:
for update_upto in range(0, counter * 300):
logits = net(x_spt_test[update_upto],
vars=fast_weights,
bn_training=False)
logits_select = []
for no, val in enumerate(
y_spt_test[update_upto, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(
1)
loss_temp += F.mse_loss(
logits, y_spt_test[update_upto, :,
0].unsqueeze(1))
loss_temp = loss_temp / (counter * 300)
incremental_results[counter].append(loss_temp.item())
my_experiment.results[
"incremental"] = incremental_results
logits = net(x_spt[k], vars=fast_weights, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss = F.mse_loss(logits, y_spt[k, :, 0].unsqueeze(1))
loss_hist[temp].append(loss.cpu().detach().numpy())
grad = torch.autograd.grad(loss, fast_weights)
# fast_weights = list(map(lambda p: p[1] - self.update_lr * p[0], zip(grad, self.net.parameters())))
#print('heyyy', args.plastic_update, args.update_lr, np.array(net.vars_plasticity[-2].cpu().detach()), np.array(net.vars_plasticity[-1].cpu().detach()))
if args.plastic_update:
fast_weights = list(
map(
lambda p: p[1] - p[0] * p[2]
if p[1].learn else p[1],
zip(grad, fast_weights, net.vars_plasticity)))
elif args.layer_level_plastic_update:
fast_weights = list(
map(
lambda p: p[1] - p[0] * p[2]
if p[1].learn else p[1],
zip(grad, fast_weights,
net.layer_level_vars_plasticity)))
else:
fast_weights = list(
map(
lambda p: p[1] - args.update_lr * p[0]
if p[1].learn else p[1], zip(grad, fast_weights)))
for params_old, params_new in zip(net.vars, fast_weights):
params_new.learn = params_old.learn
#print('param', params_new.learn)
optimizer.zero_grad()
#loss.backward()
#optimizer.step()
counter += 1
loss_temp = 0
if not counter in incremental_results:
incremental_results[counter] = []
with torch.no_grad():
if args.train_performance:
for update_upto in range(0, k):
logits = net(x_spt_test[update_upto],
vars=fast_weights,
bn_training=False)
logits_select = []
for no, val in enumerate(y_spt_test[update_upto, :,
1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_temp += F.mse_loss(
logits, y_spt_test[update_upto, :, 0].unsqueeze(1))
# lr_results[lrs].append(loss_q.item())
loss_temp = loss_temp / (k)
else:
for update_upto in range(0, counter * 300):
logits = net(x_spt_test[update_upto],
vars=fast_weights,
bn_training=False)
logits_select = []
for no, val in enumerate(y_spt_test[update_upto, :,
1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_temp += F.mse_loss(
logits, y_spt_test[update_upto, :, 0].unsqueeze(1))
# lr_results[lrs].append(loss_q.item())
loss_temp = loss_temp / (counter * 300)
incremental_results[counter].append(loss_temp.item())
my_experiment.results["incremental"] = incremental_results
#
x_spt, y_spt, x_qry, y_qry = x_spt_test, y_spt_test, x_qry_test, y_qry_test
if torch.cuda.is_available():
x_spt, y_spt, x_qry, y_qry = x_spt.cuda(), y_spt.cuda(
), x_qry.cuda(), y_qry.cuda()
with torch.no_grad():
logits = net(x_qry[0], vars=fast_weights, bn_training=False)
logits_select = []
for no, val in enumerate(y_qry[0, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_q = F.mse_loss(logits, y_qry[0, :, 0].unsqueeze(1))
print('loss', loss_q.item())
lr_results[lrs].append(loss_q.item())
counter = 0
loss = 0
for k in range(len(x_spt)):
logits = net(x_spt[k], vars=fast_weights, bn_training=False)
logits_select = []
for no, val in enumerate(y_spt[k, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss_vector[int(counter / (300))] += F.mse_loss(
logits, y_spt[k, :, 0].unsqueeze(1)) / 300
counter += 1
loss_vector_results.append(loss_vector.tolist())
np.save("loss_hist_" + args.orig_name + ".npy", loss_hist)
logger.info("Loss vector all %s", str(loss_vector_results))
logger.info("Avg MSE LOSS for lr %s = %s", str(lrs),
str(np.mean(lr_results[lrs])))
logger.info("Std MSE LOSS for lr %s = %s", str(lrs),
str(np.std(lr_results[lrs])))
loss_vector = loss_vector / runs
print("Loss vector = ", loss_vector)
my_experiment.results[str(lrs)] = str(loss_vector_results)
my_experiment.store_json()
np.save('evals/loss_vector_results_' + args.orig_name + '.npy',
loss_vector_results)
np.save('evals/final_results_' + args.orig_name + '.npy', lr_results)
np.save('evals/incremental_results_' + args.orig_name + '.npy',
incremental_results)
def main():
p = reg_parser.Parser()
total_seeds = len(p.parse_known_args()[0].seed)
run = p.parse_known_args()[0].run
all_args = vars(p.parse_known_args()[0])
args = utils.get_run(all_args, run)
my_experiment = experiment(args["name"],
args,
args["output_dir"],
sql=True,
run=int(run / total_seeds),
seed=total_seeds)
my_experiment.results["all_args"] = all_args
my_experiment.make_table("metrics", {
"run": 0,
"meta_loss": 0.0,
"step": 0
}, ("run", "step"))
metrics_keys = ["run", "meta_loss", "step"]
logger = logging.getLogger('experiment')
tasks = list(range(400))
sampler = ts.SamplerFactory.get_sampler("Sin",
tasks,
None,
capacity=args["capacity"] + 1)
model_config = mf.ModelFactory.get_model(args["model"],
"Sin",
input_dimension=args["capacity"] +
1,
output_dimension=1,
width=args["width"],
cols=args["cols"])
gpu_to_use = run % args["gpus"]
if torch.cuda.is_available():
device = torch.device('cuda:' + str(gpu_to_use))
logger.info("Using gpu : %s", 'cuda:' + str(gpu_to_use))
else:
device = torch.device('cpu')
if args.get('update_rule') == "RTRL":
logger.info("Columnar Net based gradient approximation...")
metalearner = MetaLearnerRegressionCol(args,
model_config,
device=device).to(device)
else:
logger.info("BPTT update rule...")
metalearner = MetaLearnerRegression(args, model_config,
device=device).to(device)
tmp = filter(lambda x: x.requires_grad, metalearner.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info('Total trainable tensors: %d', num)
running_meta_loss = 0
adaptation_loss = 0
loss_history = []
metrics_list = []
metrics_keys = ["run", "meta_loss", "step"]
adaptation_loss_history = []
adaptation_running_loss_history = []
meta_steps_counter = 0
LOG_INTERVAL = 2
for step in range(args["epoch"]):
if step % LOG_INTERVAL == 0:
logger.debug("####\t STEP %d \t####", step)
net = metalearner.net
meta_steps_counter += 1
t1 = np.random.choice(tasks, args["tasks"], replace=False)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_traj_meta, y_traj_meta, x_rand_meta, y_rand_meta = utils.construct_set(
iterators, sampler, steps=1)
x_traj_meta, x_rand_meta, y_traj_meta, y_rand_meta = x_traj_meta.view(
-1, 51), x_rand_meta.view(-1, 51), y_traj_meta.view(
-1, 2), y_rand_meta.view(-1, 2)
if torch.cuda.is_available():
x_traj_meta, y_traj_meta, x_rand_meta, y_rand_meta = x_traj_meta.to(
device), y_traj_meta.to(device), x_rand_meta.to(
device), y_rand_meta.to(device)
meta_loss = metalearner(x_traj_meta, y_traj_meta, x_rand_meta,
y_rand_meta)
loss_history.append(meta_loss[-1].detach().cpu().item())
running_meta_loss = running_meta_loss * 0.97 + 0.03 * meta_loss[
-1].detach().cpu()
running_meta_loss_fixed = running_meta_loss / (1 -
(0.97**
(meta_steps_counter)))
metrics_list.append((run, running_meta_loss_fixed.item(), step))
if step % LOG_INTERVAL == 0:
if running_meta_loss > 0:
logger.info("Running meta loss = %f",
running_meta_loss_fixed.item())
with torch.no_grad():
t1 = np.random.choice(tasks, args["tasks"], replace=False)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_traj, y_traj, x_rand, y_rand = utils.construct_set(iterators,
sampler,
steps=1)
x_traj, x_rand, y_traj, y_rand = x_traj.view(
-1,
51), x_rand.view(-1,
51), y_traj.view(-1,
2), y_rand.view(-1, 2)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.to(
device), y_traj.to(device), x_rand.to(
device), y_rand.to(device)
logits_select = []
for i in range(len(x_rand)):
l, _, _ = net.forward_col(x_rand[i], vars=None, grad=False)
logits_select.append(l)
logits = torch.stack(logits_select).unsqueeze(1)
current_adaptation_loss = F.mse_loss(logits,
y_rand[:, 0].unsqueeze(1))
adaptation_loss_history.append(
current_adaptation_loss.detach().item())
adaptation_loss = adaptation_loss * 0.97 + current_adaptation_loss.detach(
).cpu().item() * 0.03
adaptation_loss_fixed = adaptation_loss / (1 -
(0.97**(step + 1)))
adaptation_running_loss_history.append(adaptation_loss_fixed)
logger.info("Adaptation loss = %f", current_adaptation_loss)
if step % LOG_INTERVAL == 0:
logger.info("Running adaptation loss = %f",
adaptation_loss_fixed)
if (step + 1) % (LOG_INTERVAL * 500) == 0:
if not args["no_save"]:
torch.save(metalearner.net, my_experiment.path + "net.model")
dict_names = {}
for (name, param) in metalearner.net.named_parameters():
dict_names[name] = param.adaptation
my_experiment.insert_values("metrics", metrics_keys, metrics_list)
metrics_list = []
my_experiment.add_result("Layers meta values", dict_names)
my_experiment.add_result("Meta loss", loss_history)
my_experiment.add_result("Adaptation loss",
adaptation_loss_history)
my_experiment.add_result("Running adaption loss",
adaptation_running_loss_history)
my_experiment.store_json()
def main():
p = reg_parser.Parser()
total_seeds = len(p.parse_known_args()[0].seed)
rank = p.parse_known_args()[0].rank
all_args = vars(p.parse_known_args()[0])
args = utils.get_run(vars(p.parse_known_args()[0]), rank)
utils.set_seed(args["seed"])
my_experiment = experiment(args["name"],
args,
"../results/",
commit_changes=False,
rank=int(rank / total_seeds),
seed=total_seeds)
my_experiment.results["all_args"] = all_args
writer = SummaryWriter(my_experiment.path + "tensorboard")
logger = logging.getLogger('experiment')
pprint(args)
tasks = list(range(400))
sampler = ts.SamplerFactory.get_sampler("Sin",
tasks,
None,
capacity=args["capacity"] + 1)
model_config = mf.ModelFactory.get_model(args["model"],
"Sin",
input_dimension=args["capacity"] +
1,
output_dimension=1,
width=args["width"])
context_backbone_config = None
gpu_to_use = rank % args["gpus"]
if torch.cuda.is_available():
device = torch.device('cuda:' + str(gpu_to_use))
logger.info("Using gpu : %s", 'cuda:' + str(gpu_to_use))
else:
device = torch.device('cpu')
metalearner = MetaLearnerRegression(args, model_config,
context_backbone_config).to(device)
tmp = filter(lambda x: x.requires_grad, metalearner.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info('Total trainable tensors: %d', num)
#
running_meta_loss = 0
adaptation_loss = 0
loss_history = []
adaptation_loss_history = []
adaptation_running_loss_history = []
meta_steps_counter = 0
LOG_INTERVAL = 50
for step in range(args["epoch"]):
if step % LOG_INTERVAL == 0:
logger.debug("####\t STEP %d \t####", step)
net = metalearner.net
meta_steps_counter += 1
t1 = np.random.choice(tasks, args["tasks"], replace=False)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_traj_meta, y_traj_meta, x_rand_meta, y_rand_meta = utils.construct_set(
iterators, sampler, steps=args["update_step"])
x_traj_meta, y_traj_meta = x_traj_meta.view(-1, 1,
51), y_traj_meta.view(
-1, 1, 2)
if torch.cuda.is_available():
x_traj_meta, y_traj_meta, x_rand_meta, y_rand_meta = x_traj_meta.to(
device), y_traj_meta.to(device), x_rand_meta.to(
device), y_rand_meta.to(device)
meta_loss = metalearner(x_traj_meta, y_traj_meta, x_rand_meta,
y_rand_meta)
loss_history.append(meta_loss[-1].detach().cpu().item())
running_meta_loss = running_meta_loss * 0.97 + 0.03 * meta_loss[
-1].detach().cpu()
running_meta_loss_fixed = running_meta_loss / (1 -
(0.97**
(meta_steps_counter)))
writer.add_scalar('/metatrain/train/accuracy',
meta_loss[-1].detach().cpu(), meta_steps_counter)
writer.add_scalar('/metatrain/train/runningaccuracy',
running_meta_loss_fixed, meta_steps_counter)
if step % LOG_INTERVAL == 0:
if running_meta_loss > 0:
logger.info("Running meta loss = %f",
running_meta_loss_fixed.item())
with torch.no_grad():
t1 = np.random.choice(tasks, args["tasks"], replace=False)
iterators = []
for t in t1:
iterators.append(sampler.sample_task([t]))
x_traj, y_traj, x_rand, y_rand = utils.construct_set(
iterators, sampler, steps=args["update_step"])
x_traj, y_traj = x_traj.view(-1, 1, 51), y_traj.view(-1, 1, 2)
if torch.cuda.is_available():
x_traj, y_traj, x_rand, y_rand = x_traj.to(
device), y_traj.to(device), x_rand.to(
device), y_rand.to(device)
logits = net(x_rand[0], vars=None)
logits_select = []
assert y_rand[0, :, 1].sum() == 0
for no, val in enumerate(y_rand[0, :, 1].long()):
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
current_adaptation_loss = F.mse_loss(
logits, y_rand[0, :, 0].unsqueeze(1))
adaptation_loss_history.append(
current_adaptation_loss.detach().item())
adaptation_loss = adaptation_loss * 0.97 + current_adaptation_loss.detach(
).cpu().item() * 0.03
adaptation_loss_fixed = adaptation_loss / (1 -
(0.97**(step + 1)))
adaptation_running_loss_history.append(adaptation_loss_fixed)
logger.info("Adaptation loss = %f", current_adaptation_loss)
if step % LOG_INTERVAL == 0:
logger.info("Running adaptation loss = %f",
adaptation_loss_fixed)
writer.add_scalar('/learn/test/adaptation_loss',
current_adaptation_loss, step)
if (step + 1) % (LOG_INTERVAL * 500) == 0:
if not args["no_save"]:
torch.save(metalearner.net, my_experiment.path + "net.model")
dict_names = {}
for (name, param) in metalearner.net.named_parameters():
dict_names[name] = param.adaptation
my_experiment.add_result("Layers meta values", dict_names)
my_experiment.add_result("Meta loss", loss_history)
my_experiment.add_result("Adaptation loss",
adaptation_loss_history)
my_experiment.add_result("Running adaption loss",
adaptation_running_loss_history)
my_experiment.store_json()
def main(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
my_experiment = experiment(args.name, args, "../results/", commit_changes=args.commit)
writer = SummaryWriter(my_experiment.path + "tensorboard")
print(args)
tasks = list(range(400))
logger = logging.getLogger('experiment')
sampler = ts.SamplerFactory.get_sampler("Sin2", tasks, None, None, capacity=401)
config = mf.ModelFactory.get_model("na", "Sin", in_channels=11, num_actions=30, width=args.width)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
maml = MetaLearnerRegression(args, config).to(device)
for name, param in maml.named_parameters():
param.learn = True
for name, param in maml.net.named_parameters():
param.learn = True
tmp = filter(lambda x: x.requires_grad, maml.parameters())
num = sum(map(lambda x: np.prod(x.shape), tmp))
logger.info(maml)
logger.info('Total trainable tensors: %d', num)
#
accuracy = 0
frozen_layers = []
for temp in range(args.total_frozen * 2):
frozen_layers.append("net.vars." + str(temp))
logger.info("Frozen layers = %s", " ".join(frozen_layers))
opt = torch.optim.Adam(maml.parameters(), lr=args.lr)
meta_optim = torch.optim.lr_scheduler.MultiStepLR(opt, [5000, 8000], 0.2)
for step in range(args.epoch):
if step %300 == 0:
print(step)
for heads in range(30):
t1 = tasks
# print(tasks)
iterators = []
if not args.baseline:
for t in range(heads*10, heads*10+10):
# print(sampler.sample_task([t]))
# print(t)
iterators.append(sampler.sample_task([t]))
else:
iterators.append(sampler.get_another_complete_iterator())
x_spt, y_spt, x_qry, y_qry = construct_set(iterators, sampler, steps=args.update_step, offset =heads*10)
if torch.cuda.is_available():
x_spt, y_spt, x_qry, y_qry = x_spt.cuda(), y_spt.cuda(), x_qry.cuda(), y_qry.cuda()
# print(x_spt, y_spt)
net = maml.net
logits = net(x_qry[0], None, bn_training=False)
logits_select = []
for no, val in enumerate(y_qry[0, :, 1].long()):
# print(y_qry[0, :, 1].long())
logits_select.append(logits[no, val])
logits = torch.stack(logits_select).unsqueeze(1)
loss = F.mse_loss(logits, y_qry[0, :, 0].unsqueeze(1))
opt.zero_grad()
loss.backward()
opt.step()
meta_optim.step()
# print(loss)
accuracy = accuracy * 0.95 + 0.05 * loss
if step % 500 == 0:
writer.add_scalar('/metatrain/train/accuracy', loss, step)
writer.add_scalar('/metatrain/train/runningaccuracy', accuracy, step)
logger.info("Running average of accuracy = %s", str(accuracy.item()))
if step%500 == 0:
torch.save(maml.net, my_experiment.path + "learner.model")