def __init__(self, args):
self.args = args
self.cuda = (args.cuda and torch.cuda.is_available())
self.epoch = args.epoch
self.batch_size = args.batch_size
self.lr = args.lr
self.eps = 1e-9
self.K = args.K
self.beta = args.beta
self.num_avg = args.num_avg
self.global_iter = 0
self.global_epoch = 0
# Network & Optimizer
self.toynet = cuda(ToyNet(self.K), self.cuda)
self.toynet.weight_init()
self.toynet_ema = Weight_EMA_Update(cuda(ToyNet(self.K), self.cuda),\
self.toynet.state_dict(), decay=0.999)
self.optim = optim.Adam(self.toynet.parameters(),
lr=self.lr,
betas=(0.5, 0.999))
self.scheduler = lr_scheduler.ExponentialLR(self.optim, gamma=0.97)
self.ckpt_dir = Path(args.ckpt_dir).joinpath(args.env_name)
if not self.ckpt_dir.exists():
self.ckpt_dir.mkdir(parents=True, exist_ok=True)
self.load_ckpt = args.load_ckpt
if self.load_ckpt != '': self.load_checkpoint(self.load_ckpt)
# History
self.history = dict()
self.history['avg_acc'] = 0.
self.history['info_loss'] = 0.
self.history['class_loss'] = 0.
self.history['total_loss'] = 0.
self.history['epoch'] = 0
self.history['iter'] = 0
# Tensorboard
self.tensorboard = args.tensorboard
if self.tensorboard:
self.env_name = args.env_name
self.summary_dir = Path(args.summary_dir).joinpath(args.env_name)
if not self.summary_dir.exists():
self.summary_dir.mkdir(parents=True, exist_ok=True)
self.tf = SummaryWriter(log_dir=self.summary_dir)
self.tf.add_text(tag='argument',
text_string=str(args),
global_step=self.global_epoch)
# Dataset
self.data_loader = return_data(args)
def __init__(self, args):
self.args = args
self.device = torch.device(
args.device if torch.cuda.is_available() else 'cpu')
self.epoch = args.epoch
self.batch_size = args.batch_size
self.lr = args.lr
self.eps = 1e-9
self.K = args.K
self.beta = args.beta
self.num_avg = args.num_avg
self.global_iter = 0
self.global_epoch = 0
# Network & Optimizer
self.toynet = ToyNet(self.K).to(self.device)
self.toynet.weight_init()
self.toynet_ema = Weight_EMA_Update(ToyNet(self.K).to(self.device),
self.toynet.state_dict(),
decay=0.999)
self.optim = optim.Adam(self.toynet.parameters(),
lr=self.lr,
betas=(0.5, 0.999))
self.scheduler = lr_scheduler.ExponentialLR(self.optim, gamma=0.97)
self.ckpt_dir = Path(args.ckpt_dir)
if not self.ckpt_dir.exists():
self.ckpt_dir.mkdir(parents=True, exist_ok=True)
self.load_ckpt = args.load_ckpt
if self.load_ckpt != '':
self.load_checkpoint(self.load_ckpt)
# History
self.history = dict()
self.history['avg_acc'] = 0.
self.history['info_loss'] = 0.
self.history['class_loss'] = 0.
self.history['total_loss'] = 0.
self.history['epoch'] = 0
self.history['iter'] = 0
# Dataset
self.data_loader = return_data(args)
def __init__(self, args):
self.args = args
self.epoch = args.epoch
self.batch_size = args.batch_size
self.lr = args.lr
self.K = args.K
self.num_avg = args.num_avg
self.global_iter = 0
self.global_epoch = 0
self.log_file = args.log_file
# Network & Optimizer
self.toynet = ToyNet(args).cuda()
self.optim = optim.Adam(self.toynet.parameters(), lr=self.lr)
self.ckpt_dir = Path(args.ckpt_dir)
if not self.ckpt_dir.exists():
self.ckpt_dir.mkdir(parents=True, exist_ok=True)
self.load_ckpt = args.load_ckpt
if self.load_ckpt != '': self.load_checkpoint(self.load_ckpt)
# loss function
self.ner_lossfn = nn.NLLLoss(reduction='sum')
self.rc_lossfn = nn.BCELoss(reduction='sum')
# History
self.history = dict()
# class loss
self.history['ner_train_loss1'] = []
self.history['rc_train_loss1'] = []
self.history['ner_test_loss1'] = []
self.history['rc_test_loss1'] = []
self.history['ner_train_loss2'] = []
self.history['rc_train_loss2'] = []
self.history['ner_test_loss2'] = []
self.history['rc_test_loss2'] = []
self.history['precision_test'] = []
self.history['recall_test'] = []
self.history['F1_test'] = []
# info loss
self.history['info_train_loss'] = []
self.history['info_test_loss'] = []
# Dataset
vocab = Vocab(args.dset_dir + '/' + args.dataset + '/vocab.pkl')
self.data_loader = dict()
self.data_loader['train'] = Dataloader(
args.dset_dir + '/' + args.dataset + '/train.json',
args.batch_size, vars(args), vocab)
self.data_loader['test'] = Dataloader(args.dset_dir + '/' +
args.dataset + '/test.json',
args.batch_size,
vars(args),
vocab,
evaluation=True)
import numpy as np
import pickle
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import cv2
batch_size = 32
M = 1
max_logvar = 2.0
networks = []
for i in range(M):
network = ToyNet("eval_Ensemble-Adam_1_M1024",
project_dir="/root/evaluating_bdl/toyRegression").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyRegression/training_logs/model_Ensemble-Adam_1_M1024_%d/checkpoints/model_Ensemble-Adam_1_M1024_epoch_150.pth"
% i))
networks.append(network)
M = float(len(networks))
print(M)
val_dataset = ToyDatasetEval()
num_val_batches = int(len(val_dataset) / batch_size)
print("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
train_dataset = ToyDataset()
N = float(len(train_dataset))
print(N)
alpha = 1.0
num_train_batches = int(len(train_dataset) / batch_size)
print("num_train_batches:", num_train_batches)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
init_param_values = {}
network = ToyNet(model_id,
project_dir="/root/evaluating_bdl/toyRegression").cuda()
for name, param in network.named_parameters():
init_param_values[name] = param.data
M = 4
for i in range(M):
network = ToyNet(
model_id + "_%d" % i,
project_dir="/root/evaluating_bdl/toyRegression").cuda()
for name, param in network.named_parameters():
param.data = torch.tensor(
init_param_values[name]) # NOTE! create a copy!
optimizer = torch.optim.Adam(network.parameters(), lr=learning_rate)
import numpy as np
import pickle
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import cv2
batch_size = 32
M = 4
max_logvar = 2.0
networks = []
for i in range(M):
network = ToyNet("eval_Ensemble-MAP-Adam-Fixed_1_M4",
project_dir="/root/evaluating_bdl/toyRegression").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyRegression/training_logs/model_Ensemble-MAP-Adam-Fixed_1_M4_%d/checkpoints/model_Ensemble-MAP-Adam-Fixed_1_M4_epoch_150.pth"
% i))
networks.append(network)
M = float(len(networks))
print(M)
val_dataset = ToyDatasetEval()
num_val_batches = int(len(val_dataset) / batch_size)
print("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
gt_x_values_2_scores = pickle.load(file)
val_dataset = ToyDatasetEvalKL()
num_val_batches = int(len(val_dataset) / batch_size)
print("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size,
shuffle=False)
torch.autograd.set_grad_enabled(False)
KL_values = []
for model_i in range(num_models):
network = ToyNet(model_id,
project_dir="/root/ebms_regression/1dregression").cuda()
network.load_state_dict(
torch.load(
"/root/ebms_regression/1dregression/training_logs/model_%s_%d/checkpoints/model_%s_epoch_%d.pth"
% (model_id, model_i, model_id, epoch)))
x_values = []
x_values_2_scores = {}
network.eval() # (set in eval mode, this affects BatchNorm and dropout)
for step, (x) in enumerate(val_loader):
x = x.cuda().unsqueeze(1) # (shape: (batch_size, 1))
y_samples = np.linspace(-3.0, 3.0,
num_samples) # (shape: (num_samples, ))
y_samples = y_samples.astype(np.float32)
train_dataset = ToyDataset()
N = float(len(train_dataset))
print(N)
alpha = 1.0
num_train_batches = int(len(train_dataset) / batch_size)
print("num_train_batches:", num_train_batches)
num_steps = num_epochs * num_train_batches + 1
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
network = ToyNet(
model_id, project_dir="/root/evaluating_bdl/toyClassification").cuda()
optimizer = SGD(network.parameters(),
momentum=(1.0 - eta),
lr=learning_rate)
epoch_losses_train = []
for epoch in range(num_epochs):
print("###########################")
print("######## NEW EPOCH ########")
print("###########################")
print("epoch: %d/%d" % (epoch + 1, num_epochs))
print("model: %d/%d" % (i + 1, 10))
network.train(
) # (set in training mode, this affects BatchNorm and dropout)
import pickle
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import cv2
batch_size = 32
M = 4
x_min = -6.0
x_max = 6.0
num_points = 60
network = ToyNet("Farzaneh_eval_MC-Dropout-MAP-01-Adam_1_M10_0",
project_dir="../").cuda()
network.load_state_dict(
torch.load(
"../training_logs/model_Farzaneh_MC-Dropout-MAP-01-Adam_1_M10_0/checkpoints/model_Farzaneh_MC-Dropout-MAP-01-Adam_1_M10_epoch_300.pth"
))
M_float = float(M)
print(M_float)
network.eval()
false_prob_values = np.zeros((num_points, num_points))
x_values = np.linspace(x_min, x_max, num_points, dtype=np.float32)
for x_1_i, x_1_value in enumerate(x_values):
for x_2_i, x_2_value in enumerate(x_values):
x = torch.from_numpy(np.array(
L = int(1024 / M)
if L > 20:
L = 20
print(L)
KL_p_HMC_q_total_values = []
KL_p_HMC_q_train_values = []
for l in range(L):
networks = []
for i in range(M):
if (l * M + i) % 100 == 0:
print(l * M + i)
network = ToyNet(
"eval_Ensemble-MAP-Adam_1_M1024",
project_dir="/root/evaluating_bdl/toyClassification").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyClassification/training_logs/model_Ensemble-MAP-Adam_1_M1024_%d/checkpoints/model_Ensemble-MAP-Adam_1_M1024_epoch_150.pth"
% (l * M + i)))
networks.append(network)
M_float = float(len(networks))
# print (M_float)
for network in networks:
network.eval()
false_prob_values = np.zeros((num_points, num_points))
for x_1_i, x_1_value in enumerate(x_values):
import numpy as np
import pickle
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import cv2
batch_size = 32
M = 4
max_logvar = 2.0
network = ToyNet("eval_MC-Dropout-MAP-02-Adam_1_M10",
project_dir="/root/evaluating_bdl/toyRegression").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyRegression/training_logs/model_MC-Dropout-MAP-02-Adam_1_M10_%d/checkpoints/model_MC-Dropout-MAP-02-Adam_1_M10_epoch_300.pth"
% 0))
M_float = float(M)
print(M_float)
val_dataset = ToyDatasetEval()
num_val_batches = int(len(val_dataset) / batch_size)
print("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size,
M_values = [1, 4, 16, 64, 256]
for M in M_values:
for iter in range(6):
print(M)
if M > 1:
step_size = float(num_epochs - num_epochs_low) / float(M - 1)
else:
step_size = 0
print(step_size)
networks = []
for i in range(M):
print(int(num_epochs - i * step_size))
network = ToyNet("eval_SGLD-64", project_dir="../toyRegression")
network.load_state_dict(
torch.load(
"../toyRegression/training_logs/model_SGLD-64_%d/checkpoints/model_SGLD-64_%d_epoch_%d.pth"
% (iter + 1, iter + 1, int(num_epochs - i * step_size))))
networks.append(network)
M_float = float(len(networks))
print(M_float)
val_dataset = ToyDatasetEval()
num_val_batches = int(len(val_dataset) / batch_size)
print("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
return lr
train_dataset = ToyDataset()
N = float(len(train_dataset))
print (N)
alpha = 1.0
num_train_batches = int(len(train_dataset)/batch_size)
print ("num_train_batches:", num_train_batches)
num_steps = num_epochs*num_train_batches + 1
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
network = ToyNet(model_id, project_dir="../toyRegression")
optimizer = torch.optim.SGD(network.parameters(), lr=learning_rate)
epoch_losses_train = []
for epoch in range(num_epochs):
print ("###########################")
print ("######## NEW EPOCH ########")
print ("###########################")
print ("epoch: %d/%d" % (epoch+1, num_epochs))
print ("model: %d/%d" % (i+1, 10))
network.train() # (set in training mode, this affects BatchNorm and dropout)
batch_losses = []
for step, (x, y) in enumerate(train_loader):
x = Variable(x).unsqueeze(1) # (shape: (batch_size, 1))
import matplotlib.cm as cm
import cv2
import numpy as np
x_min = -6.0
x_max = 6.0
num_points = 60
M_values = [4, 16, 64]
for M in M_values:
for iter in range(6):
networks = []
for i in range(M):
network = ToyNet(
"eval_Ensemble-Adam-Fixed",
project_dir="/root/evaluating_bdl/toyClassification").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyClassification/training_logs/model_Ensemble-Adam-Fixed_%d_M%d_%d/checkpoints/model_Ensemble-Adam-Fixed_%d_M%d_epoch_150.pth"
% (iter + 1, M, i, iter + 1, M)))
networks.append(network)
M_float = float(len(networks))
print(M_float)
for network in networks:
network.eval()
false_prob_values = np.zeros((num_points, num_points))
x_values = np.linspace(x_min, x_max, num_points, dtype=np.float32)
alpha = 1.0
p = 0.1
loss_fn = nn.CrossEntropyLoss()
num_train_batches = int(len(train_dataset) / batch_size)
print("num_train_batches:", num_train_batches)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
M = 10
for i in range(M):
network = ToyNet(model_id + "_%d" % i, project_dir="../").cuda()
optimizer = torch.optim.Adam(network.parameters(), lr=learning_rate)
epoch_losses_train = []
for epoch in range(num_epochs):
print("###########################")
print("######## NEW EPOCH ########")
print("###########################")
print("epoch: %d/%d" % (epoch + 1, num_epochs))
print("network: %d/%d" % (i + 1, M))
network.train(
) # (set in training mode, this affects BatchNorm and dropout)
batch_losses = []
for step, (x, y) in enumerate(train_loader):
max_logvar = 2.0
L = 256
num_epochs = L*150
num_epochs_low = int(0.75*num_epochs)
print (num_epochs_low)
step_size = float(num_epochs - num_epochs_low)/float(M-1)
print (step_size)
networks = []
for i in range(M):
print (int(num_epochs - i*step_size))
network = ToyNet("eval_SGHMC-256_1", project_dir="/root/evaluating_bdl/toyRegression").cuda()
network.load_state_dict(torch.load("/root/evaluating_bdl/toyRegression/training_logs/model_SGHMC-256_1/checkpoints/model_SGHMC-256_1_epoch_%d.pth" % int(num_epochs - i*step_size)))
networks.append(network)
M = float(len(networks))
print (M)
val_dataset = ToyDatasetEval()
num_val_batches = int(len(val_dataset)/batch_size)
print ("num_val_batches:", num_val_batches)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False)
for network in networks:
network.eval()
M_values = [1, 4, 16, 64, 256]
for M in M_values:
for iter in range(6):
print (M)
if M > 1:
step_size = float(num_epochs - num_epochs_low)/float(M-1)
else:
step_size = 0
print (step_size)
networks = []
for i in range(M):
print (int(num_epochs - i*step_size))
network = ToyNet("eval_SGLD-64_1-10", project_dir="/root/evaluating_bdl/toyClassification").cuda()
network.load_state_dict(torch.load("/root/evaluating_bdl/toyClassification/training_logs/model_SGLD-64_%d/checkpoints/model_SGLD-64_%d_epoch_%d.pth" % (iter+1, iter+1, int(num_epochs - i*step_size))))
networks.append(network)
M_float = float(len(networks))
print (M_float)
for network in networks:
network.eval()
false_prob_values = np.zeros((num_points, num_points))
x_values = np.linspace(x_min, x_max, num_points, dtype=np.float32)
for x_1_i, x_1_value in enumerate(x_values):
for x_2_i, x_2_value in enumerate(x_values):
x = torch.from_numpy(np.array([x_1_value, x_2_value])).unsqueeze(0).cuda() # (shape: (1, 2))
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import cv2
batch_size = 32
M = 4
x_min = -6.0
x_max = 6.0
num_points = 60
network = ToyNet("eval_MC-Dropout-MAP-02-SGDMOM_1_M10_0",
project_dir="/root/evaluating_bdl/toyClassification").cuda()
network.load_state_dict(
torch.load(
"/root/evaluating_bdl/toyClassification/training_logs/model_MC-Dropout-MAP-02-SGDMOM_1_M10_0/checkpoints/model_MC-Dropout-MAP-02-SGDMOM_1_M10_epoch_300.pth"
))
M_float = float(M)
print(M_float)
network.eval()
false_prob_values = np.zeros((num_points, num_points))
x_values = np.linspace(x_min, x_max, num_points, dtype=np.float32)
for x_1_i, x_1_value in enumerate(x_values):
for x_2_i, x_2_value in enumerate(x_values):
x = torch.from_numpy(np.array(