def predict(self, data_loader):
pred = []
for data in data_loader:
x = torch.autograd.Variable(data.cuda())
o = self.h(x)
pred.append(o.data.cpu().numpy())
return np.vstack(pred)
def validate(model, val_loader):
model.eval()
l2norm = Metirc()
for data in val_loader:
x = torch.autograd.Variable(data.cuda())
y = model(x)
l2norm.update(y.data.cpu().numpy(), x.data.cpu().numpy())
return l2norm.get()
def test(epoch):
model.eval()
test_loss = 0
for data, _ in test_loader:
if args.cuda:
data = data.cuda()
data = Variable(data, volatile=True)
recon_batch, mu, logvar = model(data)
test_loss += loss_function(recon_batch, data, mu, logvar).data[0]
test_loss /= len(test_loader.dataset)
print('====> Test set loss: {:.4f}'.format(test_loss))
def train(model, optimizer, train_loader):
model.train()
l2norm = Metirc()
for data in train_loader:
x = torch.autograd.Variable(data.cuda())
y = model(x)
loss = torch.mean(torch.pow(x - y, 2))
l2norm.update(x.data.cpu().numpy(), y.data.cpu().numpy())
optimizer.zero_grad()
loss.backward()
optimizer.step()
return l2norm.get()
def train(epoch):
for batch_idx, (data, target) in enumerate(train_loader):
if use_gpu:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
if batch_idx % 100 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data[0]))
def train(self, epoch):
self.model.train()
t = tqdm(self.train_loader)
for batch_idx, (data, target) in enumerate(t):
if self.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
self.optimizer.zero_grad()
output = self.model(data)
loss = self.loss_fn(output, target)
self.loss.append(loss.cpu().data.numpy())
loss.backward()
self.optimizer.step()
t.set_postfix(loss=loss.data[0])
def test(self):
self.model.eval()
test_loss = 0
# correct = 0
# psnr = 0.0
for data, target in self.test_loader:
if self.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = self.model(data)
test_loss += self.loss_fn(output, target).data[0] # sum up batch loss
#psnr = 10 * log10(1 / test_loss.data[0])
#avg_psnr += psnr
test_loss /= len(self.test_loader.dataset)
print('\nTest set: Average loss: {0}'.format(test_loss))
self.loss_val.append(test_loss)
def test(epoch):
model.eval()
test_loss = 0
for i, (data, _) in enumerate(test_loader):
if args.cuda:
data = data.cuda()
data = Variable(data, volatile=True)
recon_batch, mu, logvar = model(data)
test_loss += loss_function(recon_batch, data, mu, logvar).data[0]
if i == 0:
n = min(data.size(0), 8)
comparison = torch.cat([data[:n],
recon_batch.view(args.batch_size, 1, 28, 28)[:n]])
save_image(comparison.data.cpu(),
'results/reconstruction_' + str(epoch) + '.png', nrow=n)
test_loss /= len(test_loader.dataset)
print('====> Test set loss: {:.4f}'.format(test_loss))
def test(epoch, model, dataloader, loss_F):
start = time.time()
model.train()
total_loss = 0.
t1_error = 0.
t5_error = 0.
for index, (data, label) in enumerate(dataloader):
if torch.cuda.is_available():
data = data.cuda()
label = label.cuda()
pred = model(data)
loss = loss_F(pred, label)
total_loss += loss.data.item()
t1_e, t5_e = calc_precision(pred, label)
t1_error += t1_e
t5_error += t5_e
t1_error /= len(dataloader)
t5_error /= len(dataloader)
epoch_val_error.append([t1_error, t5_error])
avg_loss = total_loss / len(dataloader)
epoch_val_loss.append(avg_loss)
print("[val] epoch: {0}, loss: {1}, top1 error: {2}, top5 error: {3}, LR: {4}, time: {5}".format(
epoch, avg_loss, t1_error, t5_error, LR, (time.time() - start)))
if viz and viz.check_connection():
global val_win, val_precision_win
val_win = viz.line(
epoch_val_loss, list(range(epoch + 1)),
win=val_win, name='val_loss',
opts=win_opts['val'], env=session)
val_precision_win = viz.line(
epoch_val_error, list(range(epoch+1)), win=val_precision_win, name='error', opts=win_opts['val_precision'],
env=session
)
# val_precision_win = viz.line(
# epoch_val_t5, list(range(epoch+1)), win=val_precision_win, name='top 5', opts=win_opts['val_precision'],
# env=session
# )
with open(model_path + '/log.txt', 'a+') as f:
f.write("[val] epoch: {0}, loss{1}, t1_error: {2}, t5_error: {3}, LR: {4}\n".format(
epoch, avg_loss, t1_error, t5_error, LR))
def train(epoch):
model.train()
train_loss = 0
for batch_idx, (data, _) in enumerate(train_loader):
data = Variable(data)
if args.cuda:
data = data.cuda()
optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
loss = loss_function(recon_batch, data, mu, logvar)
loss.backward()
train_loss += loss.data[0]
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader),
loss.data[0] / len(data)))
print('====> Epoch: {} Average loss: {:.4f}'.format(
epoch, train_loss / len(train_loader.dataset)))
def run_from_loader(loader,loss_lst=[], acc_lst=[]):
t_loss = 0
correct = 0
total = 0
for batch_idx, (data, target) in enumerate(loader):
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = C(data)
loss = criterion(output, target)
t_loss += loss.data[0]*target.size(0) # sum up batch loss
pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
total += target.size(0)
t_loss /= total
loss_lst.append(t_loss)
t_accuracy = 100. * correct / total
acc_lst.append(t_accuracy)
return t_loss, t_accuracy
def acc_test(epoch, vp):
vp.model.eval()
test_loss = 0
correct = 0
logsoftmax = nn.LogSoftmax().cuda()
confusions = {}
confusion_transitions = {}
mutex = Lock()
for data, target, frame_num, game_id in vp.testloader:
target = target.view(-1)
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = vp.model(data)
test_loss += F.nll_loss(logsoftmax(output), target).data[0]
pred = output.data.max(1)[1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
flat_pred = pred.cpu().numpy().flatten()
np_target = target.data.cpu().numpy()
mutex.acquire()
try:
for i in range(np_target.shape[0]):
if game_id[i] not in confusions:
conf_mat = np.zeros((3,3))
confusions[game_id[i]] = conf_mat
if game_id[i] not in confusion_transitions:
conf_mat = np.zeros((9,9))
confusion_transitions[game_id[i]] = conf_mat
finally:
mutex.release()
prev_pred = flat_pred[0]
prev_gt = np_target[0]
for i in range(np_target.shape[0]):
cc = confusions[game_id[i]]
cc[np_target[i], flat_pred[i]] += 1
if i > 0:
from_index = prev_pred % 3+flat_pred[i]*3
to_index = prev_gt % 3 + np_target[i]*3
dc = confusion_transitions[game_id[i]]
dc[to_index, from_index] += 1
prev_pred = flat_pred[i]
prev_gt = np_target[i]
test_loss = test_loss
test_loss /= len(vp.testloader) # loss function already averages over batch size
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(vp.testloader.dataset),
100. * correct / len(vp.testloader.dataset)))
print("-- Confusion Matrices --")
ultimate_matrix = np.zeros((3,3))
for key in confusions:
print key
print(confusions[key])
ultimate_matrix = np.add(ultimate_matrix,confusions[key])
print "average matrix"
print ultimate_matrix
print np.multiply(ultimate_matrix,1.0/len(vp.testloader.dataset))
for key in confusion_transitions:
print key
print confusion_transitions[key]
def train(model, train_loader, valid_loader, num_epochs=5, learning_rate=1e-4):
start_time = time.time()
print("Training Started...")
torch.manual_seed(42)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
# Store loss and accuracy in lists for each epoch
train_acc, val_acc, train_loss, val_loss = [], [], [], []
for epoch in range(num_epochs):
total_train_loss = 0.0
total_val_loss = 0.0
n = 1
i = 1
for data in train_loader:
if use_cuda and torch.cuda.is_available():
data = data.cuda() # send data to cuda
datam = zero_out_random_feature(data.clone(
)) # zero out one categorical feature on a cloned item
recon = model(datam) # pass through model
loss = criterion(recon,
data) # compare ground truth with prediction
loss.backward() # back propagation
optimizer.step() # update weights
optimizer.zero_grad() # zero gradient
total_train_loss += loss.item()
n = n + 1
# tracking validation loss
train_loss.append(total_train_loss / n)
for data in valid_loader:
if use_cuda and torch.cuda.is_available():
data = data.cuda() # send data to cuda
datam = zero_out_random_feature(data.clone(
)) # zero out one categorical feature on a cloned item
recon = model(datam) # pass through model
loss = criterion(recon,
data) # compare ground truth with prediction
total_val_loss += loss.item()
i = i + 1
val_loss.append(total_val_loss / i)
# tracking accuracy
train_acc.append(get_accuracy(model, train_loader))
val_acc.append(get_accuracy(model, valid_loader))
print(epoch, train_loss[-1], val_loss[-1], train_acc[-1], val_acc[-1])
n = len(train_acc)
print('Finished Training')
end_time = time.time()
elapsed_time = end_time - start_time
print("Total time elapsed: {:.2f} seconds".format(elapsed_time))
# plot loss
plt.title("Epoch vs Loss")
plt.plot(range(0, n), train_loss,
label='Train') # plotting the training accuracy
plt.plot(range(0, n), val_loss,
label='Validation') # plotting the validation accuracy
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.legend(loc='upper right')
plt.show()
print("Final Train Loss: {}".format(train_loss[-1]))
print("Final Validation Loss: {}".format(val_loss[-1]))
# plot accuracy
plt.title("Epoch vs Accuracy")
plt.plot(range(0, n), train_acc,
label='Train') # plotting the training accuracy
plt.plot(range(0, n), val_acc,
label='Validation') # plotting the validation accuracy
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.legend(loc='lower right')
plt.show()
print("Final Train Accuracy: {}".format(train_acc[-1]))
print("Final Validation Accuracy: {}".format(val_acc[-1]))
def evaluate(self):
self.model.eval()
std_loss = Accumulator('std_loss')
adv_loss = Accumulator('adv_loss')
std_corr = Accumulator('std_corr')
adv_corr = Accumulator('adv_corr')
std_logits = Accumulator('std_logits')
adv_logits = Accumulator('adv_logits')
seen_classes = []
adv_images = Accumulator('adv_images')
first_batch_images = Accumulator('first_batch_images')
from PIL import Image
for batch_idx, (data, target) in enumerate(self.val_loader[0]):
if self.cuda:
data, target = data.cuda(non_blocking=True), target.cuda(
non_blocking=True)
with torch.no_grad():
std_cpy = data.clone().detach(
) # std_cpy is used for finding the standard accuracy and has transforms applied as normal
output = self.model(std_cpy)
std_logits.update(output.cpu())
loss = F.cross_entropy(output, target, reduction='none').cpu()
std_loss.update(loss)
corr = correct(output, target)
corr = corr.view(corr.size()[0]).cpu()
std_corr.update(corr)
run_output = {'std_loss': std_loss.avg, 'std_acc': std_corr.avg}
print('Standard Batch', batch_idx)
print(run_output)
for batch_idx, (data, target) in enumerate(self.val_loader[1]):
# data is normalized at this point
if self.cuda:
data, target = data.cuda(non_blocking=True), target.cuda(
non_blocking=True)
rand_target = torch.randint(0,
self.nb_classes - 1,
target.size(),
dtype=target.dtype,
device='cuda')
rand_target = torch.remainder(target + rand_target + 1,
self.nb_classes)
data_cpy = data.clone().detach()
for idx in range(len(data_cpy)):
unnormalized = reverse_normalization(data[idx])
changed = np.swapaxes(
np.array(unnormalized.cpu().detach()) * 255.0, 0, 2)
transformed = applyTransforms(
np.swapaxes(
np.array(unnormalized.cpu().clone().detach()) * 255.0,
0, 2))
data_cpy[idx] = transforms.functional.normalize(
transformed.clone().cpu(), IMAGENET_MEAN,
IMAGENET_STD).cuda()
data_adv = self.attack(self.model,
data_cpy,
rand_target,
avoid_target=False,
scale_eps=False)
with torch.no_grad():
output_adv = self.model(data_adv)
adv_logits.update(output_adv.cpu())
loss = F.cross_entropy(output_adv, target,
reduction='none').cpu()
adv_loss.update(loss)
corr = correct(output_adv, target)
corr = corr.view(corr.size()[0]).cpu()
adv_corr.update(corr)
run_output = {'adv_loss': adv_loss.avg, 'adv_acc': adv_corr.avg}
print('Adv Batch', batch_idx)
print(run_output)
summary_dict = {
'std_acc': std_corr.avg.item(),
'adv_acc': adv_corr.avg.item()
}
print(std_loss.avg, std_corr.avg, adv_loss.avg, adv_corr.avg)
def train(model, train_x, train_y, valid_x=[], valid_y=[],
path_to_load_variables='', path_to_save_variables='',
epochs=10, batch_size=20, display_epoch=2, k=1):
if path_to_load_variables != '':
model.load_state_dict(torch.load(path_to_load_variables))
print 'loaded variables ' + path_to_load_variables
# # own_state =
# print len(model.state_dict())
# fds
# for name, param in model.state_dict():
# print name, param
# # if name not in own_state:
# # continue
# # if isinstance(param, Parameter):
# # # backwards compatibility for serialized parameters
# # param = param.data
# # own_state[name].copy_(param)
# fasdfasd
train = torch.utils.data.TensorDataset(train_x, train_y)
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
# params = list(model.parameters()) + list(model.decoder.params)
# print 'Params'
# for parameter_i in range(len(params)):
# print parameter_i, params[parameter_i].size()
# # print params[parameter_i]
# # print(parameter.size())
# print 'Params222'
# for parameter in model.parameters():
# print(parameter.size())
# optimizer = optim.Adam(params, lr=.0001)
optimizer = optim.Adam(model.parameters(), lr=.001)
# print 'passed'
if torch.cuda.is_available():
print 'GPU available, loading cuda'#, torch.cuda.is_available()
model.cuda()
for epoch in range(1, epochs + 1):
for batch_idx, (data, target) in enumerate(train_loader):
if torch.cuda.is_available():
data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor)
else:
data = Variable(data)#, Variable(target)
optimizer.zero_grad()
# start = time.time()
# print 'forward'
elbo, logpx, logpz, logqz, logpW, logqW = model.forward(data, k=k)
loss = -(elbo)
# print 'backward', epoch, display_epoch, batch_idx
# print(time.time() - start), 'forward'
# start = time.time()
loss.backward()
# print(time.time() - start), 'backward'
# start = time.time()
optimizer.step()
# print(time.time() - start), 'step'
if epoch%display_epoch==0 and batch_idx == 0:
print 'Train Epoch: {}/{} [{}/{} ({:.0f}%)]'.format(epoch, epochs,
batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader)), \
'Loss:{:.3f}'.format(loss.data[0]), \
'logpx:{:.3f}'.format(logpx.data[0]), \
'logpz:{:.3f}'.format(logpz.data[0]), \
'logqz:{:.3f}'.format(logqz.data[0]), \
'logpW:{:.3f}'.format(logpW.data[0]), \
'logqW:{:.3f}'.format(logqW.data[0])
if path_to_save_variables != '':
torch.save(model.state_dict(), path_to_save_variables)
print 'Saved variables to ' + path_to_save_variables
def train(model, train_x, train_y, valid_x=[], valid_y=[],
path_to_load_variables='', path_to_save_variables='',
epochs=10, batch_size=20, display_epoch=2, k=1):
if path_to_load_variables != '':
model.load_state_dict(torch.load(path_to_load_variables))
print 'loaded variables ' + path_to_load_variables
traindata = torch.utils.data.TensorDataset(train_x, train_y)
train_loader = torch.utils.data.DataLoader(traindata, batch_size=batch_size, shuffle=True)
validdata = torch.utils.data.TensorDataset(valid_x, valid_y)
valid_loader = torch.utils.data.DataLoader(validdata, batch_size=batch_size, shuffle=False)
optimizer = optim.Adam(model.parameters(), lr=.001)
if torch.cuda.is_available():
print 'GPU available, loading cuda'
model.cuda()
times_not_beaten_best = 0
finished = 0
best_valid_elbo = -1
for epoch in range(1, epochs + 1):
for batch_idx, (data, target) in enumerate(train_loader):
if torch.cuda.is_available():
data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor)
else:
data = Variable(data)#, Variable(target)
optimizer.zero_grad()
elbo, logpx, logpz, logqz, logpW, logqW = model.forward(data, k=k, s=1)
loss = -(elbo)
loss.backward()
optimizer.step()
if epoch%display_epoch==0 and batch_idx == 0:
print 'Train Epoch: {}/{}'.format(epoch, epochs), \
'Loss:{:.3f}'.format(loss.data[0]), \
'logpx:{:.3f}'.format(logpx.data[0]), \
'logpz:{:.3f}'.format(logpz.data[0]), \
'logqz:{:.3f}'.format(logqz.data[0]), \
'logpW:{:.3f}'.format(logpW.data[0]), \
'logqW:{:.3f}'.format(logqW.data[0])
if len(valid_x) > 0:
valid_elbos = []
for batch_idx_valid, (data, target) in enumerate(valid_loader):
if torch.cuda.is_available():
data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor)
else:
data = Variable(data)#, Variable(target)
valid_elbo = model.predictive_elbo(data, k=20, s=3)
valid_elbos.append(valid_elbo.data[0])
current_valid_elbo = np.mean(valid_elbos)
if current_valid_elbo > best_valid_elbo or best_valid_elbo == -1:
best_valid_elbo = current_valid_elbo
times_not_beaten_best = 0
#save model
if path_to_save_variables != '':
torch.save(model.state_dict(), path_to_save_variables)
print 'validation ' + str(current_valid_elbo) + ' Saved variables to ' + path_to_save_variables
else:
times_not_beaten_best += 1
print 'validation ' + str(current_valid_elbo), times_not_beaten_best
if times_not_beaten_best > 3:
finished = 1
break
if finished ==1:
break
return best_valid_elbo
Variable(torch.zeros(data.size(0), 256, 8, 8).cuda()))
encoder_h_2 = (Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()),
Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()))
encoder_h_3 = (Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()),
Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()))
decoder_h_1 = (Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()),
Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()))
decoder_h_2 = (Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()),
Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()))
decoder_h_3 = (Variable(torch.zeros(data.size(0), 256, 8, 8).cuda()),
Variable(torch.zeros(data.size(0), 256, 8, 8).cuda()))
decoder_h_4 = (Variable(torch.zeros(data.size(0), 128, 16, 16).cuda()),
Variable(torch.zeros(data.size(0), 128, 16, 16).cuda()))
patches = Variable(data.cuda())
solver.zero_grad()
losses = []
res = patches - 0.5
bp_t0 = time.time()
for _ in range(args.iterations):
encoded, encoder_h_1, encoder_h_2, encoder_h_3 = encoder(
res, encoder_h_1, encoder_h_2, encoder_h_3)
codes = binarizer(encoded)
