def predict(self, data_loader): pred = [] for data in data_loader: x = torch.autograd.Variable(data.cuda()) o = self.h(x) pred.append(o.data.cpu().numpy()) return np.vstack(pred)
def validate(model, val_loader): model.eval() l2norm = Metirc() for data in val_loader: x = torch.autograd.Variable(data.cuda()) y = model(x) l2norm.update(y.data.cpu().numpy(), x.data.cpu().numpy()) return l2norm.get()
def test(epoch): model.eval() test_loss = 0 for data, _ in test_loader: if args.cuda: data = data.cuda() data = Variable(data, volatile=True) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).data[0] test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
def train(model, optimizer, train_loader): model.train() l2norm = Metirc() for data in train_loader: x = torch.autograd.Variable(data.cuda()) y = model(x) loss = torch.mean(torch.pow(x - y, 2)) l2norm.update(x.data.cpu().numpy(), y.data.cpu().numpy()) optimizer.zero_grad() loss.backward() optimizer.step() return l2norm.get()
def train(epoch): for batch_idx, (data, target) in enumerate(train_loader): if use_gpu: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = criterion(output, target) loss.backward() optimizer.step() if batch_idx % 100 == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data[0]))
def train(self, epoch): self.model.train() t = tqdm(self.train_loader) for batch_idx, (data, target) in enumerate(t): if self.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) self.optimizer.zero_grad() output = self.model(data) loss = self.loss_fn(output, target) self.loss.append(loss.cpu().data.numpy()) loss.backward() self.optimizer.step() t.set_postfix(loss=loss.data[0])
def test(self): self.model.eval() test_loss = 0 # correct = 0 # psnr = 0.0 for data, target in self.test_loader: if self.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = self.model(data) test_loss += self.loss_fn(output, target).data[0] # sum up batch loss #psnr = 10 * log10(1 / test_loss.data[0]) #avg_psnr += psnr test_loss /= len(self.test_loader.dataset) print('\nTest set: Average loss: {0}'.format(test_loss)) self.loss_val.append(test_loss)
def test(epoch): model.eval() test_loss = 0 for i, (data, _) in enumerate(test_loader): if args.cuda: data = data.cuda() data = Variable(data, volatile=True) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).data[0] if i == 0: n = min(data.size(0), 8) comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.data.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss))
def test(epoch, model, dataloader, loss_F): start = time.time() model.train() total_loss = 0. t1_error = 0. t5_error = 0. for index, (data, label) in enumerate(dataloader): if torch.cuda.is_available(): data = data.cuda() label = label.cuda() pred = model(data) loss = loss_F(pred, label) total_loss += loss.data.item() t1_e, t5_e = calc_precision(pred, label) t1_error += t1_e t5_error += t5_e t1_error /= len(dataloader) t5_error /= len(dataloader) epoch_val_error.append([t1_error, t5_error]) avg_loss = total_loss / len(dataloader) epoch_val_loss.append(avg_loss) print("[val] epoch: {0}, loss: {1}, top1 error: {2}, top5 error: {3}, LR: {4}, time: {5}".format( epoch, avg_loss, t1_error, t5_error, LR, (time.time() - start))) if viz and viz.check_connection(): global val_win, val_precision_win val_win = viz.line( epoch_val_loss, list(range(epoch + 1)), win=val_win, name='val_loss', opts=win_opts['val'], env=session) val_precision_win = viz.line( epoch_val_error, list(range(epoch+1)), win=val_precision_win, name='error', opts=win_opts['val_precision'], env=session ) # val_precision_win = viz.line( # epoch_val_t5, list(range(epoch+1)), win=val_precision_win, name='top 5', opts=win_opts['val_precision'], # env=session # ) with open(model_path + '/log.txt', 'a+') as f: f.write("[val] epoch: {0}, loss{1}, t1_error: {2}, t5_error: {3}, LR: {4}\n".format( epoch, avg_loss, t1_error, t5_error, LR))
def train(epoch): model.train() train_loss = 0 for batch_idx, (data, _) in enumerate(train_loader): data = Variable(data) if args.cuda: data = data.cuda() optimizer.zero_grad() recon_batch, mu, logvar = model(data) loss = loss_function(recon_batch, data, mu, logvar) loss.backward() train_loss += loss.data[0] optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data[0] / len(data))) print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset)))
def run_from_loader(loader,loss_lst=[], acc_lst=[]): t_loss = 0 correct = 0 total = 0 for batch_idx, (data, target) in enumerate(loader): if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = C(data) loss = criterion(output, target) t_loss += loss.data[0]*target.size(0) # sum up batch loss pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability correct += pred.eq(target.data.view_as(pred)).cpu().sum() total += target.size(0) t_loss /= total loss_lst.append(t_loss) t_accuracy = 100. * correct / total acc_lst.append(t_accuracy) return t_loss, t_accuracy
def acc_test(epoch, vp): vp.model.eval() test_loss = 0 correct = 0 logsoftmax = nn.LogSoftmax().cuda() confusions = {} confusion_transitions = {} mutex = Lock() for data, target, frame_num, game_id in vp.testloader: target = target.view(-1) data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = vp.model(data) test_loss += F.nll_loss(logsoftmax(output), target).data[0] pred = output.data.max(1)[1] # get the index of the max log-probability correct += pred.eq(target.data).cpu().sum() flat_pred = pred.cpu().numpy().flatten() np_target = target.data.cpu().numpy() mutex.acquire() try: for i in range(np_target.shape[0]): if game_id[i] not in confusions: conf_mat = np.zeros((3,3)) confusions[game_id[i]] = conf_mat if game_id[i] not in confusion_transitions: conf_mat = np.zeros((9,9)) confusion_transitions[game_id[i]] = conf_mat finally: mutex.release() prev_pred = flat_pred[0] prev_gt = np_target[0] for i in range(np_target.shape[0]): cc = confusions[game_id[i]] cc[np_target[i], flat_pred[i]] += 1 if i > 0: from_index = prev_pred % 3+flat_pred[i]*3 to_index = prev_gt % 3 + np_target[i]*3 dc = confusion_transitions[game_id[i]] dc[to_index, from_index] += 1 prev_pred = flat_pred[i] prev_gt = np_target[i] test_loss = test_loss test_loss /= len(vp.testloader) # loss function already averages over batch size print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(vp.testloader.dataset), 100. * correct / len(vp.testloader.dataset))) print("-- Confusion Matrices --") ultimate_matrix = np.zeros((3,3)) for key in confusions: print key print(confusions[key]) ultimate_matrix = np.add(ultimate_matrix,confusions[key]) print "average matrix" print ultimate_matrix print np.multiply(ultimate_matrix,1.0/len(vp.testloader.dataset)) for key in confusion_transitions: print key print confusion_transitions[key]
def train(model, train_loader, valid_loader, num_epochs=5, learning_rate=1e-4): start_time = time.time() print("Training Started...") torch.manual_seed(42) criterion = nn.MSELoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # Store loss and accuracy in lists for each epoch train_acc, val_acc, train_loss, val_loss = [], [], [], [] for epoch in range(num_epochs): total_train_loss = 0.0 total_val_loss = 0.0 n = 1 i = 1 for data in train_loader: if use_cuda and torch.cuda.is_available(): data = data.cuda() # send data to cuda datam = zero_out_random_feature(data.clone( )) # zero out one categorical feature on a cloned item recon = model(datam) # pass through model loss = criterion(recon, data) # compare ground truth with prediction loss.backward() # back propagation optimizer.step() # update weights optimizer.zero_grad() # zero gradient total_train_loss += loss.item() n = n + 1 # tracking validation loss train_loss.append(total_train_loss / n) for data in valid_loader: if use_cuda and torch.cuda.is_available(): data = data.cuda() # send data to cuda datam = zero_out_random_feature(data.clone( )) # zero out one categorical feature on a cloned item recon = model(datam) # pass through model loss = criterion(recon, data) # compare ground truth with prediction total_val_loss += loss.item() i = i + 1 val_loss.append(total_val_loss / i) # tracking accuracy train_acc.append(get_accuracy(model, train_loader)) val_acc.append(get_accuracy(model, valid_loader)) print(epoch, train_loss[-1], val_loss[-1], train_acc[-1], val_acc[-1]) n = len(train_acc) print('Finished Training') end_time = time.time() elapsed_time = end_time - start_time print("Total time elapsed: {:.2f} seconds".format(elapsed_time)) # plot loss plt.title("Epoch vs Loss") plt.plot(range(0, n), train_loss, label='Train') # plotting the training accuracy plt.plot(range(0, n), val_loss, label='Validation') # plotting the validation accuracy plt.xlabel("Epoch") plt.ylabel("Loss") plt.legend(loc='upper right') plt.show() print("Final Train Loss: {}".format(train_loss[-1])) print("Final Validation Loss: {}".format(val_loss[-1])) # plot accuracy plt.title("Epoch vs Accuracy") plt.plot(range(0, n), train_acc, label='Train') # plotting the training accuracy plt.plot(range(0, n), val_acc, label='Validation') # plotting the validation accuracy plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.legend(loc='lower right') plt.show() print("Final Train Accuracy: {}".format(train_acc[-1])) print("Final Validation Accuracy: {}".format(val_acc[-1]))
def evaluate(self): self.model.eval() std_loss = Accumulator('std_loss') adv_loss = Accumulator('adv_loss') std_corr = Accumulator('std_corr') adv_corr = Accumulator('adv_corr') std_logits = Accumulator('std_logits') adv_logits = Accumulator('adv_logits') seen_classes = [] adv_images = Accumulator('adv_images') first_batch_images = Accumulator('first_batch_images') from PIL import Image for batch_idx, (data, target) in enumerate(self.val_loader[0]): if self.cuda: data, target = data.cuda(non_blocking=True), target.cuda( non_blocking=True) with torch.no_grad(): std_cpy = data.clone().detach( ) # std_cpy is used for finding the standard accuracy and has transforms applied as normal output = self.model(std_cpy) std_logits.update(output.cpu()) loss = F.cross_entropy(output, target, reduction='none').cpu() std_loss.update(loss) corr = correct(output, target) corr = corr.view(corr.size()[0]).cpu() std_corr.update(corr) run_output = {'std_loss': std_loss.avg, 'std_acc': std_corr.avg} print('Standard Batch', batch_idx) print(run_output) for batch_idx, (data, target) in enumerate(self.val_loader[1]): # data is normalized at this point if self.cuda: data, target = data.cuda(non_blocking=True), target.cuda( non_blocking=True) rand_target = torch.randint(0, self.nb_classes - 1, target.size(), dtype=target.dtype, device='cuda') rand_target = torch.remainder(target + rand_target + 1, self.nb_classes) data_cpy = data.clone().detach() for idx in range(len(data_cpy)): unnormalized = reverse_normalization(data[idx]) changed = np.swapaxes( np.array(unnormalized.cpu().detach()) * 255.0, 0, 2) transformed = applyTransforms( np.swapaxes( np.array(unnormalized.cpu().clone().detach()) * 255.0, 0, 2)) data_cpy[idx] = transforms.functional.normalize( transformed.clone().cpu(), IMAGENET_MEAN, IMAGENET_STD).cuda() data_adv = self.attack(self.model, data_cpy, rand_target, avoid_target=False, scale_eps=False) with torch.no_grad(): output_adv = self.model(data_adv) adv_logits.update(output_adv.cpu()) loss = F.cross_entropy(output_adv, target, reduction='none').cpu() adv_loss.update(loss) corr = correct(output_adv, target) corr = corr.view(corr.size()[0]).cpu() adv_corr.update(corr) run_output = {'adv_loss': adv_loss.avg, 'adv_acc': adv_corr.avg} print('Adv Batch', batch_idx) print(run_output) summary_dict = { 'std_acc': std_corr.avg.item(), 'adv_acc': adv_corr.avg.item() } print(std_loss.avg, std_corr.avg, adv_loss.avg, adv_corr.avg)
def train(model, train_x, train_y, valid_x=[], valid_y=[], path_to_load_variables='', path_to_save_variables='', epochs=10, batch_size=20, display_epoch=2, k=1): if path_to_load_variables != '': model.load_state_dict(torch.load(path_to_load_variables)) print 'loaded variables ' + path_to_load_variables # # own_state = # print len(model.state_dict()) # fds # for name, param in model.state_dict(): # print name, param # # if name not in own_state: # # continue # # if isinstance(param, Parameter): # # # backwards compatibility for serialized parameters # # param = param.data # # own_state[name].copy_(param) # fasdfasd train = torch.utils.data.TensorDataset(train_x, train_y) train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True) # params = list(model.parameters()) + list(model.decoder.params) # print 'Params' # for parameter_i in range(len(params)): # print parameter_i, params[parameter_i].size() # # print params[parameter_i] # # print(parameter.size()) # print 'Params222' # for parameter in model.parameters(): # print(parameter.size()) # optimizer = optim.Adam(params, lr=.0001) optimizer = optim.Adam(model.parameters(), lr=.001) # print 'passed' if torch.cuda.is_available(): print 'GPU available, loading cuda'#, torch.cuda.is_available() model.cuda() for epoch in range(1, epochs + 1): for batch_idx, (data, target) in enumerate(train_loader): if torch.cuda.is_available(): data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor) else: data = Variable(data)#, Variable(target) optimizer.zero_grad() # start = time.time() # print 'forward' elbo, logpx, logpz, logqz, logpW, logqW = model.forward(data, k=k) loss = -(elbo) # print 'backward', epoch, display_epoch, batch_idx # print(time.time() - start), 'forward' # start = time.time() loss.backward() # print(time.time() - start), 'backward' # start = time.time() optimizer.step() # print(time.time() - start), 'step' if epoch%display_epoch==0 and batch_idx == 0: print 'Train Epoch: {}/{} [{}/{} ({:.0f}%)]'.format(epoch, epochs, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader)), \ 'Loss:{:.3f}'.format(loss.data[0]), \ 'logpx:{:.3f}'.format(logpx.data[0]), \ 'logpz:{:.3f}'.format(logpz.data[0]), \ 'logqz:{:.3f}'.format(logqz.data[0]), \ 'logpW:{:.3f}'.format(logpW.data[0]), \ 'logqW:{:.3f}'.format(logqW.data[0]) if path_to_save_variables != '': torch.save(model.state_dict(), path_to_save_variables) print 'Saved variables to ' + path_to_save_variables
def train(model, train_x, train_y, valid_x=[], valid_y=[], path_to_load_variables='', path_to_save_variables='', epochs=10, batch_size=20, display_epoch=2, k=1): if path_to_load_variables != '': model.load_state_dict(torch.load(path_to_load_variables)) print 'loaded variables ' + path_to_load_variables traindata = torch.utils.data.TensorDataset(train_x, train_y) train_loader = torch.utils.data.DataLoader(traindata, batch_size=batch_size, shuffle=True) validdata = torch.utils.data.TensorDataset(valid_x, valid_y) valid_loader = torch.utils.data.DataLoader(validdata, batch_size=batch_size, shuffle=False) optimizer = optim.Adam(model.parameters(), lr=.001) if torch.cuda.is_available(): print 'GPU available, loading cuda' model.cuda() times_not_beaten_best = 0 finished = 0 best_valid_elbo = -1 for epoch in range(1, epochs + 1): for batch_idx, (data, target) in enumerate(train_loader): if torch.cuda.is_available(): data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor) else: data = Variable(data)#, Variable(target) optimizer.zero_grad() elbo, logpx, logpz, logqz, logpW, logqW = model.forward(data, k=k, s=1) loss = -(elbo) loss.backward() optimizer.step() if epoch%display_epoch==0 and batch_idx == 0: print 'Train Epoch: {}/{}'.format(epoch, epochs), \ 'Loss:{:.3f}'.format(loss.data[0]), \ 'logpx:{:.3f}'.format(logpx.data[0]), \ 'logpz:{:.3f}'.format(logpz.data[0]), \ 'logqz:{:.3f}'.format(logqz.data[0]), \ 'logpW:{:.3f}'.format(logpW.data[0]), \ 'logqW:{:.3f}'.format(logqW.data[0]) if len(valid_x) > 0: valid_elbos = [] for batch_idx_valid, (data, target) in enumerate(valid_loader): if torch.cuda.is_available(): data = Variable(data.cuda())#, Variable(target)#.type(torch.cuda.LongTensor) else: data = Variable(data)#, Variable(target) valid_elbo = model.predictive_elbo(data, k=20, s=3) valid_elbos.append(valid_elbo.data[0]) current_valid_elbo = np.mean(valid_elbos) if current_valid_elbo > best_valid_elbo or best_valid_elbo == -1: best_valid_elbo = current_valid_elbo times_not_beaten_best = 0 #save model if path_to_save_variables != '': torch.save(model.state_dict(), path_to_save_variables) print 'validation ' + str(current_valid_elbo) + ' Saved variables to ' + path_to_save_variables else: times_not_beaten_best += 1 print 'validation ' + str(current_valid_elbo), times_not_beaten_best if times_not_beaten_best > 3: finished = 1 break if finished ==1: break return best_valid_elbo
Variable(torch.zeros(data.size(0), 256, 8, 8).cuda())) encoder_h_2 = (Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()), Variable(torch.zeros(data.size(0), 512, 4, 4).cuda())) encoder_h_3 = (Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()), Variable(torch.zeros(data.size(0), 512, 2, 2).cuda())) decoder_h_1 = (Variable(torch.zeros(data.size(0), 512, 2, 2).cuda()), Variable(torch.zeros(data.size(0), 512, 2, 2).cuda())) decoder_h_2 = (Variable(torch.zeros(data.size(0), 512, 4, 4).cuda()), Variable(torch.zeros(data.size(0), 512, 4, 4).cuda())) decoder_h_3 = (Variable(torch.zeros(data.size(0), 256, 8, 8).cuda()), Variable(torch.zeros(data.size(0), 256, 8, 8).cuda())) decoder_h_4 = (Variable(torch.zeros(data.size(0), 128, 16, 16).cuda()), Variable(torch.zeros(data.size(0), 128, 16, 16).cuda())) patches = Variable(data.cuda()) solver.zero_grad() losses = [] res = patches - 0.5 bp_t0 = time.time() for _ in range(args.iterations): encoded, encoder_h_1, encoder_h_2, encoder_h_3 = encoder( res, encoder_h_1, encoder_h_2, encoder_h_3) codes = binarizer(encoded)