def train(self, epochs, learning_rate, kernel_size, hidden_size, model_cls, interaction, dropout): if model_cls == "cnn": model = CNN(embedding=self.data_train.vocab_embedding, embedding_size=self.data_train.vocab_embedding_size, lengths=self.data_train.lengths(), kernel_size=kernel_size, hidden_size=hidden_size, interaction=interaction, dropout=dropout) else: model = GRU(embedding=self.data_train.vocab_embedding, embedding_size=self.data_train.vocab_embedding_size, encoding_size=hidden_size, interaction=interaction, dropout=dropout) if self.use_gpu: model = model.cuda() loader = self.data_train.get_loader() loss_fn = torch.nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) losses = [] accuracies = [] for epoch in range(1, epochs + 1): e_loss = [] print("\nStarting epoch {}".format(epoch)) for i, (s1, s2, labels) in enumerate(loader): if self.use_gpu: s1, s2, labels = s1.cuda(), s2.cuda(), labels.cuda() model.train() optimizer.zero_grad() # Forward pass logits = model(s1, s2) instance_loss = loss_fn(logits, labels) # Backward and optimize instance_loss.backward() optimizer.step() losses.append(instance_loss.item()) e_loss.append(instance_loss.item()) # validate every 100 iterations if i > 0 and i % 100 == 0: val_acc = self.validate(model) accuracies.append(val_acc) print( 'Epoch: [{}/{}]\tStep: [{}/{}]\tValidation Acc: {:.4f}' .format(epoch, epochs, i, len(loader), val_acc)) # self.analyzer.plot_live_lr(e_loss, title="Epoch {}".format(epoch)) avg_acc = sum(accuracies[-5:]) / 5 self.analyzer.record(model.cpu(), losses, epochs=epochs, accuracies=accuracies, learning_rate=learning_rate, hidden_size=hidden_size, kernel_size=kernel_size, validation_accuracy=avg_acc, model_name=model_cls, dropout=dropout, interaction=interaction, data_length=32 * len(loader)) self.analyzer.print_validation_results(self, model_cls, model) print("Final Accuracy: {}".format(avg_acc))
def train(args): device = args.device train_lines = open(args.train_file).readlines() val_lines = open(args.val_file).readlines() log_every = args.log_every valid_iter = args.valid_iter train_iter = 0 cum_loss = 0 avg_loss = 0 valid_num = 0 patience = 0 num_trial = 0 hist_valid_scores = [] begin_time = time.time() vocab = get_vocab(args.vocab_file) model = CNN(args, vocab) if args.use_embed == 1: model.load_vector(args, vocab) if args.device == 'cuda': model.cuda() lr = args.lr optim = torch.optim.Adam(list(model.parameters()), lr=lr) criterion = torch.nn.CrossEntropyLoss().to(device=device) model.train() for ep in range(args.max_epochs): train_iter = 0 val_iter = 0 for examples, labels in batch_iter(train_lines, vocab, args.batch_size, \ args.max_sent_len, shuffle=True): train_iter += 1 optim.zero_grad() labels = torch.tensor(labels).to(device=device) examples = torch.tensor(examples).to(device=device) output = model(examples) loss = criterion(output, labels) avg_loss += loss.item() cum_loss += loss.item() loss.backward() torch.nn.utils.clip_grad_norm_(list(model.parameters()), args.clip_grad) optim.step() if train_iter % log_every == 0: print('epoch %d, iter %d, avg.loss %.2f, time elapsed %.2f'\ % (ep + 1, train_iter, avg_loss / log_every, time.time() - begin_time), file=sys.stderr) begin_time = time.time() avg_loss = 0 if train_iter % valid_iter == 0: print('epoch %d, iter %d, cum.loss %.2f, time elapsed %.2f'\ % (ep + 1, train_iter, cum_loss / valid_iter, time.time() - begin_time), file=sys.stderr) cum_loss = 0 valid_num += 1 print("Begin Validation ", file=sys.stderr) model.eval() acc = test(val_lines, model, vocab, args) model.train() print('validation: iter %d, acc %f' % (train_iter, acc), file=sys.stderr) is_better = (len(hist_valid_scores) == 0) or (acc > max(hist_valid_scores)) hist_valid_scores.append(acc) if is_better: patience = 0 print("Save the current model and optimiser state") torch.save(model, args.model_save_path) torch.save(optim.state_dict(), args.model_save_path + '.optim') elif patience < args.patience: patience += 1 print('hit patience %d' % patience, file=sys.stderr) if patience == args.patience: num_trial += 1 print('hit #%d trial' % num_trial, file=sys.stderr) if num_trial == args.max_num_trials: print('early stop!', file=sys.stderr) return lr = lr * args.lr_decay print( 'load previously best model and decay learning rate to %f' % lr, file=sys.stderr) model = load(args.model_save_path) print('restore parameters of the optimizers', file=sys.stderr) optim = torch.optim.Adam(list(model.parameters()), lr=lr) optim.load_state_dict( torch.load(args.model_save_path + '.optim')) for state in optim.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.to(args.device) for group in optim.param_groups: group['lr'] = lr patience = 0 print("Training Finished", file=sys.stderr)
def get_reward(self,action,pre_acc,trainset,validset): action=[action[0][0][x:x+4] for x in range(0, len(action[0][0]),4)] model = CNN(num_input=1,num_classes=10,action=action) criterion = nn.CrossEntropyLoss() model = model.cuda() criterion = criterion.cuda() cudnn.benchmark = True optimizer = torch.optim.Adam(model.parameters(),self.learning_rate) model.train() for i in range(self.max_step_per_action): for steps, (input, target) in enumerate(trainset): input = Variable(input, requires_grad=False) input = input.cuda(non_blocking=True) target = Variable(target, requires_grad=False) target = target.cuda(non_blocking=True) logits = model(input) loss = criterion(logits,target) #loss = criterion(logits, target) optimizer.zero_grad() loss.backward() optimizer.step() if(steps % 200 == 0): stepss = steps + i*500 print("Step " + str(stepss) + ", Minibatch Loss= " + "{:.4f}".format(loss)) #if (steps == 3): # break; test_loss =0 correct =0 stepp = 0 for steps, (input, target) in enumerate(validset): input = Variable(input, requires_grad=False) input = input.cuda(non_blocking=True) target = Variable(target, requires_grad=False) target = target.cuda(non_blocking=True) output =model(input) test_loss += F.nll_loss(output,target, reduction='sum').item() # sum up batch loss pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability correct += pred.eq(target.view_as(pred)).sum().item() # 몇개나 같은지,,, stepp = steps # validset 개수 #if(steps == 2): # break; test_loss /= (stepp+1) acc_ = correct / ((stepp+1)*100) # step별 100개 있으니까.. print("validation accuracy : "+ str(acc_)) if acc_ - pre_acc <= 0.01: return acc_, acc_ else: return 0.01, acc_
def parse_individual(self, indi): torch_device = torch.device('cuda') cnn = CNN(indi) cnn.cuda() print(cnn) complexity = get_total_params(cnn.cuda(), (220, 30, 30)) train_loader = get_data.get_mixed_train_loader(self.batch_size) # Loss and optimizer 3.定义损失函数, 使用的是最小平方误差函数 criterion = nn.MSELoss() criterion = criterion.to(torch_device) # 4.定义迭代优化算法, 使用的是Adam,SGD不行 learning_rate = 0.004 optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) loss_dict = [] num_epochs = train_loader.__len__() # Train the model 5. 迭代训练 cnn.train() for i, data in enumerate(train_loader, 0): # Convert numpy arrays to torch tensors 5.1 准备tensor的训练数据和标签 inputs, labels = data labels = get_data.get_size_labels(1, labels) inputs = inputs.cuda() labels = labels.cuda() # labels = get_data.get_size_labels(indi.get_layer_size(),labels) # Forward pass 5.2 前向传播计算网络结构的输出结果 optimizer.zero_grad() outputs = cnn(inputs) # 5.3 计算损失函数 loss = criterion(outputs, labels) loss = loss.cuda() # Backward and optimize 5.4 反向传播更新参数 loss.backward() optimizer.step() # 可选 5.5 打印训练信息和保存loss loss_dict.append(loss.item()) if (i + 1) % 100 == 0: print('Epoch [{}/{}], Loss: {:.4f}'.format(i + 1, num_epochs, loss.item())) # evaluate cnn.eval() eval_loss_dict = [] valid_loader = get_data.get_mixed_validate_loader(self.batch_size) for i, data in enumerate(valid_loader, 0): inputs, labels = data labels = get_data.get_size_labels(1, labels) inputs = inputs.cuda() labels = labels.cuda() outputs = cnn(inputs) loss = criterion(outputs, labels) loss = loss.cuda() eval_loss_dict.append(loss.item()) mean_test_loss = np.mean(eval_loss_dict) std_test_loss = np.std(eval_loss_dict) print("valid mean:{},std:{}".format(mean_test_loss, std_test_loss)) return mean_test_loss, std_test_loss, complexity
else: decomposer.replace_layer(keys=args.key, type=args.type) path_o = f'models/{args.model}_{args.factorization}_{args.key}_{args.type}.pth' torch.save(decomposer.model, path_o) logger.info('===========saved============') logger.info(f'saved to {path_o}') elif args.eval: path = f'models/{args.model}.pth' logger.info(f'model: {path}') model = torch.load(path) model.eval() device_name = args.device if device_name == 'cpu': model.cpu() use_cuda = False else: model.cuda() use_cuda = True device = torch.device(device_name) summary(model, (3, 224, 224), device=device_name) labels = ['cat', 'dog'] test_loader = load_test_dataset(args.test_path) accuracy = predict_loader(model, test_loader, device, args.verbose) print('accuracy:', accuracy)
import torch.utils.data as Data import torchvision import matplotlib.pyplot as plt import numpy as np from sklearn.model_selection import train_test_split from cnn import CNN from math import inf torch.manual_seed(1) EPOCH = 10 LR = 0.001 if __name__ == "__main__": cnn = CNN() cnn.cuda() print(cnn) optimizer = torch.optim.Adam(cnn.parameters(), lr=LR) loss_function = nn.MSELoss() for epoch in range(EPOCH): best_test_loss = inf total_step = 0 for month in range(1, 7): print("Epoch: %d, Month: %d" % (epoch, month)) xs = np.load("dataset_month%d/xs.npy" % month) ys = np.load("dataset_month%d/ys.npy" % month) X_train, X_test, y_train, y_test = train_test_split( xs, ys, test_size=0.2, random_state=42)