def train_lenet(device, dataset_path): train_loader, valid_loader, test_loader = get_data_loaders(dataset_path) model = LeNet(35) optimizer = optim.Adam(model.parameters(), lr=Consts.lr, weight_decay=Consts.weight_decay) loss_criterion = torch.nn.NLLLoss() model.apply(weight_init) model.to(device) train_loss = [] val_loss = [] val_acc = [] for epoch in range(Consts.epochs): t_loss = train(model, train_loader, optimizer, loss_criterion, device) v_loss, v_acc = evaluation(model, valid_loader, loss_criterion, device) torch.save(model.state_dict(), f'models/epoch-{epoch + 1}.pth') train_loss.append(t_loss) val_loss.append(v_loss) val_acc.append(v_acc) print(f'train loss in epoch {epoch + 1} is: {t_loss}') print(f'validation loss in epoch {epoch + 1} is: {v_loss}') print(f'validation accuracy in epoch {epoch + 1} is: {v_acc}') plot_loss(train_loss, val_loss, val_acc) test_loss, test_acc = test_model(model, test_loader, loss_criterion, val_loss, device, 'models/')
def main(): transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) # 50000张训练图片 # 第一次使用时要将download设置为True才会自动去下载数据集 train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=False, transform=transform) train_loader = torch.utils.data.DataLoader(train_set, batch_size=36, shuffle=True, num_workers=0) # 10000张验证图片 # 第一次使用时要将download设置为True才会自动去下载数据集 val_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=False, transform=transform) val_loader = torch.utils.data.DataLoader(val_set, batch_size=5000, shuffle=False, num_workers=0) val_data_iter = iter(val_loader) val_image, val_label = val_data_iter.next() # classes = ('plane', 'car', 'bird', 'cat', # 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') net = LeNet() loss_function = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=0.001) for epoch in range(5): # loop over the dataset multiple times running_loss = 0.0 for step, data in enumerate(train_loader, start=0): # get the inputs; data is a list of [inputs, labels] inputs, labels = data # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = net(inputs) loss = loss_function(outputs, labels) loss.backward() optimizer.step() # print statistics running_loss += loss.item() if step % 500 == 499: # print every 500 mini-batches with torch.no_grad(): outputs = net(val_image) # [batch, 10] predict_y = torch.max(outputs, dim=1)[1] accuracy = (predict_y == val_label).sum().item() / val_label.size(0) print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' % (epoch + 1, step + 1, running_loss / 500, accuracy)) running_loss = 0.0 print('Finished Training') save_path = './Lenet.pth' torch.save(net.state_dict(), save_path)
def trainModel(EPOCH_NUM=50, save=False, show=False): print("Train Start:") net = LeNet().to(devices) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.RMSprop(net.parameters(), lr=LR, alpha=0.9, eps=1e-08, weight_decay=0, momentum=0, centered=False) #x,trainloss,trainacc,testacc = [],[],[],[] batch, batchloss = [], [] for epoch in range(EPOCH_NUM): sum_loss = 0.0 acc = 0 iter = 0 for i, (inputs, labels) in enumerate(trainLoader): inputs, labels = inputs.to(devices), labels.to(devices) # forward and backward optimizer.zero_grad() outputs = net(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() sum_loss += loss.item() _, pred = torch.max(outputs.data, 1) acc += (pred == labels).sum() iter = iter + 1 batch.append(i) batchloss.append(loss.item()) if show == True: plt.figure() plt.plot(batch, batchloss, 'b') plt.title('one epoch') plt.xlabel('iteration') plt.ylabel('loss') plt.show() # trainloss.append(sum_loss/iter) # trainacc.append(100*acc/len(trainData)) # x.append(epoch) print('Epoch [%d] : loss [%f]' % (epoch + 1, sum_loss / iter)) print('train accuracy = %f%%' % (100 * acc / len(trainData))) #with torch.no_grad(): # correct = 0 # total = 0 # for data in testLoader: # images, labels = data # images, labels = images.to(devices), labels.to(devices) # outputs = net(images) # _, predicted = torch.max(outputs.data, 1) # total += labels.size(0) # correct += (predicted == labels).sum() #print('test accuracy = %f%%'%(100*correct/total)) #testacc.append(100*correct/total) if save == True: torch.save(net.state_dict(), 'MNIST_Model.pth')
class Model(TemplateModel): def __init__(self, args=None): super().__init__() self.writer = tX.SummaryWriter(log_dir=log_dir, comment='LeNet') self.train_logger = None self.eval_logger = None self.args = args self.step = 0 self.epoch = 0 self.best_error = float('Inf') self.device = torch.device('cpu') self.model = LeNet().to(self.device) self.optimizer = optim.Adam(self.model.parameters(), lr=lr) self.criterion = torch.nn.CrossEntropyLoss() self.metric = metric transform = tfs.Compose( [tfs.ToTensor(), tfs.Normalize((0.1307, ), (0.3081, ))]) train_dataset = MNIST(root='MNIST', train=True, transform=transform, download=True) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) test_dataset = MNIST(root='MNIST', train=False, transform=transform, download=True) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) self.train_loader = train_loader self.test_loader = test_loader self.ckpt_dir = ckpt_dir self.log_per_step = log_per_step # self.eval_per_epoch = None self.check_init()
def main(config): GPUManager.auto_chooce() engine = DefaultClassificationEngine() engine.hooks.update( dict( on_sample=on_sample_hook, on_start=on_start_hook, on_forward=on_forward_hook, ) ) net = LeNet(n_channels=config.n_channels, size=config.size) print(net) if torch.cuda.is_available(): net = net.cuda() train_loader = torch.utils.data.DataLoader( config.dataset(train=True), batch_size=config.batch_size, shuffle=True, num_workers=8 ) val_loader = torch.utils.data.DataLoader( config.dataset(train=False), batch_size=1000, shuffle=True, num_workers=8) optimizer = optim.SGD(net.parameters(), lr=config.lr, momentum=0.9) recorder = defaultdict(list) recorder.update( dict( dataset_name=config.dataset.__name__.split("_")[0], lr=config.lr, batch_size=config.batch_size ) ) pprint(recorder) for epoch in range(config.maxepoch): state=engine.train(network=net, iterator=train_loader, maxepoch=1, optimizer=optimizer) recorder["train_loss"].append(state["loss_meter"].value()) recorder["train_acc"].append(state["acc_meter"].value()) state=engine.validate(network=net, iterator=val_loader) recorder["val_loss"].append(state["loss_meter"].value()) recorder["val_acc"].append(state["acc_meter"].value()) filename = f"{recorder['dataset_name']}_" + time.strftime("%Y%m%d_%H%M%S", time.localtime()) with open(f"../result/{filename}.static", "wb") as f: pickle.dump(recorder, f)
def gpu_train(): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(device) # 或者 # device = torch.device("cuda") # 或者 # device = torch.device("cpu") net = LeNet() net.to(device) # 将网络分配到指定的device中 loss_function = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=0.001) for epoch in range(5): running_loss = 0.0 time_start = time.perf_counter() for step, data in enumerate(train_loader, start=0): inputs, labels = data optimizer.zero_grad() outputs = net(inputs.to(device)) # 将inputs分配到指定的device中 loss = loss_function(outputs, labels.to(device)) # 将labels分配到指定的device中 loss.backward() optimizer.step() running_loss += loss.item() if step % 1000 == 999: with torch.no_grad(): outputs = net(test_image.to(device)) # 将test_image分配到指定的device中 predict_y = torch.max(outputs, dim=1)[1] accuracy = (predict_y == test_label.to(device)).sum().item() / test_label.size(0) # 将test_label分配到指定的device中 print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' % (epoch + 1, step + 1, running_loss / 1000, accuracy)) print('%f s' % (time.perf_counter() - time_start)) running_loss = 0.0 print('Finished Training') save_path = './Lenet.pth' torch.save(net.state_dict(), save_path)
def main(): class Args(): def __init__(self): pass args = Args() args.batch_size = 64 args.test_batch_size = 2 args.epochs = 10 args.lr = 0.0001 args.momentum = 0.5 args.no_cuda = False args.seed = 1 args.log_interval = 100 args.save_model = True use_cuda = not args.no_cuda and torch.cuda.is_available() # torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) dataset_dir = Path(os.environ['HOME'])/"datasets/mnist" train_dataset = MnistDataset(root_dir=dataset_dir/"train", transform=transform) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) test_dataset = MnistDataset(root_dir=dataset_dir/"test", transform=transform) test_loader = DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=True, num_workers=4) model = LeNet().to(device) # model = nn.DataParallel(model) optimizer = optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) if (args.save_model): torch.save(model.state_dict(), "mnist_cnn.pth")
def usually_train(): net = LeNet() # 定义训练的网络模型 loss_function = nn.CrossEntropyLoss() # 定义损失函数为交叉熵损失函数 optimizer = optim.Adam(net.parameters(), lr=0.001) # 定义优化器(训练参数,学习率) for epoch in range(5): # 一个epoch即对整个训练集进行一次训练 running_loss = 0.0 time_start = time.perf_counter() for step, data in enumerate(train_loader, start=0): # 遍历训练集,step从0开始计算 inputs, labels = data # 获取训练集的图像和标签 optimizer.zero_grad() # 清除历史梯度 # forward + backward + optimize outputs = net(inputs) # 正向传播 loss = loss_function(outputs, labels) # 计算损失 loss.backward() # 反向传播 optimizer.step() # 优化器更新参数 # 打印耗时、损失、准确率等数据 running_loss += loss.item() if step % 1000 == 999: # print every 1000 mini-batches,每1000步打印一次 with torch.no_grad(): # 在以下步骤中(验证过程中)不用计算每个节点的损失梯度,防止内存占用 outputs = net(test_image) # 测试集传入网络(test_batch_size=10000),output维度为[10000,10] predict_y = torch.max(outputs, dim=1)[1] # 以output中值最大位置对应的索引(标签)作为预测输出 accuracy = (predict_y == test_label).sum().item() / test_label.size(0) print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' % # 打印epoch,step,loss,accuracy (epoch + 1, step + 1, running_loss / 1000, accuracy)) print('%f s' % (time.perf_counter() - time_start)) # 打印耗时 running_loss = 0.0 print('Finished Training') # 保存训练得到的参数 save_path = './Lenet.pth' torch.save(net.state_dict(), save_path)
shuffle=False, num_workers=4, pin_memory=has_cuda) loaderOneByOne = torch.utils.data.DataLoader(subset, batch_size=1, shuffle=False, num_workers=4, pin_memory=has_cuda) # Retrieve pre trained model classes = 10 model = LeNet() d = torch.load('model/best.pth.tar', map_location='cpu') model.load_state_dict(d['state_dict'], strict=False) model.eval() for p in model.parameters(): p.requires_grad_(False) #change criterion for ImageNet-1k and CIFAR100 to Top5Criterion criterion = lambda x, y: Top1Criterion(x, y, model) if args.norm == 'L2': norm = 2 elif args.norm == 'Linf': norm = np.inf elif args.norm == 'L0': norm = 0 elif args.norm == 'L1': norm = 1 if has_cuda: model = model.cuda()
from train import Trainer from test import Tester from model import LeNet from dataset import DataSet, transform, train_set, train_loader, test_set, test_loader import torch as t import torch.nn as nn import torchvision as tv from torch import optim from torch.autograd import Variable # CIFAR-10的全部类别,一共10类 classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # 数据集 dataSet = DataSet(transform, train_set, train_loader, test_set, test_loader, classes) # 网络结构 net = LeNet() # 交叉熵损失函数 criterion = nn.CrossEntropyLoss() # SGD优化器 optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) trainer = Trainer(net, criterion, optimizer, dataSet.train_loader) tester = Tester(dataSet.test_loader, net) trainer.train(epochs=10) tester.test()
def main(): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print("using {} device.".format(device)) batch_size = 16 epochs = 200 data_transform = { "train": transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), "val": transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) } data_root = os.path.abspath(os.path.join(os.getcwd(), ".")) # get data root path image_path = os.path.join(data_root, "data_set", "flower_data") # flower data set path assert os.path.exists(image_path), "{} path does not exist.".format( image_path) train_dataset = datasets.ImageFolder(root=os.path.join( image_path, "train"), transform=data_transform["train"]) train_num = len(train_dataset) # {'daisy':0, 'dandelion':1, 'roses':2, 'sunflower':3, 'tulips':4} flower_list = train_dataset.class_to_idx cla_dict = dict((val, key) for key, val in flower_list.items()) # write dict into json file json_str = json.dumps(cla_dict, indent=4) with open('class_indices.json', 'w') as json_file: json_file.write(json_str) nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]) # number of workers print('Using {} dataloader workers every process'.format(nw)) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=nw) validate_dataset = datasets.ImageFolder(root=os.path.join( image_path, "val"), transform=data_transform["val"]) val_num = len(validate_dataset) validate_loader = torch.utils.data.DataLoader(validate_dataset, batch_size=batch_size, shuffle=False, num_workers=nw) print("using {} images for training, {} images for validation.".format( train_num, val_num)) # create model net = LeNet(num_classes=5) # load pretrained weights # download url: https://download.pytorch.org/models/mobilenet_v3_large-8738ca79.pth # download url: https://download.pytorch.org/models/mobilenet_v3_small-047dcff4.pth # model_weight_path = "weights/LeNet_pretrained.pth" # assert os.path.exists(model_weight_path), "file {} dose not exist.".format(model_weight_path) # pre_weights = torch.load(model_weight_path, map_location=device) # delete classifier weights # pre_dict = {k: v for k, v in pre_weights.items() if net.state_dict()[k].numel() == v.numel()} # missing_keys, unexpected_keys = net.load_state_dict(pre_dict, strict=False) # # # freeze features weights # for param in net.conv_stem.parameters(): # param.requires_grad = False # # for param in net.bn1.parameters(): # param.requires_grad = False # # for param in net.act1.parameters(): # param.requires_grad = False # # for param in net.blocks.parameters(): # param.requires_grad = False net.to(device) # define loss function loss_function = nn.CrossEntropyLoss() # construct an optimizer params = [p for p in net.parameters() if p.requires_grad] optimizer = optim.Adam(params, lr=0.0001) best_acc = 0.0 save_path = 'weights/lenet.pth' train_steps = len(train_loader) for epoch in range(epochs): # train net.train() running_loss = 0.0 train_bar = tqdm(train_loader) for step, data in enumerate(train_bar): images, labels = data optimizer.zero_grad() logits = net(images.to(device)) loss = loss_function(logits, labels.to(device)) loss.backward() optimizer.step() # print statistics running_loss += loss.item() train_bar.desc = "train epoch[{}/{}] loss:{:.3f}".format( epoch + 1, epochs, loss) # validate net.eval() acc = 0.0 # accumulate accurate number / epoch with torch.no_grad(): val_bar = tqdm(validate_loader) for val_data in val_bar: val_images, val_labels = val_data outputs = net(val_images.to(device)) # loss = loss_function(outputs, test_labels) predict_y = torch.max(outputs, dim=1)[1] acc += torch.eq(predict_y, val_labels.to(device)).sum().item() val_bar.desc = "valid epoch[{}/{}]".format(epoch + 1, epochs) val_accurate = acc / val_num print('[epoch %d] train_loss: %.3f val_accuracy: %.3f' % (epoch + 1, running_loss / train_steps, val_accurate)) if val_accurate > best_acc: best_acc = val_accurate torch.save(net.state_dict(), save_path) print('Finished Training')
cifar_test = dataset.Cifar10Dataset('./cifar10/test', transform=transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) cifar_train_loader = DataLoader(cifar_train, batch_size=batch_size, shuffle=True) cifar_test_loader = DataLoader(cifar_test, batch_size=batch_size, shuffle=False) net = LeNet() if MULTI_GPU: net = nn.DataParallel(net,device_ids=device_ids) net.to(device) criteon = nn.CrossEntropyLoss() optimizer=optim.Adam(net.parameters(), lr=1e-3) scheduler = StepLR(optimizer, step_size=100, gamma=0.1) if MULTI_GPU: optimizer = nn.DataParallel(optimizer, device_ids=device_ids) scheduler = nn.DataParallel(scheduler, device_ids=device_ids) # print(net) for epoch in range(epoch_num): for batchidx, (label, img) in enumerate(cifar_train_loader): net.train() logits = net(img.to(device)) loss = criteon(logits, label.long().to(device)) optimizer.zero_grad() loss.backward() optimizer.module.step() scheduler.module.step()
shuffle=False) #实现单张图片可视化 # images,labels = next(iter(train_loader)) # img = torchvision.utils.make_grid(images) # img = img.numpy().transpose(1,2,0) # # img.shape # std = [0.5,0.5,0.5] # mean = [0.5,0.5,0.5] # img = img*std +mean # cv2.imshow('win',img) # key_pressed = cv2.waitKey(0) net = LeNet().to(device) criterion = nn.CrossEntropyLoss() #定义损失函数 optimizer = torch.optim.SGD(net.parameters(), lr=LR, momentum=Momentum) epoch = 10 if __name__ == '__main__': for epoch in range(epoch): sum_loss = 0.0 for i, data in enumerate(train_loader): inputs, labels = data inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() #将梯度归零 outputs = net(inputs) #将数据传入网络进行前向运算 loss = criterion(outputs, labels) #得到损失函数 loss.backward() #反向传播 optimizer.step() #通过梯度做一步参数更新 # print(loss)
def main(args): # Data Loader (Input Pipeline) model_str = args.dataset + '_%s_' % args.model_type + args.noise_type + '_' + str(args.noise_rate) + '_' + str(args.seed) txtfile = save_dir + "/" + model_str + ".txt" nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S') if os.path.exists(txtfile): os.system('mv %s %s' % (txtfile, txtfile + ".bak-%s" % nowTime)) # Data Loader (Input Pipeline) print('loading dataset...') train_dataset, val_dataset, test_dataset = load_data(args) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=True) val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) # Define models print('building model...') if args.dataset == 'mnist': clf1 = LeNet() optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'fmnist': clf1 = resnet.ResNet50(input_channel=1, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'cifar10': clf1 = resnet.ResNet50(input_channel=3, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) elif args.dataset == 'cifar100': clf1 = resnet.ResNet50(input_channel=3, num_classes=100) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) clf1.cuda() with open(txtfile, "a") as myfile: myfile.write('epoch train_acc1 val_acc1 test_acc1\n') epoch = 0 train_acc1 = 0 # evaluate models with random weights val_acc1 = evaluate(val_loader, clf1) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(val_dataset), val_acc1)) test_acc1 = evaluate(test_loader, clf1) print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") val_acc_list = [] test_acc_list = [] for epoch in range(0, args.n_epoch): scheduler1.step() print(optimizer1.state_dict()['param_groups'][0]['lr']) clf1.train() train_acc1 = train(train_loader, epoch, clf1, optimizer1, args) val_acc1 = evaluate(val_loader, clf1) val_acc_list.append(val_acc1) test_acc1 = evaluate(test_loader, clf1) test_acc_list.append(test_acc1) # save results print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %% ' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") id = np.argmax(np.array(val_acc_list)) test_acc_max = test_acc_list[id] return test_acc_max
download=True) train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_set, batch_size=batch_size, shuffle=False) print("==>>> total number of training batches : {}".format(len(train_loader))) print("==>>> total number of testing batches : {}".format(len(test_loader))) print("==>>> Batch Size is : {}".format(batch_size)) model = LeNet(num_classes).to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr) num_batches = len(train_loader) for epoch in range(num_epochs): for idx, (inputs, labels) in enumerate(train_loader): inputs = inputs.to(device) labels = labels.to(device) outputs = model(inputs) loss = criterion(outputs, labels) optimizer.zero_grad() loss.backward() optimizer.step() if ((idx + 1) % 100 == 0): print("epoch is {}/{} Step is: {}/{} loss is: {}".format( epoch, num_epochs, idx, num_batches, loss.item()))
import os import sys pardir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(pardir) from dataset import * from result_save_visualization import * from model import LeNet import torch import torch.optim as optim import time net = LeNet().to(device) optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) num_epochs = 50 data_dict = { 'epoch': [], 'time': [], 'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': [] } start = time.time() for epoch in range(num_epochs): # train net.train()
correct += pred.eq(target.data.view_as(pred)).cpu().sum() test_loss /= len(validate_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(validate_loader.dataset), 100. * correct / len(validate_loader.dataset))) if args.lenet: model = LeNet() else: model = Net() if args.cuda: model.cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) model.train() for epoch in range(args.epochs): for batch_idx, (data, target) in enumerate(train_loader): if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
learning_rate = 1e-3 batch_size = 64 epoches = 50 trans_img = transforms.ToTensor() trainset = MNIST('./data', train=True, download=True, transform=trans_img) testset = MNIST('./data', train=False, download=True, transform=trans_img) trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4) testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4) lenet = LeNet() criterion = nn.CrossEntropyLoss(size_average=False) optimizer = optim.SGD(lenet.parameters(), lr=learning_rate) trainer = Trainer(lenet, criterion) print('starting train......') trainer.train(10, trainloader, optimizer) print('finish train..................') print() print() sum = 0 runloss = 0 correct = 0 for _, batch in enumerate(testloader): input, target = batch sum += len(target)
transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader( datasets.MNIST('data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size, shuffle=False) # Define which model to use model = LeNet(mask=True).to(device) # Define Optimizer optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0001) initial_optimizer_state_dict = optimizer.state_dict() # Initial training print("--------------- Initial training ---------------") train(model, optimizer, train_loader, Nepochs) accuracy = test(model, test_loader) print("Initial Accuracy:",accuracy) torch.save(model, f"saves/initial_model.ptmodel") print("--------------- Stats Before Pruning ---------------") print_nonzeros(model) print("") # Pruning print("--------------- Pruning ---------------") model.prune_by_std(sensitivity)
import torch import torch.nn as nn from model import LeNet import torch.optim as optim from data import data_train_loader,data_test_loader from prog import args model=LeNet() model.train() #切换模型到训练状态 #lr=0.01 criterion=nn.CrossEntropyLoss() optimizer=optim.SGD(model.parameters(),lr=args.lr,momentum=0.9,weight_decay=5e-4) epoch=5 for epoch_id in range(epoch): train_loss=0 correct=0 total=0 for batch_idx, (inputs,targets) in enumerate(data_train_loader): optimizer.zero_grad() outputs=model(inputs) loss=criterion(outputs,targets) loss.backward() optimizer.step() train_loss+=loss.item() _,predicted=outputs.max(1) total+=targets.size(0) correct+=predicted.eq(targets).sum().item() print(epoch_id, batch_idx,len(data_train_loader),'Loss:%.3f|Acc:%.3f%%(%d/%d)'%(train_loss/(batch_idx+1),100.*correct/total,correct,total)) save_info={
""" train_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True, drop_last=True) valid_loader = DataLoader(dataset=valid_set, batch_size=batch_size, shuffle=False, drop_last=True) model = LeNet().to(device) # Define loss & Optimizer criterion = nn.CrossEntropyLoss().to(device) # Multiclass classification 전용 optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) # SGD total_batch_train = len(train_loader) total_batch_val = len(valid_loader) # Training/Validation Loss, and Accuracy History train_loss_log = [] train_accuracy_log = [] top5_train_accuracy_log = [] valid_loss_log = [] valid_accuracy_log = [] top5_valid_accuracy_log = [] # History for Confusion Matrix pred_list = torch.zeros(0, dtype=torch.long, device='cpu')
def train(): save_flag = True trainloader = get_dataset(r'H:/DataSet_All/猫狗识别/gpu/train', batch_size=64, imageindex=0) validationloader = get_dataset(r'H:/DataSet_All/猫狗识别/gpu/test', batch_size=64, imageindex=0) save_dir = 'output/' my_model = LeNet() my_model = model_init(my_model) criterion = nn.CrossEntropyLoss().to(device) optimizer = optim.Adam(my_model.parameters(), lr=0.001) epochs = 300 # 训练 for epoch in range(epochs): loss_list = [] acc_list = [] set_learning_rate(optimizer, epoch) learning_rate = optimizer.param_groups[0]['lr'] tq = tqdm.tqdm(trainloader, desc='train') tq.set_description('train Epoch{} lr{}'.format( epoch, learning_rate)) for images, labels in tq: images = images.to(device) labels = labels.to(device) outputs = my_model(images) loss = criterion(outputs, labels) my_model.zero_grad() loss.backward() optimizer.step() loss_list.append(loss.item()) loss_ave = sum(loss_list) / len(loss_list) _, predicted = torch.max(outputs.data, 1) accuracy = (predicted == labels).sum().float() / labels.size(0) acc_list.append(accuracy) acc_ave = sum(acc_list) / len(acc_list) tq.set_postfix( loss="%.4f accuracy:%.4f loss_ave:%.5f acc_ave:%.5f " % (loss.item(), accuracy, loss_ave, acc_ave)) if save_flag: log = "\ntrain \tEpoch {}/{} \t Learning rate: {:.5f} \t Train loss_ave: {:.5f} \t acc_ave: {:.5f} \t " \ .format(epoch, epochs, learning_rate, loss_ave, acc_ave ) # print(log) logFile = open(save_dir + '/log.txt', 'a') logFile.write(log + '\n') torch.save(my_model.state_dict(), save_dir + '/model_lastest.pt') if epoch % 1 == 0: loss_list = [] acc_list = [] with torch.no_grad(): tq = tqdm.tqdm(validationloader, desc='teat') for images, labels in tq: images = images.to(device) labels = labels.to(device) outputs = my_model(images) validation_loss = criterion(outputs, labels) loss_list.append(validation_loss.item()) _, predicted = torch.max(outputs.data, 1) accuracy = (predicted == labels).sum().float() / labels.size(0) acc_list.append(accuracy) acc_ave = sum(acc_list) / len(acc_list) loss_ave = sum(loss_list) / len(loss_list) tq.set_postfix( test_loss= "%.4f acc:%.4f loss_ave:%.5f acc_ave:%.5f " % (validation_loss, accuracy, loss_ave, acc_ave)) log = "\ntest \tEpoch {}/{} \t Learning rate: {:.5f} \t Train loss_ave: {:.5f} \t acc_ave: {:.5f} \t " \ .format(epoch, epochs, learning_rate, loss_ave, acc_ave) # print(log) logFile = open(save_dir + '/log.txt', 'a') logFile.write(log + '\n')
""" 该代码定义了LeNet模型的训练过程 """ import torch import torch.nn as nn from model import LeNet # ... 此处略去定义训练数据载入器的代码,具体可参考代码4.3 model = LeNet() # 定义LeNet模型 model.train() # 切换模型到训练状态 lr = 0.01 # 定义学习率 criterion = nn.CrossEntropyLoss() # 定义损失函数 optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=5e-4) # 定义随机梯度下降优化器 train_loss = 0 correct = 0 total = 0 for batch_idx, (inputs, targets) in enumerate(data_train_loader): optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs, targets) loss.backward() optimizer.step() train_loss += loss.item() _, predicted = outputs.max(1) total += targets.size(0)
data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=0) #8个线程处理,随机抽取,batch=256 # data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8) ## 数据载入 # from data import data_train_loader #### 定义模型训练参数部分 model = LeNet() model.cpu() model.train() ## 切换到模型的训练模式 lr = 0.001 loss_define = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=5e-4) train_loss = 0 ## 训练损失 correct = 0 ## 识别正确个数 total = 0 ## 总个数 # momentum=0.9, epoch_num = 10 # int(input('input epoch:')) loss_plot = [] for epoch in range(epoch_num): #### for batch_idx, (inputs, targets) in enumerate(data_train_loader): output = model(inputs) ##识别10个手写数字(0~9),因此output输出10个概率值 loss = loss_define(output, targets) ##前向传播计算出损失 loss.backward() ##对损失进行反向传播
train=False, download=False, transform=transform) val_loader = torch.utils.data.DataLoader(val_set, batch_size=5000, shuffle=False, num_workers=0) val_data_iter = iter(val_loader) val_image, val_label = val_data_iter.next() classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') net = LeNet() loss_function = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=0.001) for epoch in range(50): # loop over the dataset multiple times running_loss = 0.0 for step, data in enumerate(train_loader, start=0): # get the inputs; data is a list of [inputs, labels] inputs, labels = data # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = net(inputs) loss = loss_function(outputs, labels) loss.backward() optimizer.step()
def train(): device = torch.device("cuda:0" if opt.cuda else "cpu") utils.set_seed() # ============================ step 1/5 数据 ============================ norm_mean = [0.485, 0.456, 0.406] norm_std = [0.229, 0.224, 0.225] train_transform = transforms.Compose([ transforms.Resize((32, 32)), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std), ]) valid_transform = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std), ]) # 构建MyDataset train_data = RMBDataset(data_dir=opt.train_dir, transform=train_transform) valid_data = RMBDataset(data_dir=opt.valid_dir, transform=valid_transform) # 构建DataLoader train_loader = DataLoader(dataset=train_data, batch_size=opt.batch_size, shuffle=True) valid_loader = DataLoader(dataset=valid_data, batch_size=opt.batch_size) # ============================ step 2/5 模型 ============================ net = LeNet(classes=2) net.to(device) # net.initialize_weights() # ============================ step 3/5 损失函数 ============================ criterion = nn.CrossEntropyLoss() # ============================ step 4/5 优化器 ============================ optimizer = optim.SGD(net.parameters(), lr=opt.lr, momentum=0.9) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) # ============================ step 5/5 训练 ============================ train_curve = list() valid_curve = list() for epoch in range(opt.epochs): loss_mean = 0. correct = 0. total = 0. net.train() for i, data in enumerate(train_loader): inputs, labels = data inputs = inputs.to(device) labels = labels.to(device) outputs = net(inputs) optimizer.zero_grad() loss = criterion(outputs, labels) loss.backward() optimizer.step() # 统计分类情况 _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).squeeze().sum().to("cpu").numpy() # 打印训练信息 loss_mean += loss.item() train_curve.append(loss.item()) if (i + 1) % opt.log_interval == 0: loss_mean = loss_mean / opt.log_interval print( "Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}" .format(epoch, opt.epochs, i + 1, len(train_loader), loss_mean, correct / total)) loss_mean = 0. scheduler.step() # 更新学习率 if (epoch + 1) % opt.val_interval == 0: correct_val = 0. total_val = 0. loss_val = 0. net.eval() with torch.no_grad(): for j, data in enumerate(valid_loader): inputs, labels = data inputs = inputs.to(device) labels = labels.to(device) outputs = net(inputs) loss = criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) total_val += labels.size(0) correct_val += ( predicted == labels).squeeze().sum().to("cpu").numpy() loss_val += loss.item() valid_curve.append(loss_val) print( "Valid:\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}" .format(epoch, opt.epochs, j + 1, len(valid_loader), loss_val, correct / total)) utils.loss_picture(train_curve, train_loader, valid_curve, opt.val_interval) # 保存模型参数 net_state_dict = net.state_dict() torch.save(net_state_dict, opt.path_state_dict) print("模型保存成功")
def train_and_test(flags, corruption_level=0, gold_fraction=0.5, get_C=uniform_mix_C): np.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed(1) C = get_C(corruption_level) gold, silver = prepare_data(C, gold_fraction) print("Gold shape = {}, Silver shape = {}".format(gold.images.shape, silver.images.shape)) # TODO : test on whole set test_x = torch.from_numpy(mnist.test.images[:500].reshape([-1, 1, 28, 28])) test_y = torch.from_numpy(mnist.test.labels[:500]).type(torch.LongTensor) print("Test shape = {}".format(test_x.shape)) model = LeNet() optimizer = torch.optim.Adam([p for p in model.parameters()], lr=0.001) for step in range(flags.num_steps): x, y = silver.next_batch(flags.batch_size) y, y_true = np.array([l[0] for l in y]), np.array([l[1] for l in y]) x_val, y_val = gold.next_batch(min(flags.batch_size, flags.nval)) x, y = torch.from_numpy(x.reshape( [-1, 1, 28, 28])), torch.from_numpy(y).type(torch.LongTensor) x_val, y_val = torch.from_numpy(x_val.reshape( [-1, 1, 28, 28])), torch.from_numpy(y_val).type(torch.LongTensor) # forward if flags.method == "l2w": ex_wts = reweight_autodiff(model, x, y, x_val, y_val) logits, loss = model.loss(x, y, ex_wts) if step % dbg_steps == 0: tbrd.log_histogram("ex_wts", ex_wts, step=step) tbrd.log_value("More_than_0.01", sum([x > 0.01 for x in ex_wts]), step=step) tbrd.log_value("More_than_0.05", sum([x > 0.05 for x in ex_wts]), step=step) tbrd.log_value("More_than_0.1", sum([x > 0.1 for x in ex_wts]), step=step) mean_on_clean_labels = np.mean( [ex_wts[i] for i in range(len(y)) if y[i] == y_true[i]]) mean_on_dirty_labels = np.mean( [ex_wts[i] for i in range(len(y)) if y[i] != y_true[i]]) tbrd.log_value("mean_on_clean_labels", mean_on_clean_labels, step=step) tbrd.log_value("mean_on_dirty_labels", mean_on_dirty_labels, step=step) else: logits, loss = model.loss(x, y) print("Loss = {}".format(loss)) # backward optimizer.zero_grad() loss.backward() optimizer.step() tbrd.log_value("loss", loss, step=step) if step % dbg_steps == 0: model.eval() pred = torch.max(model.forward(test_x), 1)[1] test_acc = torch.sum(torch.eq(pred, test_y)).item() / float( test_y.shape[0]) model.train() print("Test acc = {}.".format(test_acc)) tbrd.log_value("test_acc", test_acc, step=step)
shuffle=True, num_workers=4) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4) #define LeNet model model = LeNet(1, 10) use_gpu = torch.cuda.is_available() if use_gpu: model = model.cuda() # define loss and optimizer function criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=learning_rate) # start training for epoch in range(num_epoches): train_loss = 0.0 train_acc = 0.0 for i, data in enumerate(train_loader, 1): img, label = data if use_gpu: img = img.cuda() label = label.cuda() img = Variable(img) label = Variable(label) # forward propagate
# build model if args.arc == 'LeNet': model = LeNet() elif args.arc.startswith('VGG'): model = VGG(args.arc) else: model = LeNet() if args.cuda: print('Using CUDA with {0} GPUs'.format(torch.cuda.device_count())) model = torch.nn.DataParallel(model, device_ids=[0]).cuda() # define optimizer if args.optimizer.lower() == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr) elif args.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) else: optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) best_valid_loss = np.inf iteration = 0 epoch = 1 # don't do early stopping because the validation set is contaminated now while epoch < args.epochs + 1:
train_data = read_images(os.path.join(args.data_dir, 'train-images.idx3-ubyte')) train_labels = read_labels( os.path.join(args.data_dir, 'train-labels.idx1-ubyte')) test_data = read_images(os.path.join(args.data_dir, 't10k-images.idx3-ubyte')) test_labels = read_labels(os.path.join(args.data_dir, 't10k-labels.idx1-ubyte')) # normalize train_data = (train_data - train_data.mean( (1, 2), keepdims=True)) / train_data.std((1, 2), keepdims=True) test_data = (test_data - test_data.mean( (1, 2), keepdims=True)) / test_data.std((1, 2), keepdims=True) my_net = LeNet() optimizer = SGD(my_net.parameters(), lr, momentum) loss_history = [] epoch_steps = train_data.shape[0] // batch + 1 avg_loss = avg_acc = 0 for e in range(epoch): if e and e % 3 == 0: optimizer.lr *= 0.1 train_loss = train_acc = 0 e_data, e_labels = shuffle(train_data, train_labels) with tqdm(total=epoch_steps) as pbar: for x, t in zip(np.array_split(e_data, epoch_steps),