def train_lenet(device, dataset_path):
train_loader, valid_loader, test_loader = get_data_loaders(dataset_path)
model = LeNet(35)
optimizer = optim.Adam(model.parameters(),
lr=Consts.lr,
weight_decay=Consts.weight_decay)
loss_criterion = torch.nn.NLLLoss()
model.apply(weight_init)
model.to(device)
train_loss = []
val_loss = []
val_acc = []
for epoch in range(Consts.epochs):
t_loss = train(model, train_loader, optimizer, loss_criterion, device)
v_loss, v_acc = evaluation(model, valid_loader, loss_criterion, device)
torch.save(model.state_dict(), f'models/epoch-{epoch + 1}.pth')
train_loss.append(t_loss)
val_loss.append(v_loss)
val_acc.append(v_acc)
print(f'train loss in epoch {epoch + 1} is: {t_loss}')
print(f'validation loss in epoch {epoch + 1} is: {v_loss}')
print(f'validation accuracy in epoch {epoch + 1} is: {v_acc}')
plot_loss(train_loss, val_loss, val_acc)
test_loss, test_acc = test_model(model, test_loader, loss_criterion,
val_loss, device, 'models/')
def main():
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
# 50000张训练图片
# 第一次使用时要将download设置为True才会自动去下载数据集
train_set = torchvision.datasets.CIFAR10(root='./data', train=True,
download=False, transform=transform)
train_loader = torch.utils.data.DataLoader(train_set, batch_size=36,
shuffle=True, num_workers=0)
# 10000张验证图片
# 第一次使用时要将download设置为True才会自动去下载数据集
val_set = torchvision.datasets.CIFAR10(root='./data', train=False,
download=False, transform=transform)
val_loader = torch.utils.data.DataLoader(val_set, batch_size=5000,
shuffle=False, num_workers=0)
val_data_iter = iter(val_loader)
val_image, val_label = val_data_iter.next()
# classes = ('plane', 'car', 'bird', 'cat',
# 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
net = LeNet()
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)
for epoch in range(5): # loop over the dataset multiple times
running_loss = 0.0
for step, data in enumerate(train_loader, start=0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = loss_function(outputs, labels)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
if step % 500 == 499: # print every 500 mini-batches
with torch.no_grad():
outputs = net(val_image) # [batch, 10]
predict_y = torch.max(outputs, dim=1)[1]
accuracy = (predict_y == val_label).sum().item() / val_label.size(0)
print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' %
(epoch + 1, step + 1, running_loss / 500, accuracy))
running_loss = 0.0
print('Finished Training')
save_path = './Lenet.pth'
torch.save(net.state_dict(), save_path)
def trainModel(EPOCH_NUM=50, save=False, show=False):
print("Train Start:")
net = LeNet().to(devices)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.RMSprop(net.parameters(),
lr=LR,
alpha=0.9,
eps=1e-08,
weight_decay=0,
momentum=0,
centered=False)
#x,trainloss,trainacc,testacc = [],[],[],[]
batch, batchloss = [], []
for epoch in range(EPOCH_NUM):
sum_loss = 0.0
acc = 0
iter = 0
for i, (inputs, labels) in enumerate(trainLoader):
inputs, labels = inputs.to(devices), labels.to(devices)
# forward and backward
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
sum_loss += loss.item()
_, pred = torch.max(outputs.data, 1)
acc += (pred == labels).sum()
iter = iter + 1
batch.append(i)
batchloss.append(loss.item())
if show == True:
plt.figure()
plt.plot(batch, batchloss, 'b')
plt.title('one epoch')
plt.xlabel('iteration')
plt.ylabel('loss')
plt.show()
# trainloss.append(sum_loss/iter)
# trainacc.append(100*acc/len(trainData))
# x.append(epoch)
print('Epoch [%d] : loss [%f]' % (epoch + 1, sum_loss / iter))
print('train accuracy = %f%%' % (100 * acc / len(trainData)))
#with torch.no_grad():
# correct = 0
# total = 0
# for data in testLoader:
# images, labels = data
# images, labels = images.to(devices), labels.to(devices)
# outputs = net(images)
# _, predicted = torch.max(outputs.data, 1)
# total += labels.size(0)
# correct += (predicted == labels).sum()
#print('test accuracy = %f%%'%(100*correct/total))
#testacc.append(100*correct/total)
if save == True:
torch.save(net.state_dict(), 'MNIST_Model.pth')
class Model(TemplateModel):
def __init__(self, args=None):
super().__init__()
self.writer = tX.SummaryWriter(log_dir=log_dir, comment='LeNet')
self.train_logger = None
self.eval_logger = None
self.args = args
self.step = 0
self.epoch = 0
self.best_error = float('Inf')
self.device = torch.device('cpu')
self.model = LeNet().to(self.device)
self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
self.criterion = torch.nn.CrossEntropyLoss()
self.metric = metric
transform = tfs.Compose(
[tfs.ToTensor(),
tfs.Normalize((0.1307, ), (0.3081, ))])
train_dataset = MNIST(root='MNIST',
train=True,
transform=transform,
download=True)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True)
test_dataset = MNIST(root='MNIST',
train=False,
transform=transform,
download=True)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False)
self.train_loader = train_loader
self.test_loader = test_loader
self.ckpt_dir = ckpt_dir
self.log_per_step = log_per_step
# self.eval_per_epoch = None
self.check_init()
def main(config):
GPUManager.auto_chooce()
engine = DefaultClassificationEngine()
engine.hooks.update(
dict(
on_sample=on_sample_hook,
on_start=on_start_hook,
on_forward=on_forward_hook,
)
)
net = LeNet(n_channels=config.n_channels, size=config.size)
print(net)
if torch.cuda.is_available():
net = net.cuda()
train_loader = torch.utils.data.DataLoader(
config.dataset(train=True),
batch_size=config.batch_size, shuffle=True, num_workers=8
)
val_loader = torch.utils.data.DataLoader(
config.dataset(train=False),
batch_size=1000, shuffle=True, num_workers=8)
optimizer = optim.SGD(net.parameters(), lr=config.lr, momentum=0.9)
recorder = defaultdict(list)
recorder.update(
dict(
dataset_name=config.dataset.__name__.split("_")[0],
lr=config.lr,
batch_size=config.batch_size
)
)
pprint(recorder)
for epoch in range(config.maxepoch):
state=engine.train(network=net, iterator=train_loader, maxepoch=1, optimizer=optimizer)
recorder["train_loss"].append(state["loss_meter"].value())
recorder["train_acc"].append(state["acc_meter"].value())
state=engine.validate(network=net, iterator=val_loader)
recorder["val_loss"].append(state["loss_meter"].value())
recorder["val_acc"].append(state["acc_meter"].value())
filename = f"{recorder['dataset_name']}_" + time.strftime("%Y%m%d_%H%M%S", time.localtime())
with open(f"../result/{filename}.static", "wb") as f:
pickle.dump(recorder, f)
def gpu_train():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
# 或者
# device = torch.device("cuda")
# 或者
# device = torch.device("cpu")
net = LeNet()
net.to(device) # 将网络分配到指定的device中
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)
for epoch in range(5):
running_loss = 0.0
time_start = time.perf_counter()
for step, data in enumerate(train_loader, start=0):
inputs, labels = data
optimizer.zero_grad()
outputs = net(inputs.to(device)) # 将inputs分配到指定的device中
loss = loss_function(outputs, labels.to(device)) # 将labels分配到指定的device中
loss.backward()
optimizer.step()
running_loss += loss.item()
if step % 1000 == 999:
with torch.no_grad():
outputs = net(test_image.to(device)) # 将test_image分配到指定的device中
predict_y = torch.max(outputs, dim=1)[1]
accuracy = (predict_y == test_label.to(device)).sum().item() / test_label.size(0) # 将test_label分配到指定的device中
print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' %
(epoch + 1, step + 1, running_loss / 1000, accuracy))
print('%f s' % (time.perf_counter() - time_start))
running_loss = 0.0
print('Finished Training')
save_path = './Lenet.pth'
torch.save(net.state_dict(), save_path)
def main():
class Args():
def __init__(self):
pass
args = Args()
args.batch_size = 64
args.test_batch_size = 2
args.epochs = 10
args.lr = 0.0001
args.momentum = 0.5
args.no_cuda = False
args.seed = 1
args.log_interval = 100
args.save_model = True
use_cuda = not args.no_cuda and torch.cuda.is_available()
# torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
dataset_dir = Path(os.environ['HOME'])/"datasets/mnist"
train_dataset = MnistDataset(root_dir=dataset_dir/"train", transform=transform)
train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4)
test_dataset = MnistDataset(root_dir=dataset_dir/"test", transform=transform)
test_loader = DataLoader(test_dataset, batch_size=args.test_batch_size, shuffle=True, num_workers=4)
model = LeNet().to(device)
# model = nn.DataParallel(model)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader)
if (args.save_model):
torch.save(model.state_dict(), "mnist_cnn.pth")
def usually_train():
net = LeNet() # 定义训练的网络模型
loss_function = nn.CrossEntropyLoss() # 定义损失函数为交叉熵损失函数
optimizer = optim.Adam(net.parameters(), lr=0.001) # 定义优化器（训练参数，学习率）
for epoch in range(5): # 一个epoch即对整个训练集进行一次训练
running_loss = 0.0
time_start = time.perf_counter()
for step, data in enumerate(train_loader, start=0): # 遍历训练集，step从0开始计算
inputs, labels = data # 获取训练集的图像和标签
optimizer.zero_grad() # 清除历史梯度
# forward + backward + optimize
outputs = net(inputs) # 正向传播
loss = loss_function(outputs, labels) # 计算损失
loss.backward() # 反向传播
optimizer.step() # 优化器更新参数
# 打印耗时、损失、准确率等数据
running_loss += loss.item()
if step % 1000 == 999: # print every 1000 mini-batches，每1000步打印一次
with torch.no_grad(): # 在以下步骤中（验证过程中）不用计算每个节点的损失梯度，防止内存占用
outputs = net(test_image) # 测试集传入网络（test_batch_size=10000），output维度为[10000,10]
predict_y = torch.max(outputs, dim=1)[1] # 以output中值最大位置对应的索引（标签）作为预测输出
accuracy = (predict_y == test_label).sum().item() / test_label.size(0)
print('[%d, %5d] train_loss: %.3f test_accuracy: %.3f' % # 打印epoch，step，loss，accuracy
(epoch + 1, step + 1, running_loss / 1000, accuracy))
print('%f s' % (time.perf_counter() - time_start)) # 打印耗时
running_loss = 0.0
print('Finished Training')
# 保存训练得到的参数
save_path = './Lenet.pth'
torch.save(net.state_dict(), save_path)
shuffle=False,
num_workers=4,
pin_memory=has_cuda)
loaderOneByOne = torch.utils.data.DataLoader(subset,
batch_size=1,
shuffle=False,
num_workers=4,
pin_memory=has_cuda)
# Retrieve pre trained model
classes = 10
model = LeNet()
d = torch.load('model/best.pth.tar', map_location='cpu')
model.load_state_dict(d['state_dict'], strict=False)
model.eval()
for p in model.parameters():
p.requires_grad_(False)
#change criterion for ImageNet-1k and CIFAR100 to Top5Criterion
criterion = lambda x, y: Top1Criterion(x, y, model)
if args.norm == 'L2':
norm = 2
elif args.norm == 'Linf':
norm = np.inf
elif args.norm == 'L0':
norm = 0
elif args.norm == 'L1':
norm = 1
if has_cuda:
model = model.cuda()
from train import Trainer
from test import Tester
from model import LeNet
from dataset import DataSet, transform, train_set, train_loader, test_set, test_loader
import torch as t
import torch.nn as nn
import torchvision as tv
from torch import optim
from torch.autograd import Variable
# CIFAR-10的全部类别，一共10类
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
# 数据集
dataSet = DataSet(transform, train_set, train_loader, test_set, test_loader, classes)
# 网络结构
net = LeNet()
# 交叉熵损失函数
criterion = nn.CrossEntropyLoss()
# SGD优化器
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
trainer = Trainer(net, criterion, optimizer, dataSet.train_loader)
tester = Tester(dataSet.test_loader, net)
trainer.train(epochs=10)
tester.test()
def main():
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("using {} device.".format(device))
batch_size = 16
epochs = 200
data_transform = {
"train":
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
"val":
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
}
data_root = os.path.abspath(os.path.join(os.getcwd(),
".")) # get data root path
image_path = os.path.join(data_root, "data_set",
"flower_data") # flower data set path
assert os.path.exists(image_path), "{} path does not exist.".format(
image_path)
train_dataset = datasets.ImageFolder(root=os.path.join(
image_path, "train"),
transform=data_transform["train"])
train_num = len(train_dataset)
# {'daisy':0, 'dandelion':1, 'roses':2, 'sunflower':3, 'tulips':4}
flower_list = train_dataset.class_to_idx
cla_dict = dict((val, key) for key, val in flower_list.items())
# write dict into json file
json_str = json.dumps(cla_dict, indent=4)
with open('class_indices.json', 'w') as json_file:
json_file.write(json_str)
nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0,
8]) # number of workers
print('Using {} dataloader workers every process'.format(nw))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=nw)
validate_dataset = datasets.ImageFolder(root=os.path.join(
image_path, "val"),
transform=data_transform["val"])
val_num = len(validate_dataset)
validate_loader = torch.utils.data.DataLoader(validate_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=nw)
print("using {} images for training, {} images for validation.".format(
train_num, val_num))
# create model
net = LeNet(num_classes=5)
# load pretrained weights
# download url: https://download.pytorch.org/models/mobilenet_v3_large-8738ca79.pth
# download url: https://download.pytorch.org/models/mobilenet_v3_small-047dcff4.pth
# model_weight_path = "weights/LeNet_pretrained.pth"
# assert os.path.exists(model_weight_path), "file {} dose not exist.".format(model_weight_path)
# pre_weights = torch.load(model_weight_path, map_location=device)
# delete classifier weights
# pre_dict = {k: v for k, v in pre_weights.items() if net.state_dict()[k].numel() == v.numel()}
# missing_keys, unexpected_keys = net.load_state_dict(pre_dict, strict=False)
#
# # freeze features weights
# for param in net.conv_stem.parameters():
# param.requires_grad = False
#
# for param in net.bn1.parameters():
# param.requires_grad = False
#
# for param in net.act1.parameters():
# param.requires_grad = False
#
# for param in net.blocks.parameters():
# param.requires_grad = False
net.to(device)
# define loss function
loss_function = nn.CrossEntropyLoss()
# construct an optimizer
params = [p for p in net.parameters() if p.requires_grad]
optimizer = optim.Adam(params, lr=0.0001)
best_acc = 0.0
save_path = 'weights/lenet.pth'
train_steps = len(train_loader)
for epoch in range(epochs):
# train
net.train()
running_loss = 0.0
train_bar = tqdm(train_loader)
for step, data in enumerate(train_bar):
images, labels = data
optimizer.zero_grad()
logits = net(images.to(device))
loss = loss_function(logits, labels.to(device))
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
train_bar.desc = "train epoch[{}/{}] loss:{:.3f}".format(
epoch + 1, epochs, loss)
# validate
net.eval()
acc = 0.0 # accumulate accurate number / epoch
with torch.no_grad():
val_bar = tqdm(validate_loader)
for val_data in val_bar:
val_images, val_labels = val_data
outputs = net(val_images.to(device))
# loss = loss_function(outputs, test_labels)
predict_y = torch.max(outputs, dim=1)[1]
acc += torch.eq(predict_y, val_labels.to(device)).sum().item()
val_bar.desc = "valid epoch[{}/{}]".format(epoch + 1, epochs)
val_accurate = acc / val_num
print('[epoch %d] train_loss: %.3f val_accuracy: %.3f' %
(epoch + 1, running_loss / train_steps, val_accurate))
if val_accurate > best_acc:
best_acc = val_accurate
torch.save(net.state_dict(), save_path)
print('Finished Training')
cifar_test = dataset.Cifar10Dataset('./cifar10/test', transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]))
cifar_train_loader = DataLoader(cifar_train, batch_size=batch_size, shuffle=True)
cifar_test_loader = DataLoader(cifar_test, batch_size=batch_size, shuffle=False)
net = LeNet()
if MULTI_GPU:
net = nn.DataParallel(net,device_ids=device_ids)
net.to(device)
criteon = nn.CrossEntropyLoss()
optimizer=optim.Adam(net.parameters(), lr=1e-3)
scheduler = StepLR(optimizer, step_size=100, gamma=0.1)
if MULTI_GPU:
optimizer = nn.DataParallel(optimizer, device_ids=device_ids)
scheduler = nn.DataParallel(scheduler, device_ids=device_ids)
# print(net)
for epoch in range(epoch_num):
for batchidx, (label, img) in enumerate(cifar_train_loader):
net.train()
logits = net(img.to(device))
loss = criteon(logits, label.long().to(device))
optimizer.zero_grad()
loss.backward()
optimizer.module.step()
scheduler.module.step()
shuffle=False)
#实现单张图片可视化
# images,labels = next(iter(train_loader))
# img = torchvision.utils.make_grid(images)
# img = img.numpy().transpose(1,2,0)
# # img.shape
# std = [0.5,0.5,0.5]
# mean = [0.5,0.5,0.5]
# img = img*std +mean
# cv2.imshow('win',img)
# key_pressed = cv2.waitKey(0)
net = LeNet().to(device)
criterion = nn.CrossEntropyLoss() #定义损失函数
optimizer = torch.optim.SGD(net.parameters(), lr=LR, momentum=Momentum)
epoch = 10
if __name__ == '__main__':
for epoch in range(epoch):
sum_loss = 0.0
for i, data in enumerate(train_loader):
inputs, labels = data
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad() #将梯度归零
outputs = net(inputs) #将数据传入网络进行前向运算
loss = criterion(outputs, labels) #得到损失函数
loss.backward() #反向传播
optimizer.step() #通过梯度做一步参数更新
# print(loss)
def main(args):
# Data Loader (Input Pipeline)
model_str = args.dataset + '_%s_' % args.model_type + args.noise_type + '_' + str(args.noise_rate) + '_' + str(args.seed)
txtfile = save_dir + "/" + model_str + ".txt"
nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
if os.path.exists(txtfile):
os.system('mv %s %s' % (txtfile, txtfile + ".bak-%s" % nowTime))
# Data Loader (Input Pipeline)
print('loading dataset...')
train_dataset, val_dataset, test_dataset = load_data(args)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=True)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
# Define models
print('building model...')
if args.dataset == 'mnist':
clf1 = LeNet()
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1)
elif args.dataset == 'fmnist':
clf1 = resnet.ResNet50(input_channel=1, num_classes=10)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1)
elif args.dataset == 'cifar10':
clf1 = resnet.ResNet50(input_channel=3, num_classes=10)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1)
elif args.dataset == 'cifar100':
clf1 = resnet.ResNet50(input_channel=3, num_classes=100)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1)
clf1.cuda()
with open(txtfile, "a") as myfile:
myfile.write('epoch train_acc1 val_acc1 test_acc1\n')
epoch = 0
train_acc1 = 0
# evaluate models with random weights
val_acc1 = evaluate(val_loader, clf1)
print('Epoch [%d/%d] Val Accuracy on the %s val data: Model1 %.4f %%' % (
epoch + 1, args.n_epoch, len(val_dataset), val_acc1))
test_acc1 = evaluate(test_loader, clf1)
print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % (
epoch + 1, args.n_epoch, len(test_dataset), test_acc1))
# save results
with open(txtfile, "a") as myfile:
myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n")
val_acc_list = []
test_acc_list = []
for epoch in range(0, args.n_epoch):
scheduler1.step()
print(optimizer1.state_dict()['param_groups'][0]['lr'])
clf1.train()
train_acc1 = train(train_loader, epoch, clf1, optimizer1, args)
val_acc1 = evaluate(val_loader, clf1)
val_acc_list.append(val_acc1)
test_acc1 = evaluate(test_loader, clf1)
test_acc_list.append(test_acc1)
# save results
print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %% ' % (
epoch + 1, args.n_epoch, len(test_dataset), test_acc1))
with open(txtfile, "a") as myfile:
myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n")
id = np.argmax(np.array(val_acc_list))
test_acc_max = test_acc_list[id]
return test_acc_max
download=True)
train_loader = torch.utils.data.DataLoader(dataset=train_set,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_set,
batch_size=batch_size,
shuffle=False)
print("==>>> total number of training batches : {}".format(len(train_loader)))
print("==>>> total number of testing batches : {}".format(len(test_loader)))
print("==>>> Batch Size is : {}".format(batch_size))
model = LeNet(num_classes).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr)
num_batches = len(train_loader)
for epoch in range(num_epochs):
for idx, (inputs, labels) in enumerate(train_loader):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
loss = criterion(outputs, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if ((idx + 1) % 100 == 0):
print("epoch is {}/{} Step is: {}/{} loss is: {}".format(
epoch, num_epochs, idx, num_batches, loss.item()))
import os
import sys
pardir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(pardir)
from dataset import *
from result_save_visualization import *
from model import LeNet
import torch
import torch.optim as optim
import time
net = LeNet().to(device)
optimizer = optim.SGD(net.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=5e-4)
num_epochs = 50
data_dict = {
'epoch': [],
'time': [],
'train_loss': [],
'train_acc': [],
'val_loss': [],
'val_acc': []
}
start = time.time()
for epoch in range(num_epochs):
# train
net.train()
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
test_loss /= len(validate_loader.dataset)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(validate_loader.dataset),
100. * correct / len(validate_loader.dataset)))
if args.lenet:
model = LeNet()
else:
model = Net()
if args.cuda:
model.cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
model.train()
for epoch in range(args.epochs):
for batch_idx, (data, target) in enumerate(train_loader):
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
learning_rate = 1e-3
batch_size = 64
epoches = 50
trans_img = transforms.ToTensor()
trainset = MNIST('./data', train=True, download=True, transform=trans_img)
testset = MNIST('./data', train=False, download=True, transform=trans_img)
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4)
testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)
lenet = LeNet()
criterion = nn.CrossEntropyLoss(size_average=False)
optimizer = optim.SGD(lenet.parameters(), lr=learning_rate)
trainer = Trainer(lenet, criterion)
print('starting train......')
trainer.train(10, trainloader, optimizer)
print('finish train..................')
print()
print()
sum = 0
runloss = 0
correct = 0
for _, batch in enumerate(testloader):
input, target = batch
sum += len(target)
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=batch_size, shuffle=False)
# Define which model to use
model = LeNet(mask=True).to(device)
# Define Optimizer
optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0001)
initial_optimizer_state_dict = optimizer.state_dict()
# Initial training
print("--------------- Initial training ---------------")
train(model, optimizer, train_loader, Nepochs)
accuracy = test(model, test_loader)
print("Initial Accuracy:",accuracy)
torch.save(model, f"saves/initial_model.ptmodel")
print("--------------- Stats Before Pruning ---------------")
print_nonzeros(model)
print("")
# Pruning
print("--------------- Pruning ---------------")
model.prune_by_std(sensitivity)
import torch
import torch.nn as nn
from model import LeNet
import torch.optim as optim
from data import data_train_loader,data_test_loader
from prog import args
model=LeNet()
model.train() #切换模型到训练状态
#lr=0.01
criterion=nn.CrossEntropyLoss()
optimizer=optim.SGD(model.parameters(),lr=args.lr,momentum=0.9,weight_decay=5e-4)
epoch=5
for epoch_id in range(epoch):
train_loss=0
correct=0
total=0
for batch_idx, (inputs,targets) in enumerate(data_train_loader):
optimizer.zero_grad()
outputs=model(inputs)
loss=criterion(outputs,targets)
loss.backward()
optimizer.step()
train_loss+=loss.item()
_,predicted=outputs.max(1)
total+=targets.size(0)
correct+=predicted.eq(targets).sum().item()
print(epoch_id, batch_idx,len(data_train_loader),'Loss:%.3f|Acc:%.3f%%(%d/%d)'%(train_loss/(batch_idx+1),100.*correct/total,correct,total))
save_info={
"""
train_loader = DataLoader(dataset=train_data,
batch_size=batch_size,
shuffle=True,
drop_last=True)
valid_loader = DataLoader(dataset=valid_set,
batch_size=batch_size,
shuffle=False,
drop_last=True)
model = LeNet().to(device)
# Define loss & Optimizer
criterion = nn.CrossEntropyLoss().to(device) # Multiclass classification 전용
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) # SGD
total_batch_train = len(train_loader)
total_batch_val = len(valid_loader)
# Training/Validation Loss, and Accuracy History
train_loss_log = []
train_accuracy_log = []
top5_train_accuracy_log = []
valid_loss_log = []
valid_accuracy_log = []
top5_valid_accuracy_log = []
# History for Confusion Matrix
pred_list = torch.zeros(0, dtype=torch.long, device='cpu')
def train():
save_flag = True
trainloader = get_dataset(r'H:/DataSet_All/猫狗识别/gpu/train',
batch_size=64,
imageindex=0)
validationloader = get_dataset(r'H:/DataSet_All/猫狗识别/gpu/test',
batch_size=64,
imageindex=0)
save_dir = 'output/'
my_model = LeNet()
my_model = model_init(my_model)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(my_model.parameters(), lr=0.001)
epochs = 300
# 训练
for epoch in range(epochs):
loss_list = []
acc_list = []
set_learning_rate(optimizer, epoch)
learning_rate = optimizer.param_groups[0]['lr']
tq = tqdm.tqdm(trainloader, desc='train')
tq.set_description('train Epoch{} lr{}'.format(
epoch, learning_rate))
for images, labels in tq:
images = images.to(device)
labels = labels.to(device)
outputs = my_model(images)
loss = criterion(outputs, labels)
my_model.zero_grad()
loss.backward()
optimizer.step()
loss_list.append(loss.item())
loss_ave = sum(loss_list) / len(loss_list)
_, predicted = torch.max(outputs.data, 1)
accuracy = (predicted == labels).sum().float() / labels.size(0)
acc_list.append(accuracy)
acc_ave = sum(acc_list) / len(acc_list)
tq.set_postfix(
loss="%.4f accuracy:%.4f loss_ave:%.5f acc_ave:%.5f " %
(loss.item(), accuracy, loss_ave, acc_ave))
if save_flag:
log = "\ntrain \tEpoch {}/{} \t Learning rate: {:.5f} \t Train loss_ave: {:.5f} \t acc_ave: {:.5f} \t " \
.format(epoch, epochs, learning_rate, loss_ave, acc_ave )
# print(log)
logFile = open(save_dir + '/log.txt', 'a')
logFile.write(log + '\n')
torch.save(my_model.state_dict(), save_dir + '/model_lastest.pt')
if epoch % 1 == 0:
loss_list = []
acc_list = []
with torch.no_grad():
tq = tqdm.tqdm(validationloader, desc='teat')
for images, labels in tq:
images = images.to(device)
labels = labels.to(device)
outputs = my_model(images)
validation_loss = criterion(outputs, labels)
loss_list.append(validation_loss.item())
_, predicted = torch.max(outputs.data, 1)
accuracy = (predicted
== labels).sum().float() / labels.size(0)
acc_list.append(accuracy)
acc_ave = sum(acc_list) / len(acc_list)
loss_ave = sum(loss_list) / len(loss_list)
tq.set_postfix(
test_loss=
"%.4f acc:%.4f loss_ave:%.5f acc_ave:%.5f " %
(validation_loss, accuracy, loss_ave, acc_ave))
log = "\ntest \tEpoch {}/{} \t Learning rate: {:.5f} \t Train loss_ave: {:.5f} \t acc_ave: {:.5f} \t " \
.format(epoch, epochs, learning_rate, loss_ave, acc_ave)
# print(log)
logFile = open(save_dir + '/log.txt', 'a')
logFile.write(log + '\n')
""" 该代码定义了LeNet模型的训练过程
"""
import torch
import torch.nn as nn
from model import LeNet
# ... 此处略去定义训练数据载入器的代码，具体可参考代码4.3
model = LeNet() # 定义LeNet模型
model.train() # 切换模型到训练状态
lr = 0.01 # 定义学习率
criterion = nn.CrossEntropyLoss() # 定义损失函数
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9,
weight_decay=5e-4) # 定义随机梯度下降优化器
train_loss = 0
correct = 0
total = 0
for batch_idx, (inputs, targets) in enumerate(data_train_loader):
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
train_loss += loss.item()
_, predicted = outputs.max(1)
total += targets.size(0)
data_train_loader = DataLoader(data_train,
batch_size=256,
shuffle=True,
num_workers=0) #8个线程处理，随机抽取，batch=256
# data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8)
## 数据载入
# from data import data_train_loader
#### 定义模型训练参数部分
model = LeNet()
model.cpu()
model.train() ## 切换到模型的训练模式
lr = 0.001
loss_define = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=5e-4)
train_loss = 0 ## 训练损失
correct = 0 ## 识别正确个数
total = 0 ## 总个数
# momentum=0.9,
epoch_num = 10
# int(input('input epoch:'))
loss_plot = []
for epoch in range(epoch_num):
####
for batch_idx, (inputs, targets) in enumerate(data_train_loader):
output = model(inputs) ##识别10个手写数字（0~9），因此output输出10个概率值
loss = loss_define(output, targets) ##前向传播计算出损失
loss.backward() ##对损失进行反向传播
train=False,
download=False,
transform=transform)
val_loader = torch.utils.data.DataLoader(val_set,
batch_size=5000,
shuffle=False,
num_workers=0)
val_data_iter = iter(val_loader)
val_image, val_label = val_data_iter.next()
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
net = LeNet()
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)
for epoch in range(50): # loop over the dataset multiple times
running_loss = 0.0
for step, data in enumerate(train_loader, start=0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = loss_function(outputs, labels)
loss.backward()
optimizer.step()
def train():
device = torch.device("cuda:0" if opt.cuda else "cpu")
utils.set_seed()
# ============================ step 1/5 数据 ============================
norm_mean = [0.485, 0.456, 0.406]
norm_std = [0.229, 0.224, 0.225]
train_transform = transforms.Compose([
transforms.Resize((32, 32)),
transforms.RandomCrop(32, padding=4),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std),
])
valid_transform = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std),
])
# 构建MyDataset
train_data = RMBDataset(data_dir=opt.train_dir, transform=train_transform)
valid_data = RMBDataset(data_dir=opt.valid_dir, transform=valid_transform)
# 构建DataLoader
train_loader = DataLoader(dataset=train_data,
batch_size=opt.batch_size,
shuffle=True)
valid_loader = DataLoader(dataset=valid_data, batch_size=opt.batch_size)
# ============================ step 2/5 模型 ============================
net = LeNet(classes=2)
net.to(device)
# net.initialize_weights()
# ============================ step 3/5 损失函数 ============================
criterion = nn.CrossEntropyLoss()
# ============================ step 4/5 优化器 ============================
optimizer = optim.SGD(net.parameters(), lr=opt.lr, momentum=0.9)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
# ============================ step 5/5 训练 ============================
train_curve = list()
valid_curve = list()
for epoch in range(opt.epochs):
loss_mean = 0.
correct = 0.
total = 0.
net.train()
for i, data in enumerate(train_loader):
inputs, labels = data
inputs = inputs.to(device)
labels = labels.to(device)
outputs = net(inputs)
optimizer.zero_grad()
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# 统计分类情况
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).squeeze().sum().to("cpu").numpy()
# 打印训练信息
loss_mean += loss.item()
train_curve.append(loss.item())
if (i + 1) % opt.log_interval == 0:
loss_mean = loss_mean / opt.log_interval
print(
"Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}"
.format(epoch, opt.epochs, i + 1, len(train_loader),
loss_mean, correct / total))
loss_mean = 0.
scheduler.step() # 更新学习率
if (epoch + 1) % opt.val_interval == 0:
correct_val = 0.
total_val = 0.
loss_val = 0.
net.eval()
with torch.no_grad():
for j, data in enumerate(valid_loader):
inputs, labels = data
inputs = inputs.to(device)
labels = labels.to(device)
outputs = net(inputs)
loss = criterion(outputs, labels)
_, predicted = torch.max(outputs.data, 1)
total_val += labels.size(0)
correct_val += (
predicted == labels).squeeze().sum().to("cpu").numpy()
loss_val += loss.item()
valid_curve.append(loss_val)
print(
"Valid:\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}"
.format(epoch, opt.epochs, j + 1, len(valid_loader),
loss_val, correct / total))
utils.loss_picture(train_curve, train_loader, valid_curve,
opt.val_interval)
# 保存模型参数
net_state_dict = net.state_dict()
torch.save(net_state_dict, opt.path_state_dict)
print("模型保存成功")
def train_and_test(flags,
corruption_level=0,
gold_fraction=0.5,
get_C=uniform_mix_C):
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed(1)
C = get_C(corruption_level)
gold, silver = prepare_data(C, gold_fraction)
print("Gold shape = {}, Silver shape = {}".format(gold.images.shape,
silver.images.shape))
# TODO : test on whole set
test_x = torch.from_numpy(mnist.test.images[:500].reshape([-1, 1, 28, 28]))
test_y = torch.from_numpy(mnist.test.labels[:500]).type(torch.LongTensor)
print("Test shape = {}".format(test_x.shape))
model = LeNet()
optimizer = torch.optim.Adam([p for p in model.parameters()], lr=0.001)
for step in range(flags.num_steps):
x, y = silver.next_batch(flags.batch_size)
y, y_true = np.array([l[0] for l in y]), np.array([l[1] for l in y])
x_val, y_val = gold.next_batch(min(flags.batch_size, flags.nval))
x, y = torch.from_numpy(x.reshape(
[-1, 1, 28, 28])), torch.from_numpy(y).type(torch.LongTensor)
x_val, y_val = torch.from_numpy(x_val.reshape(
[-1, 1, 28, 28])), torch.from_numpy(y_val).type(torch.LongTensor)
# forward
if flags.method == "l2w":
ex_wts = reweight_autodiff(model, x, y, x_val, y_val)
logits, loss = model.loss(x, y, ex_wts)
if step % dbg_steps == 0:
tbrd.log_histogram("ex_wts", ex_wts, step=step)
tbrd.log_value("More_than_0.01",
sum([x > 0.01 for x in ex_wts]),
step=step)
tbrd.log_value("More_than_0.05",
sum([x > 0.05 for x in ex_wts]),
step=step)
tbrd.log_value("More_than_0.1",
sum([x > 0.1 for x in ex_wts]),
step=step)
mean_on_clean_labels = np.mean(
[ex_wts[i] for i in range(len(y)) if y[i] == y_true[i]])
mean_on_dirty_labels = np.mean(
[ex_wts[i] for i in range(len(y)) if y[i] != y_true[i]])
tbrd.log_value("mean_on_clean_labels",
mean_on_clean_labels,
step=step)
tbrd.log_value("mean_on_dirty_labels",
mean_on_dirty_labels,
step=step)
else:
logits, loss = model.loss(x, y)
print("Loss = {}".format(loss))
# backward
optimizer.zero_grad()
loss.backward()
optimizer.step()
tbrd.log_value("loss", loss, step=step)
if step % dbg_steps == 0:
model.eval()
pred = torch.max(model.forward(test_x), 1)[1]
test_acc = torch.sum(torch.eq(pred, test_y)).item() / float(
test_y.shape[0])
model.train()
print("Test acc = {}.".format(test_acc))
tbrd.log_value("test_acc", test_acc, step=step)
shuffle=True,
num_workers=4)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4)
#define LeNet model
model = LeNet(1, 10)
use_gpu = torch.cuda.is_available()
if use_gpu:
model = model.cuda()
# define loss and optimizer function
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# start training
for epoch in range(num_epoches):
train_loss = 0.0
train_acc = 0.0
for i, data in enumerate(train_loader, 1):
img, label = data
if use_gpu:
img = img.cuda()
label = label.cuda()
img = Variable(img)
label = Variable(label)
# forward propagate
# build model
if args.arc == 'LeNet':
model = LeNet()
elif args.arc.startswith('VGG'):
model = VGG(args.arc)
else:
model = LeNet()
if args.cuda:
print('Using CUDA with {0} GPUs'.format(torch.cuda.device_count()))
model = torch.nn.DataParallel(model, device_ids=[0]).cuda()
# define optimizer
if args.optimizer.lower() == 'adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr)
elif args.optimizer.lower() == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
else:
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
best_valid_loss = np.inf
iteration = 0
epoch = 1
# don't do early stopping because the validation set is contaminated now
while epoch < args.epochs + 1:
train_data = read_images(os.path.join(args.data_dir,
'train-images.idx3-ubyte'))
train_labels = read_labels(
os.path.join(args.data_dir, 'train-labels.idx1-ubyte'))
test_data = read_images(os.path.join(args.data_dir, 't10k-images.idx3-ubyte'))
test_labels = read_labels(os.path.join(args.data_dir,
't10k-labels.idx1-ubyte'))
# normalize
train_data = (train_data - train_data.mean(
(1, 2), keepdims=True)) / train_data.std((1, 2), keepdims=True)
test_data = (test_data - test_data.mean(
(1, 2), keepdims=True)) / test_data.std((1, 2), keepdims=True)
my_net = LeNet()
optimizer = SGD(my_net.parameters(), lr, momentum)
loss_history = []
epoch_steps = train_data.shape[0] // batch + 1
avg_loss = avg_acc = 0
for e in range(epoch):
if e and e % 3 == 0:
optimizer.lr *= 0.1
train_loss = train_acc = 0
e_data, e_labels = shuffle(train_data, train_labels)
with tqdm(total=epoch_steps) as pbar:
for x, t in zip(np.array_split(e_data, epoch_steps),
