def submit(**kwargs):
'''
测试验证集，并生成可提交的json文件
'''
opt.parse(kwargs)
# 模型
model = getattr(models,opt.model)(opt)
model.load(opt.load_path)
model.eval().cuda()
# 数据
dataset = ClsDataset(opt)
dataset.test()
dataloader = t.utils.data.DataLoader(dataset,opt.batch_size, shuffle=False, num_workers=opt.workers,pin_memory=True)
# 测试
results = []
for ii, data in tqdm.tqdm(enumerate(dataloader)):
input, label,image_ids = data
val_input = Variable(input, volatile=True).cuda()
val_label = Variable(label.type(t.LongTensor), volatile=True).cuda()
score = model(val_input)
predict = score.data.topk(k=3)[1].tolist()
result = [ {"image_id": image_id,
"label_id": label_id }
for image_id,label_id in zip(image_ids,predict) ]
results+=result
# 保存文件
with open(opt.result_path,'w') as f:
json.dump(results,f)
def train(**kwargs):
# train model
opt.parse(kwargs)
batch_size, validationRatio = opt.batch_size, opt.validationRatio
LR, epoch, global_seed = opt.LR, opt.epoch, opt.global_seed
vis.vis.env = opt.env
fixSeed(opt.global_seed)
min_loss = 10
model = models.SimpleNet()
for i in range(epoch):
print("Ensamble number:" + str(i))
#model = models.ResNetLike(BasicBlock, [1, 3, 3, 1], num_channels=2, num_classes=1)
data, full_img = readSuffleData(opt.global_seed, opt.BASE_FOLDER)
train_loader, val_loader, train_ds, val_ds = getTrainValLoaders(
data, full_img, batch_size, num_workers, validationRatio)
#train_loader, val_loader, train_ds, val_ds = getCustomTrainValLoaders(data,full_img,batch_size,num_workers,validationRatio,global_seed)
model, val_result, train_result = generateSingleModel(
model, train_loader, val_loader, train_ds, val_ds, LR,
opt.global_epoches)
# if min_loss<val_result:
# LR = LR * 0.5
# else:
# min_loss = val_result
vis.plot('val_loss', val_result)
vis.plot("train_loss", train_result)
#print (model)
df_pred = testModel(model)
savePred(df_pred, val_result)
def submit(**kwargs):
'''
测试验证集，并生成可提交的json文件
'''
opt.parse(kwargs)
# 模型
model = getattr(models, opt.model)(opt)
model.load(opt.load_path)
model.eval().cuda()
# 数据
dataset = ClsDataset(opt)
dataset.test()
dataloader = t.utils.data.DataLoader(dataset,
opt.batch_size,
shuffle=False,
num_workers=opt.workers,
pin_memory=True)
# 测试
results = []
for ii, data in tqdm.tqdm(enumerate(dataloader)):
input, label, image_ids = data
bs, ncrops, c, h, w = input.size()
test_input = Variable(input.view(-1, c, h, w), volatile=True).cuda()
scores = model(test_input)
prob = t.nn.functional.softmax(scores)
prob_avg = prob.view(bs, ncrops, -1).mean(1)
predict = prob_avg.data.topk(k=3)[1].tolist()
result = [{
"image_id": image_id,
"label_id": label_id
} for image_id, label_id in zip(image_ids, predict)]
results += result
# 保存文件
with open(opt.result_path, 'w') as f:
json.dump(results, f)
def train(**kwargs):
'''
训练模型
'''
opt.parse(kwargs)
lr1, lr2 = opt.lr1, opt.lr2
vis.vis.env = opt.env
# 模型
model = getattr(models,opt.model)(opt)
if opt.load_path:
model.load(opt.load_path)
print(model)
model.cuda()
optimizer = model.get_optimizer(lr1,lr2)
criterion = getattr(models,opt.loss)()
# 指标：求均值
loss_meter = meter.AverageValueMeter()
acc_meter = meter.AverageValueMeter()
top1_meter = meter.AverageValueMeter()
step = 0
max_acc = 0
vis.vis.texts = ''
# 数据
dataset = ClsDataset(opt)
dataloader = t.utils.data.DataLoader(dataset, opt.batch_size, opt.shuffle, num_workers=opt.workers,pin_memory=True)
# 训练
for epoch in range(opt.max_epoch):
loss_meter.reset()
acc_meter.reset()
top1_meter.reset()
for ii, data in tqdm.tqdm(enumerate(dataloader, 0)):
# 训练
optimizer.zero_grad()
input, label, _ = data
input = Variable(input.cuda())
label = Variable(label.cuda())
output = model(input).squeeze()
error = criterion(output, label)
error.backward()
optimizer.step()
# 计算损失的均值和训练集的准确率均值
loss_meter.add(error.data[0])
acc = topk_acc(output.data,label.data)
acc_meter.add(acc)
top1_acc = topk_acc(output.data,label.data,k=1)
top1_meter.add(top1_acc)
# 可视化
if (ii+1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
log_values = dict(loss = loss_meter.value()[0],
train_acc = acc_meter.value()[0],
epoch = epoch,
ii = ii,
train_top1_acc= top1_meter.value()[0]
)
vis.plot_many(log_values)
# 数据跑一遍之后，计算在验证集上的分数
accuracy,top1_accuracy = val(model,dataset)
vis.plot('val_acc', accuracy)
vis.plot('val_top1',top1_accuracy)
info = time.strftime('[%m%d_%H%M%S]') + 'epoch:{epoch},val_acc:{val_acc},lr:{lr},val_top1:{val_top1},train_acc:{train_acc}<br>'.format(
epoch=epoch,
lr=lr1,
val_acc=accuracy,
val_top1=top1_accuracy,
train_acc=acc_meter.value()
)
vis.vis.texts += info
vis.vis.text(vis.vis.texts, win=u'log')
# 调整学习率
# 如果验证集上准确率降低了，就降低学习率，并加载之前的最好模型
# 否则保存模型，并记下模型保存路径
if accuracy > max_acc:
max_acc = accuracy
best_path = model.save(accuracy)
else:
if lr1==0: lr1=lr2
model.load(best_path)
lr1, lr2 = lr1 *opt.lr_decay, lr2 * opt.lr_decay
optimizer = model.get_optimizer(lr1,lr2)
vis.vis.save([opt.env])
def train(**kwargs):
opt.parse(kwargs)
lr1, lr2 = opt.lr1, opt.lr2
lr3 = opt.lr3
vis.vis.env = opt.env
max_acc = 0
# 模型
model = getattr(models, opt.model)(opt)
optimizer = model.get_optimizer(opt.model, lr1, lr2, lr3)
if opt.load_path: #load optimizer + model
#checkpoint = t.load(opt.load_path,lambda storage, loc: storage)
checkpoint = t.load(opt.load_path)
model.load_state_dict(checkpoint['d'])
optimizer.load_state_dict(checkpoint['optimizer'])
max_acc = checkpoint['acc']
print('using checkpoint:{}'.format(opt.load_path))
print('old config:')
print(checkpoint['opt'])
print(model)
model.cuda()
criterion = getattr(models, opt.loss)()
# 指标：求均值
loss_meter = meter.AverageValueMeter()
acc_meter = meter.AverageValueMeter()
top1_meter = meter.AverageValueMeter()
vis.vis.texts = ''
# 数据
dataset = ClsDataset()
dataloader = t.utils.data.DataLoader(dataset,
opt.batch_size,
opt.shuffle,
num_workers=opt.workers,
pin_memory=True)
time_begin = time.time()
for epoch in range(opt.max_epoch):
loss_meter.reset()
acc_meter.reset()
top1_meter.reset()
for ii, data in tqdm.tqdm(enumerate(dataloader, 0)):
# 训练
optimizer.zero_grad()
input, label, _ = data
input = Variable(input.cuda())
label = Variable(label.cuda())
output = model(input).squeeze()
error = criterion(output, label)
error.backward()
optimizer.step()
# 计算损失的均值和训练集的准确率均值
loss_meter.add(error.data[0])
acc = topk_acc(output.data, label.data)
acc_meter.add(acc)
top1_acc = topk_acc(output.data, label.data, k=1)
top1_meter.add(top1_acc)
# 可视化
if (ii + 1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
log_values = dict(loss=loss_meter.value()[0],
train_acc=acc_meter.value()[0],
epoch=epoch,
ii=ii,
train_top1_acc=top1_meter.value()[0])
vis.plot_many(log_values)
# 数据跑一遍之后，计算在验证集上的分数
accuracy, top1_accuracy = val(model, dataset)
vis.plot('val_acc', accuracy)
vis.plot('val_top1', top1_accuracy)
info = time.strftime(
'[%m%d_%H%M%S]'
) + 'epoch:{epoch},train_acc:{train_acc},mac_acc:{max_acc},val_acc:{val_acc},lr:{lr}<br>'.format(
epoch=epoch,
lr=lr1,
train_acc=acc_meter.value(),
val_acc=accuracy,
max_acc=max_acc
#val_top1=top1_accuracy
)
vis.vis.texts += info
# 调整学习率
# 如果验证集上准确率降低了，就降低学习率，并加载之前的最好模型
# 否则保存模型，并记下模型保存路径
if accuracy > max_acc:
max_acc = accuracy
best_path = model.save(accuracy)
else:
if lr1 == 0: lr1 = lr2
if lr3:
lr3 = lr1
lr3 = lr3 * opt.lr_decay
model.load(best_path)
lr1, lr2 = lr1 * opt.lr_decay, lr2 * opt.lr_decay
optimizer = model.get_optimizer(opt.model, lr1, lr2, lr3)
vis.vis.texts += 'change learning_rate'
#for param_group in optimizer.param_groups:
# lr = init_lr * (0.5 ** (epoch // lr_decay_epoch))
# param_group['lr'] = lr
# param_group['weight_decay'] = weight_decay
vis.vis.text(vis.vis.texts, win=u'log')
vis.vis.save([opt.env])
time_all = time.time() - time_begin
print(time_all)
print('Training complete in {:.0f}hour {:.0f}min'.format(
time_all // 3600, time_all // 60))
print('Best val Acc: {:4f}'.format(max_acc))
def train(**kwargs):
'''
训练函数
:param kwargs: fire传进来的训练参数
:return:
'''
opt.parse(kwargs)
for k_,v_ in kwargs.items():
setattr(opt,k_,v_)
if opt.vis:
vis = Visualizer(opt.env)
#step1：数据预处理
transforms = tv.transforms.Compose([
tv.transforms.Resize(opt.image_size),
tv.transforms.CenterCrop(opt.image_size),
tv.transforms.ToTensor(),
tv.transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
])
dataset = tv.datasets.ImageFolder(opt.data_path,transform=transforms)
dataloader = t.utils.data.DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
drop_last=True)
#step2: 定义网络
netg,netd = NetG(opt),NetD(opt)
map_location = lambda storage,loc:storage
if opt.netd_path:
netd.load_state_dict(t.load(opt.netd_path, map_location=map_location))
if opt.netg_path:
netg.load_state_dict(t.load(opt.netg_path, map_location=map_location))
#定义优化器和损失函数
optimizer_g = t.optim.Adam(netg.parameters(), opt.lrG, betas=(0.5, 0.999))
optimizer_d = t.optim.Adam(netd.parameters(), opt.lrD, betas=(0.5, 0.999))
criterion = t.nn.BCELoss()
#真图片label为1，加图片label为0
#noise为网络的输入
true_labels = t.ones(opt.batch_size)
fake_labels = t.zeros(opt.batch_size)
fix_noises = t.randn(opt.batch_size,opt.nz,1,1)
noises = t.randn(opt.batch_size,opt.nz,1,1)
errord_meter = AverageValueMeter()
errorg_meter = AverageValueMeter()
if opt.gpu:
device = t.device("cuda:0" if t.cuda.is_available() else "cpu")
netd.to(device)
netg.to(device)
criterion.to(device)
true_labels,fake_labels = true_labels.to(device),fake_labels.to(device)
fix_noises,noises = fix_noises.to(device),noises.to(device)
epochs = range(140)
for epoch in iter(epochs):
for ii,(img,_) in tqdm.tqdm(enumerate(dataloader),total=len(dataloader)):
if opt.gpu:
real_img = img.to(device)
if ii%opt.d_every == 0: #每个batch训练一次鉴别器
optimizer_d.zero_grad()
output = netd(real_img) #判断真图片(使其尽可能大)
error_d_real = criterion(output,true_labels)
error_d_real.backward()
##尽可能把假图片判断为错误
noises.data.copy_(t.randn(opt.batch_size,opt.nz,1,1))
fake_img = netg(noises).detach() #根据噪声生成假图
output = netd(fake_img)
error_d_fake = criterion(output,fake_labels)
error_d_fake.backward()
optimizer_d.step()
error_d = error_d_fake + error_d_real
errord_meter.add(error_d.item())
if ii%opt.g_every == 0: #每5个batch更新一次生成器
#训练生成器
optimizer_g.zero_grad()
noises.data.copy_(t.randn(opt.batch_size, opt.nz, 1, 1))
fake_img = netg(noises)
output = netd(fake_img)
error_g = criterion(output,true_labels)
error_g.backward()
optimizer_g.step()
errord_meter.add(error_g.item())
if opt.vis and ii%opt.plot_time == opt.plot_time - 1:
##可视化
fix_fake_img = netg(fix_noises) #使用噪声生成图片
vis.images(fix_fake_img.data.cpu().numpy()[:64]*0.5+0.5,win = 'fixfake')
# vis.images(real_img.data.cpu().numpy()[:64]*0.5+0.5,win = 'real')
vis.plot('errord', errord_meter.value()[0])
vis.plot('errorg', errorg_meter.value()[0])
if epoch%opt.decay_every == opt.decay_every-1:
#保存模型，图片
tv.utils.save_image(fix_fake_img.data[:64],'%s/new%s.png'%(opt.save_path,epoch),
normalize=True,range=(-1,1))
t.save(netd.state_dict(), 'checkpoints/new_netd_%s.pth' % epoch)
t.save(netg.state_dict(), 'checkpoints/new_netg_%s.pth' % epoch)
errord_meter.reset()
errorg_meter.reset()
optimizer_g = t.optim.Adam(netg.parameters(), opt.lrG, betas=(0.5, 0.999))
optimizer_d = t.optim.Adam(netd.parameters(), opt.lrD, betas=(0.5, 0.999))
