def submit(**kwargs): ''' 测试验证集,并生成可提交的json文件 ''' opt.parse(kwargs) # 模型 model = getattr(models,opt.model)(opt) model.load(opt.load_path) model.eval().cuda() # 数据 dataset = ClsDataset(opt) dataset.test() dataloader = t.utils.data.DataLoader(dataset,opt.batch_size, shuffle=False, num_workers=opt.workers,pin_memory=True) # 测试 results = [] for ii, data in tqdm.tqdm(enumerate(dataloader)): input, label,image_ids = data val_input = Variable(input, volatile=True).cuda() val_label = Variable(label.type(t.LongTensor), volatile=True).cuda() score = model(val_input) predict = score.data.topk(k=3)[1].tolist() result = [ {"image_id": image_id, "label_id": label_id } for image_id,label_id in zip(image_ids,predict) ] results+=result # 保存文件 with open(opt.result_path,'w') as f: json.dump(results,f)
def train(**kwargs): # train model opt.parse(kwargs) batch_size, validationRatio = opt.batch_size, opt.validationRatio LR, epoch, global_seed = opt.LR, opt.epoch, opt.global_seed vis.vis.env = opt.env fixSeed(opt.global_seed) min_loss = 10 model = models.SimpleNet() for i in range(epoch): print("Ensamble number:" + str(i)) #model = models.ResNetLike(BasicBlock, [1, 3, 3, 1], num_channels=2, num_classes=1) data, full_img = readSuffleData(opt.global_seed, opt.BASE_FOLDER) train_loader, val_loader, train_ds, val_ds = getTrainValLoaders( data, full_img, batch_size, num_workers, validationRatio) #train_loader, val_loader, train_ds, val_ds = getCustomTrainValLoaders(data,full_img,batch_size,num_workers,validationRatio,global_seed) model, val_result, train_result = generateSingleModel( model, train_loader, val_loader, train_ds, val_ds, LR, opt.global_epoches) # if min_loss<val_result: # LR = LR * 0.5 # else: # min_loss = val_result vis.plot('val_loss', val_result) vis.plot("train_loss", train_result) #print (model) df_pred = testModel(model) savePred(df_pred, val_result)
def submit(**kwargs): ''' 测试验证集,并生成可提交的json文件 ''' opt.parse(kwargs) # 模型 model = getattr(models, opt.model)(opt) model.load(opt.load_path) model.eval().cuda() # 数据 dataset = ClsDataset(opt) dataset.test() dataloader = t.utils.data.DataLoader(dataset, opt.batch_size, shuffle=False, num_workers=opt.workers, pin_memory=True) # 测试 results = [] for ii, data in tqdm.tqdm(enumerate(dataloader)): input, label, image_ids = data bs, ncrops, c, h, w = input.size() test_input = Variable(input.view(-1, c, h, w), volatile=True).cuda() scores = model(test_input) prob = t.nn.functional.softmax(scores) prob_avg = prob.view(bs, ncrops, -1).mean(1) predict = prob_avg.data.topk(k=3)[1].tolist() result = [{ "image_id": image_id, "label_id": label_id } for image_id, label_id in zip(image_ids, predict)] results += result # 保存文件 with open(opt.result_path, 'w') as f: json.dump(results, f)
def train(**kwargs): ''' 训练模型 ''' opt.parse(kwargs) lr1, lr2 = opt.lr1, opt.lr2 vis.vis.env = opt.env # 模型 model = getattr(models,opt.model)(opt) if opt.load_path: model.load(opt.load_path) print(model) model.cuda() optimizer = model.get_optimizer(lr1,lr2) criterion = getattr(models,opt.loss)() # 指标:求均值 loss_meter = meter.AverageValueMeter() acc_meter = meter.AverageValueMeter() top1_meter = meter.AverageValueMeter() step = 0 max_acc = 0 vis.vis.texts = '' # 数据 dataset = ClsDataset(opt) dataloader = t.utils.data.DataLoader(dataset, opt.batch_size, opt.shuffle, num_workers=opt.workers,pin_memory=True) # 训练 for epoch in range(opt.max_epoch): loss_meter.reset() acc_meter.reset() top1_meter.reset() for ii, data in tqdm.tqdm(enumerate(dataloader, 0)): # 训练 optimizer.zero_grad() input, label, _ = data input = Variable(input.cuda()) label = Variable(label.cuda()) output = model(input).squeeze() error = criterion(output, label) error.backward() optimizer.step() # 计算损失的均值和训练集的准确率均值 loss_meter.add(error.data[0]) acc = topk_acc(output.data,label.data) acc_meter.add(acc) top1_acc = topk_acc(output.data,label.data,k=1) top1_meter.add(top1_acc) # 可视化 if (ii+1) % opt.plot_every == 0: if os.path.exists(opt.debug_file): ipdb.set_trace() log_values = dict(loss = loss_meter.value()[0], train_acc = acc_meter.value()[0], epoch = epoch, ii = ii, train_top1_acc= top1_meter.value()[0] ) vis.plot_many(log_values) # 数据跑一遍之后,计算在验证集上的分数 accuracy,top1_accuracy = val(model,dataset) vis.plot('val_acc', accuracy) vis.plot('val_top1',top1_accuracy) info = time.strftime('[%m%d_%H%M%S]') + 'epoch:{epoch},val_acc:{val_acc},lr:{lr},val_top1:{val_top1},train_acc:{train_acc}<br>'.format( epoch=epoch, lr=lr1, val_acc=accuracy, val_top1=top1_accuracy, train_acc=acc_meter.value() ) vis.vis.texts += info vis.vis.text(vis.vis.texts, win=u'log') # 调整学习率 # 如果验证集上准确率降低了,就降低学习率,并加载之前的最好模型 # 否则保存模型,并记下模型保存路径 if accuracy > max_acc: max_acc = accuracy best_path = model.save(accuracy) else: if lr1==0: lr1=lr2 model.load(best_path) lr1, lr2 = lr1 *opt.lr_decay, lr2 * opt.lr_decay optimizer = model.get_optimizer(lr1,lr2) vis.vis.save([opt.env])
def train(**kwargs): opt.parse(kwargs) lr1, lr2 = opt.lr1, opt.lr2 lr3 = opt.lr3 vis.vis.env = opt.env max_acc = 0 # 模型 model = getattr(models, opt.model)(opt) optimizer = model.get_optimizer(opt.model, lr1, lr2, lr3) if opt.load_path: #load optimizer + model #checkpoint = t.load(opt.load_path,lambda storage, loc: storage) checkpoint = t.load(opt.load_path) model.load_state_dict(checkpoint['d']) optimizer.load_state_dict(checkpoint['optimizer']) max_acc = checkpoint['acc'] print('using checkpoint:{}'.format(opt.load_path)) print('old config:') print(checkpoint['opt']) print(model) model.cuda() criterion = getattr(models, opt.loss)() # 指标:求均值 loss_meter = meter.AverageValueMeter() acc_meter = meter.AverageValueMeter() top1_meter = meter.AverageValueMeter() vis.vis.texts = '' # 数据 dataset = ClsDataset() dataloader = t.utils.data.DataLoader(dataset, opt.batch_size, opt.shuffle, num_workers=opt.workers, pin_memory=True) time_begin = time.time() for epoch in range(opt.max_epoch): loss_meter.reset() acc_meter.reset() top1_meter.reset() for ii, data in tqdm.tqdm(enumerate(dataloader, 0)): # 训练 optimizer.zero_grad() input, label, _ = data input = Variable(input.cuda()) label = Variable(label.cuda()) output = model(input).squeeze() error = criterion(output, label) error.backward() optimizer.step() # 计算损失的均值和训练集的准确率均值 loss_meter.add(error.data[0]) acc = topk_acc(output.data, label.data) acc_meter.add(acc) top1_acc = topk_acc(output.data, label.data, k=1) top1_meter.add(top1_acc) # 可视化 if (ii + 1) % opt.plot_every == 0: if os.path.exists(opt.debug_file): ipdb.set_trace() log_values = dict(loss=loss_meter.value()[0], train_acc=acc_meter.value()[0], epoch=epoch, ii=ii, train_top1_acc=top1_meter.value()[0]) vis.plot_many(log_values) # 数据跑一遍之后,计算在验证集上的分数 accuracy, top1_accuracy = val(model, dataset) vis.plot('val_acc', accuracy) vis.plot('val_top1', top1_accuracy) info = time.strftime( '[%m%d_%H%M%S]' ) + 'epoch:{epoch},train_acc:{train_acc},mac_acc:{max_acc},val_acc:{val_acc},lr:{lr}<br>'.format( epoch=epoch, lr=lr1, train_acc=acc_meter.value(), val_acc=accuracy, max_acc=max_acc #val_top1=top1_accuracy ) vis.vis.texts += info # 调整学习率 # 如果验证集上准确率降低了,就降低学习率,并加载之前的最好模型 # 否则保存模型,并记下模型保存路径 if accuracy > max_acc: max_acc = accuracy best_path = model.save(accuracy) else: if lr1 == 0: lr1 = lr2 if lr3: lr3 = lr1 lr3 = lr3 * opt.lr_decay model.load(best_path) lr1, lr2 = lr1 * opt.lr_decay, lr2 * opt.lr_decay optimizer = model.get_optimizer(opt.model, lr1, lr2, lr3) vis.vis.texts += 'change learning_rate' #for param_group in optimizer.param_groups: # lr = init_lr * (0.5 ** (epoch // lr_decay_epoch)) # param_group['lr'] = lr # param_group['weight_decay'] = weight_decay vis.vis.text(vis.vis.texts, win=u'log') vis.vis.save([opt.env]) time_all = time.time() - time_begin print(time_all) print('Training complete in {:.0f}hour {:.0f}min'.format( time_all // 3600, time_all // 60)) print('Best val Acc: {:4f}'.format(max_acc))
def train(**kwargs): ''' 训练函数 :param kwargs: fire传进来的训练参数 :return: ''' opt.parse(kwargs) for k_,v_ in kwargs.items(): setattr(opt,k_,v_) if opt.vis: vis = Visualizer(opt.env) #step1:数据预处理 transforms = tv.transforms.Compose([ tv.transforms.Resize(opt.image_size), tv.transforms.CenterCrop(opt.image_size), tv.transforms.ToTensor(), tv.transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5)) ]) dataset = tv.datasets.ImageFolder(opt.data_path,transform=transforms) dataloader = t.utils.data.DataLoader(dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, drop_last=True) #step2: 定义网络 netg,netd = NetG(opt),NetD(opt) map_location = lambda storage,loc:storage if opt.netd_path: netd.load_state_dict(t.load(opt.netd_path, map_location=map_location)) if opt.netg_path: netg.load_state_dict(t.load(opt.netg_path, map_location=map_location)) #定义优化器和损失函数 optimizer_g = t.optim.Adam(netg.parameters(), opt.lrG, betas=(0.5, 0.999)) optimizer_d = t.optim.Adam(netd.parameters(), opt.lrD, betas=(0.5, 0.999)) criterion = t.nn.BCELoss() #真图片label为1,加图片label为0 #noise为网络的输入 true_labels = t.ones(opt.batch_size) fake_labels = t.zeros(opt.batch_size) fix_noises = t.randn(opt.batch_size,opt.nz,1,1) noises = t.randn(opt.batch_size,opt.nz,1,1) errord_meter = AverageValueMeter() errorg_meter = AverageValueMeter() if opt.gpu: device = t.device("cuda:0" if t.cuda.is_available() else "cpu") netd.to(device) netg.to(device) criterion.to(device) true_labels,fake_labels = true_labels.to(device),fake_labels.to(device) fix_noises,noises = fix_noises.to(device),noises.to(device) epochs = range(140) for epoch in iter(epochs): for ii,(img,_) in tqdm.tqdm(enumerate(dataloader),total=len(dataloader)): if opt.gpu: real_img = img.to(device) if ii%opt.d_every == 0: #每个batch训练一次鉴别器 optimizer_d.zero_grad() output = netd(real_img) #判断真图片(使其尽可能大) error_d_real = criterion(output,true_labels) error_d_real.backward() ##尽可能把假图片判断为错误 noises.data.copy_(t.randn(opt.batch_size,opt.nz,1,1)) fake_img = netg(noises).detach() #根据噪声生成假图 output = netd(fake_img) error_d_fake = criterion(output,fake_labels) error_d_fake.backward() optimizer_d.step() error_d = error_d_fake + error_d_real errord_meter.add(error_d.item()) if ii%opt.g_every == 0: #每5个batch更新一次生成器 #训练生成器 optimizer_g.zero_grad() noises.data.copy_(t.randn(opt.batch_size, opt.nz, 1, 1)) fake_img = netg(noises) output = netd(fake_img) error_g = criterion(output,true_labels) error_g.backward() optimizer_g.step() errord_meter.add(error_g.item()) if opt.vis and ii%opt.plot_time == opt.plot_time - 1: ##可视化 fix_fake_img = netg(fix_noises) #使用噪声生成图片 vis.images(fix_fake_img.data.cpu().numpy()[:64]*0.5+0.5,win = 'fixfake') # vis.images(real_img.data.cpu().numpy()[:64]*0.5+0.5,win = 'real') vis.plot('errord', errord_meter.value()[0]) vis.plot('errorg', errorg_meter.value()[0]) if epoch%opt.decay_every == opt.decay_every-1: #保存模型,图片 tv.utils.save_image(fix_fake_img.data[:64],'%s/new%s.png'%(opt.save_path,epoch), normalize=True,range=(-1,1)) t.save(netd.state_dict(), 'checkpoints/new_netd_%s.pth' % epoch) t.save(netg.state_dict(), 'checkpoints/new_netg_%s.pth' % epoch) errord_meter.reset() errorg_meter.reset() optimizer_g = t.optim.Adam(netg.parameters(), opt.lrG, betas=(0.5, 0.999)) optimizer_d = t.optim.Adam(netd.parameters(), opt.lrD, betas=(0.5, 0.999))