def __init__(self, track):
self.__track = track
self.__vis = Visualizer(track.get_track(), {
Generator.VALID: Color.WHITE,
Generator.INVALID: Color.BLACK,
Generator.FINISH: Color.LIME,
Generator.START: Color.GRAY
},
cell_size=10)
def generate(**kwargs):
# step: configure
opt._parse(**kwargs)
device = t.device('cuda') if opt.use_gpu else t.device('cpu')
vis = Visualizer(env=opt.env)
# step: data
# 这里加载数据的方式和train的略微不一样，
# 这个问题其实从第七章就应该注意到，因为gen和val的区别主要在与data上，val的data和train的data是差不多的，但是gen的data一般只是一个
# train直接获取了dataloader，但是这里
# 这里的data有两个，一个是字， 一个是图片
# word
# train he generate dui shuju de chuli ,s shifou keyi fangzai yiqi ????????????????????????????
data = t.load(opt.caption_data_path, map_location = lambda s, _:s)
word2ix, ix2word, end = data['word2ix'], data['ix2word'], data['end']
# picture
# 这里的代码和feature_extract中的代码有很大的相似性。但是离main已经隔了一层了，所以重新写吧
#
transforms = T.Compose([
T.Resize(opt.scale_size),
T.CenterCrop(opt.img_size),
T.ToTensor(),
normalize
])
img = Image.open(opt.test_img).convert('RGB')
img = transforms(img).unsqueeze(0)
img = img.to(device)
# step: model: FeatureExtractModel:resnet50 CaptionModel:caption_model
resnet50 = FeatureExtractModel()
resnet50.to(device)
resnet50.eval()
caption_model = CaptionModel(opt, len(ix2word))
caption_model.load(opt.model_ckpt_g)
caption_model.to(device)
caption_model.eval()
# step: generate
img_feats = resnet50(img) # 1*2048
img_feats = img_feats.data[0] # 2048
eos_id = word2ix[end]
cap_sentences, cap_scores = caption_model.generate(img = img_feats,eos_id = eos_id) #
cap_sentences = [ ''.join([ix2word[ix.item()] for ix in sentence]) for sentence in cap_sentences]
# vis.img('image', img)
info = '<br>'.join(cap_sentences)
vis.log(u'generate caption', info, False)
return(cap_sentences,cap_scores)
def train(self, train_data, val_data=None):
print('Now we begin training')
train_dataloader = DataLoader(train_data,
batch_size=self.opt.batch_size,
shuffle=True)
#val_dataloader = DataLoader(val_data,self.opt.batch_size,shuffle=True)
vis = Visualizer(env=self.opt.env)
if self.opt.use_gpu:
self.model.cuda()
previous_loss = 1e10
loss_meter = meter.AverageValueMeter()
Confusion_matrix = meter.ConfusionMeter(10)
for epoch in range(self.opt.max_epoch):
loss_meter.reset()
Confusion_matrix.reset()
for i, (data, label) in enumerate(train_dataloader, 0):
if self.opt.use_gpu:
data = data.cuda()
label = label.cuda()
self.optimizer.zero_grad()
score = self.model(data)
out_classes = T.argmax(score, 1)
target_digit = T.argmax(label, 1)
loss = self.criterion(score, label)
loss.backward()
self.optimizer.step()
#指标更新
loss_meter.add(loss.data.cpu())
Confusion_matrix.add(out_classes, target_digit)
accuracy = 100 * sum(
Confusion_matrix.value()[i, i]
for i in range(10)) / Confusion_matrix.value().sum()
if i % self.opt.print_freq == self.opt.print_freq - 1:
print('EPOCH:{0},i:{1},loss:%.6f'.format(epoch, i) %
loss.data.cpu())
vis.plot('loss', loss_meter.value()[0])
vis.plot('test_accuracy', accuracy)
if val_data:
val_cm, val_ac = self.test(val_data, val=True)
vis.plot('Val_accuracy', val_ac)
vis.img('Val Confusion_matrix', T.Tensor(val_cm.value()))
# 若损失不再下降则降低学习率
if loss_meter.value()[-1] > previous_loss:
self.opt.lr = self.opt.lr * self.opt.lr_decay
print('learning rate:{}'.format(self.opt.lr))
for param_group in self.optimizer.param_groups:
param_group['lr'] = self.opt.lr
previous_loss = loss_meter.value()[-1]
def train(opt):
seq = iaa.Sequential([
iaa.CropToFixedSize(opt.fineSize, opt.fineSize),
])
dataset_train = ImageDataset(opt.source_root_train,
opt.gt_root_train,
transform=seq)
dataset_test = ImageDataset(opt.source_root_test,
opt.gt_root_test,
transform=seq)
dataloader_train = DataLoader(dataset_train,
batch_size=opt.batchSize,
shuffle=True,
num_workers=opt.nThreads)
dataloader_test = DataLoader(dataset_test,
batch_size=opt.batchSize,
shuffle=False,
num_workers=opt.nThreads)
model = StainNet(opt.input_nc, opt.output_nc, opt.n_layer, opt.channels)
model = nn.DataParallel(model).cuda()
optimizer = SGD(model.parameters(), lr=opt.lr)
loss_function = torch.nn.L1Loss()
lrschedulr = lr_scheduler.CosineAnnealingLR(optimizer, opt.epoch)
vis = Visualizer(env=opt.name)
best_psnr = 0
for i in range(opt.epoch):
for j, (source_image,
target_image) in tqdm(enumerate(dataloader_train)):
target_image = target_image.cuda()
source_image = source_image.cuda()
output = model(source_image)
loss = loss_function(output, target_image)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (j + 1) % opt.display_freq == 0:
vis.plot("loss", float(loss))
vis.img("target image", target_image[0] * 0.5 + 0.5)
vis.img("source image", source_image[0] * 0.5 + 0.5)
vis.img("output", (output[0] * 0.5 + 0.5).clamp(0, 1))
if (i + 1) % 5 == 0:
test_result = test(model, dataloader_test)
vis.plot_many(test_result)
if best_psnr < test_result["psnr"]:
save_path = "{}/{}_best_psnr_layer{}_ch{}.pth".format(
opt.checkpoints_dir, opt.name, opt.n_layer, opt.channels)
best_psnr = test_result["psnr"]
torch.save(model.module.state_dict(), save_path)
print(save_path, test_result)
lrschedulr.step()
print("lrschedulr=", lrschedulr.get_last_lr())
def __init__(self, **kwargs):
''' externar -> init param -> data prepare -> model load -> learning def'''
opt.parse(kwargs)
self.env = opt.env
self.vis = Visualizer(opt.env)
self.vis.text(opt.notes)
self.evaluator = Evaluator(opt.num_classes)
self.best_acc = self.best_epoch = -1
self.train_loader, self.val_loader = self.data_process()
self.model = self.set_model()
self.criterion, self.optimizer, self.scheduler = self.learning(
self.model)
def train(**kwargs):
# Change configuration
opt.parse(kwargs)
# Create a new visualizer
vis = Visualizer(opt.env)
# Use string to aquire the desired model
# Equals to model = models.AlexNet()
model = getattr(models, opt.model)()
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu:
model.to(opt.device)
model.train()
# Data
trainset = Dogcat(opt.train_data_root, train = True, test = False)
valset = Dogcat(opt.train_data_root, train = False, test = False)
trainset_loader = DataLoader(trainset, opt.batch_size, shuffle = True, num_workers = opt.num_workers)
valset_loader = DataLoader(valset, opt.batch_size, shuffle = False, num_workers = opt.num_workers)
# Criterion(loss function) and Optimizer
criterion = torch.nn.CrossEntropyLoss()
lr = opt.lr
optimizer = torch.optim.Adam(model.parameters(), lr, weight_decay = opt.weight_decay)
for ep in range(opt.max_epoch):
for batch_idx, (data, label) in enumerate(trainset_loader):
if opt.use_gpu:
data, label = data.to(opt.device), label.to(opt.device)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, label)
def __init__(self, opt):
self.opt = opt
if self.opt.batchsize % self.opt.split != 0:
raise ValueError("#batchsize must be divisible w.r.t #split ")
self.opt.batchsize //= self.opt.split
self.epoch = self.opt.niter
self.visualizer = Visualizer(self.opt)
self.device = opt.device
self.global_iter = 0
self.dataloader_setup()
if self.opt.dataset == 'KDD99' and self.opt.setting != 'egbad':
self.create_D = MLP_Discriminator
self.create_G = MLP_Generator
elif self.opt.dataset == 'KDD99' and self.opt.setting == 'egbad':
self.create_En = MLP_Encoder
self.create_De = MLP_Decoder
self.creare_D = MLP_Discriminator
else:
self.create_D = define_D
self.create_G = define_G
if self.opt.phase == 'train':
# TODO: initialize network and optimizer
self.generator_setup()
self.discriminator_setup()
if self.opt.setting != 'egbad':
self.optims = {
"gen": self.optimizer_Gs,
"disc": self.optimizer_Ds
}
# TODO: define discriminator loss function
self.l_adv = nn.BCELoss()
self.l_con = nn.L1Loss()
self.l_lat = l2_loss
def __init__(self, opt):
self.opt = opt
if self.opt.batchsize % self.opt.split != 0:
raise ValueError("#batchsize must be divisible w.r.t #split ")
self.opt.batchsize //= self.opt.split
self.epoch = self.opt.niter
self.visualizer = Visualizer(self.opt)
self.device = opt.device
self.global_iter = 0
self.dataloader_setup()
if self.opt.dataset == 'KDD99':
self.create_D = MLP_Discriminator
self.create_G = MLP_Generator
else:
if self.opt.arch == 'DCGAN':
self.create_D = Discriminator
self.create_G = Generator
else:
self.create_D = define_D
self.create_G = define_G
if self.opt.phase == 'train':
# TODO: initialize network and optimizer
self.generator_setup()
self.discriminator_setup()
self.optims = {"gen": self.optimizer_Gs, "disc": self.optimizer_Ds}
# TODO: define discriminator loss function
self.l_adv = nn.BCELoss(reduction='mean')
self.l_con = con_loss(b=self.opt.scale_con, reduction='mean')
self.l_lat = lat_loss(sigma=self.opt.sigma_lat, reduction='mean')
def test(args):
data = [
'https://farm4.staticflickr.com/3491/4023151748_cb042e1794_z.jpg',
'https://farmhand.ie/wp-content/uploads/2016/04/35-1024x683.jpg',
'https://upload.wikimedia.org/wikipedia/commons/c/c0/See_armisvesi_in_finnland.JPG',
'https://images.pexels.com/photos/804463/pexels-photo-804463.jpeg?auto=compress&cs=tinysrgb&dpr=1&w=500',
'https://www.holidify.com/images/cmsuploads/compressed/800px-Hong_Kong_International_Airport_duty-free_shops_01_20200123112249.JPG',
'https://pbs.twimg.com/media/D9lWa1rUYAA_N-u.png',
'https://pbs.twimg.com/media/CJexA6hWIAAgr_F.jpg:large',
'https://upload.wikimedia.org/wikipedia/commons/9/94/TIFF_Event_Pecaut_02.jpg',
'https://res-1.cloudinary.com/hc2kqirjb/image/upload/c_fill,h_1200,q_45,w_2000/v1478680842/ftkx9c8lq7wayhjl785y.png'
]
visualizer = Visualizer()
print('Test started ...')
for i, f in enumerate(data):
detections = infer(f, args.server_url, args.threshold,
args.iou_threshold, args.class_id, args.poly_points)
visualizer(f, detections,
os.path.join(args.output_dir, f'image_{i}.jpg'))
print(f'- {i+1}/{len(data)} sample done ...')
print('Test ended.')
async def _view_spt(ctx, *ids):
""""""
if not ids:
return await ctx.send("Please give some ids to view those teams.")
if len(ids) == 1 and ids[0].lower() != "all":
party = await manager.get_static_party(ids[0])
if not party:
return await ctx.send("Team does not Exist")
return await ctx.send(embed=party.embed(bot))
if ids[0].lower() == "all":
events = await manager.get_all_static_parties()
if len(ids) > 1:
events = await manager.get_static_parties(ids)
if not events:
return await ctx.send("IDs do not Exist")
reactor = Visualizer(bot, ctx, events)
await reactor.start()
def main(opt):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = getattr(models, opt.model)().to(device)
train_loader = dataloader(opt, is_train=True)
test_loader = dataloader(opt, is_train=False)
vis = Visualizer(opt.env)
optimizer = torch.optim.Adam(params=model.parameters(),
lr=opt.lr,
weight_decay=opt.weight_decay)
if os.path.exists(opt.model_path):
try:
checkpoint = torch.load(opt.model_path)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoints '{}' (epoch {})".format(
opt.model_path, checkpoint['epoch']))
except RuntimeError:
start_epoch = 1
print("=> no checkpoints found at '{}'".format(opt.model_path))
else:
start_epoch = 1
print("=> no checkpoints found at '{}'".format(opt.model_path))
if opt.model == 'Basical':
criterion = torch.nn.MSELoss().to(device)
for epoch in range(start_epoch, opt.max_epoch + 1):
train_basical(model, criterion, optimizer, train_loader, epoch,
vis)
elif opt.model == 'Conv':
criterion = torch.nn.MSELoss().to(device)
for epoch in range(start_epoch, opt.max_epoch + 1):
train_conv(model, criterion, optimizer, train_loader, epoch, vis)
elif opt.model == 'VAE':
for epoch in range(start_epoch, opt.max_epoch + 1):
train_vae(model, vae_loss_function, optimizer, train_loader,
start_epoch, vis)
else:
print(
"No module nameed '{}', module's name must be '{} | {} | {}' please check again!"
.format(opt.model, 'Basical', 'Conv', 'VAE'))
async def _view_ev(ctx, *ids):
"""View Events
Options:
passing IDs will show those events. Example 1 2 3
passing 'all' will show all events which have not started yet
"""
if not ids:
return await ctx.send("Please give some ids to view those events")
if len(ids) == 1 and ids[0] != "all":
event = await manager.get_event(ids[0])
return await ctx.send(embed=event.embed(bot))
if ids[0] == "all":
events = await manager.events.get_entries("started = 'False'")
fmt = []
for i in events:
foo = Event(manager.events, i.id)
await foo.initialize()
fmt.append(foo)
events = fmt
else:
events = await manager.get_events(ids)
reactor = Visualizer(bot, ctx, events)
await reactor.start()
def visualize(**kwargs):
# 根据命令行参数更新配置
opt.parse(kwargs)
vis = Visualizer(opt.env)
# step1: 模型
model = getattr(models, opt.model)()
'''
model_ft = torchvision.models.vgg16_bn(pretrained = True)
pretrained_dict = model_ft.state_dict()
model_dict = model.state_dict()
# 将pretrained_dict里不属于model_dict的键剔除掉
pretrained_dict = {k: v for k, v in pretrained_dict.items()
if k in model_dict}
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
'''
if opt.load_model_path:
model.load(opt.load_model_path)
if opt.use_gpu:
model.cuda()
summary(model, (3, 224, 224))
print(opt)
# step2: 数据
train_data = myData(
filelists=opt.train_filelists,
#transform = data_transforms['train'],
scale=opt.cropscale,
transform=None,
test=False,
data_source='none')
val_data = myData(
filelists=opt.test_filelists,
#transform =data_transforms['val'],
transform=None,
scale=opt.cropscale,
test=False,
data_source='none')
train_loader = DataLoader(dataset=train_data, batch_size=1, shuffle=False)
val_loader = DataLoader(dataset=val_data, batch_size=1, shuffle=False)
dataloaders = {'train': train_loader, 'val': val_loader}
dataset_sizes = {'train': len(train_data), 'val': len(val_data)}
imgshape = 64
imgwidth_num = 32
def vis(train_loader, outputjpg, imgshape, imgwidth_num):
print(len(train_loader))
showtmp = np.zeros((imgshape, imgshape, 3), dtype=np.uint8)
showall = None
lastnum = imgwidth_num - len(train_loader) % imgwidth_num
for step, batch in enumerate(
tqdm(train_loader,
desc='Visual Cropface On Anti-spoofing',
unit='batch')):
inputs, labels = batch
inputs = inputs.numpy().squeeze()
inputs = np.transpose(inputs, (1, 2, 0))
inputs = np.uint8(inputs)
inputs = cv2.resize(inputs, (imgshape, imgshape))
if step % imgwidth_num == 0:
if showall is not None:
showall = np.vstack([showall, showtmp])
elif step > 0:
showall = showtmp
#print(showtmp.shape)
showtmp = inputs
else:
showtmp = np.hstack([showtmp, inputs])
#print(showtmp.shape)
for i in range(lastnum):
showtmp = np.hstack(
[showtmp,
np.zeros((imgshape, imgshape, 3), dtype=np.uint8)])
#print(showtmp.shape)
showall = np.vstack([showall, showtmp])
cv2.imwrite(outputjpg, showall)
vis(train_loader, 'data/showcropface_train.jpg', imgshape, imgwidth_num)
vis(val_loader, 'data/showcropface_val.jpg', imgshape, imgwidth_num)
from config.config import Config
from torch.nn import DataParallel
from torch.optim.lr_scheduler import StepLR
from test import *
def save_model(model, save_path, name, iter_cnt):
save_name = os.path.join(save_path, name + '_' + str(iter_cnt) + '.pth')
torch.save(model.state_dict(), save_name)
return save_name
if __name__ == '__main__':
opt = Config(os.getcwd())
if opt.display:
visualizer = Visualizer()
if torch.cuda.is_available(): # GPUが利用可能か確認
device = 'cuda'
else:
device = 'cpu'
print('device: {}'.format(device))
train_dataset = Dataset(opt.train_root, opt.train_list, phase='train', input_shape=opt.input_shape)
trainloader = data.DataLoader(train_dataset,
batch_size=opt.train_batch_size,
shuffle=True,
num_workers=opt.num_workers)
val_dataset = Dataset(opt.train_root, opt.val_list, phase='val', input_shape=opt.input_shape)
val_loader = data.DataLoader(val_dataset,
batch_size=opt.test_batch_size,
def main(args):
exp_path = os.path.join(args.output_dir, args.exp_name)
log_path = os.path.join(exp_path, 'logs')
checkpoint_path = os.path.join(exp_path, 'checkpoints')
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
if os.path.exists(exp_path):
print('Error: Experiment already exists, please rename --exp-name')
exit()
os.makedirs(log_path)
os.mkdir(checkpoint_path)
print("All experiment outputs will be saved within:", exp_path)
# set seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
# get models and load pre-trained disparity network
disp_net = DispNet.DispNet(1).to(device)
disp_net.init_weights()
if args.disp_net is not None:
disp_net.load_state_dict(torch.load(args.disp_net, map_location='cpu'))
disp_net.train()
pose_net = PoseNet.PoseNet(1, args.sequence_length - 1).to(device)
pose_net.init_weights()
pose_net.train()
# joint optimizer (pose and depth)
optim_params = [{
'params': disp_net.parameters(),
'lr': args.learning_rate
}, {
'params': pose_net.parameters(),
'lr': args.learning_rate
}]
optim = torch.optim.Adam(optim_params,
betas=(args.momentum, args.beta),
weight_decay=args.weight_decay)
# get sequential dataset
train_set = CPETDataset.CPET(args.dataset_dir, 'train',
args.sequence_length, args.seed)
val_set = CPETDataset.CPET(args.dataset_dir, 'val', args.sequence_length,
args.seed)
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True)
# custom view synthesis loss and depth smoothness loss
criterion = ViewSynthesisLoss(device, args.rotation_mode,
args.padding_mode)
w_synth, w_smooth = args.photo_loss_weight, args.smooth_loss_weight
# visualizer
visualizer = Visualizer(exp_path, device)
# commence experiment
print(
"Experiment commencing on 4 train seq and 1 validation seq for {} epochs..."
.format(args.epochs))
start_time = time.time()
# track losses and absolute trajectory error
train_loss = np.zeros((args.epochs, 3))
val_loss = np.zeros((args.epochs, 3))
val_ate_mean = np.zeros(args.epochs)
total_time = np.zeros(args.epochs)
for epo in range(args.epochs):
# run training epoch and generate / save random visualizations
l_train = train_epoch(disp_net, pose_net, train_loader, criterion,
optim, w_synth, w_smooth)
train_loss[epo, :] = l_train[:]
visualizer.generate_random_visuals(disp_net, pose_net, train_loader,
criterion, args.vis_per_epoch, epo,
'train')
# run validation epoch and acquire pose estimation metrics. Plot trajectories
l_val, ate, ate_mean, gt_traj, pred_traj = validate(
disp_net, pose_net, val_loader, criterion, w_synth, w_smooth)
val_loss[epo, :] = l_val[:]
val_ate_mean[epo] = ate_mean
# visualization of disparity maps, BEV trajectories, and 3D trajectories
visualizer.generate_random_visuals(disp_net, pose_net, val_loader,
criterion, args.vis_per_epoch, epo,
'val')
visualizer.generate_trajectory(pred_traj, 'pred', 'Estimated', epo,
'val')
visualizer.generate_trajectories(gt_traj, pred_traj, "Horns", epo,
'val')
visualizer.generate_3d_trajectory(gt_traj, pred_traj, "Horns", epo,
'val')
if epo == 0:
visualizer.generate_trajectory(gt_traj, 'gt', 'True', epo, 'val')
total_time[epo] = time.time() - start_time
print_str = "epo - {}/{} | train_loss - {:.3f} | val_loss - {:.3f} | ".format(
epo, args.epochs, train_loss[epo, 0], val_loss[epo, 0])
print_str += "val_ate - {:.3f} | total_time - {}".format(
ate_mean, datetime.timedelta(seconds=total_time[epo]))
print(print_str)
# save models
if (epo + 1) % args.save_freq == 0:
model_checkpoint(disp_net, 'disp_net_' + str(epo + 1),
checkpoint_path)
model_checkpoint(pose_net, 'pose_net_' + str(epo + 1),
checkpoint_path)
# save current stats
np.savetxt(os.path.join(log_path, 'train_loss.txt'), train_loss)
np.savetxt(os.path.join(log_path, 'val_loss.txt'), val_loss)
np.savetxt(os.path.join(log_path, 'val_ate_mean.txt'), val_ate_mean)
np.savetxt(os.path.join(log_path, 'time_log.txt'), total_time)
# generate metric curves
generate_curve([train_loss[:, 0], val_loss[:, 0]], ['train', 'val'],
'loss', 'Train vs Val Combined Loss', log_path)
generate_curve([train_loss[:, 1], val_loss[:, 1]], ['train', 'val'],
'photometric loss',
'Train vs Val Photometric Reconstruction Loss', log_path)
generate_curve([train_loss[:, 2], val_loss[:, 2]], ['train', 'val'],
'depth smooth loss', 'Train vs Val Depth Smoothness Loss',
log_path)
generate_curve([val_ate_mean], ['val'], 'ATE',
'Validation Absolute Trajectory Error', log_path)
def train(**kwargs):
# step: configure
opt._parse(**kwargs)
device = t.device('cuda') if opt.use_gpu else t.device('cpu')
vis = Visualizer(env=opt.env)
# step: data 这里是指要从数据加载过程中加载所需要的数据，所有数据只加载一次，也可以通过self的方式获取，
dataloader = get_dataloader(opt)
_data = dataloader.dataset.data
# word2ix, ix2word = _data.word2ix, _data.ix2word
# ix2id, id2ix = _data.ix2id, _data.id2ix
word2ix, ix2word = _data['word2ix'], _data['ix2word']
ix2id, id2ix, end = _data['ix2id'], _data['id2ix'], _data['end']
eos_id = word2ix[end]
# step: model
# 刚刚看了看作者写的模型，在保存模型的时候把opt也一并保存了，这是要做什么的，貌似是为了在进行生成的时候用的
# 为了避免以后有漏洞，在这里定义模型的时候输入参数暂且按照作者的来，然后在生成的时候再返回来看各个参数的作用
# 因为word2ix，ix2word是定义数据集的时候用的，按理来说跟模型没有关系才对
model = CaptionModel(opt, len(ix2word) )
if opt.model_ckpt:
model.load(opt.model_ckpt)
model.to(device)
# step: meter criterion optimizer
loss_meter = meter.AverageValueMeter()
criterion = t.nn.CrossEntropyLoss()
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
model.save()
# step: train
for epoch in range(opt.max_epoch):
loss_meter.reset()
for ii,(imgs, (captions, lengths), indexes) in tqdm.tqdm(enumerate(dataloader), total = len(dataloader)):
optimizer.zero_grad()
imgs = imgs.to(device)
captions = captions.to(device)
input_captions = captions[:-1]
target_captions = pack_padded_sequence(captions, lengths).data # len
score, _ = model(imgs, input_captions, lengths) # len*vocab
loss = criterion(score, target_captions)
loss.backward()
optimizer.step()
loss_meter.add(loss.data.item())
# step: visulize
if(ii+1)%opt.print_freq == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
# picture+caption
# indexes在这里用到了，因为要可视化图片，就需要知道当前是第几张图片，而主模型的输入是直接2048维特征。没有直接指示第几行图片，
# 同时也说明了，有序的重要性，所以在提取图片特征的时候，不是直接读取文件，而是从id2ix中获取，来使得标题和图片都可以从id2ix中找到对应关系
# 如果是我，我肯定不会想到用id和序号做一个对应关系，说不定直接用列表存储所有图片名称，
# 用列表终归不如用dict好，因为dict是可以反推回他是第几张图片 ix2id和id2ix,而列表只能是知道第几张图片的位置，不能反推。
img_path = os.path.join(opt.img_path, ix2id[indexes[0]])
raw_img = Image.open(img_path).convert('RGB')
raw_img = tv.transforms.ToTensor()(raw_img)
# captions_np = np.array(captions) # zheli shi weile bimian ziji buzhuyi er daozhi jisuantu de cunzai suoyi duiyu moxing de shuru he shuchu zuo qi ta caozuo shi douxian jiangqi xianshi zhuanhua cheng meiyou jisuantu biru detach() biru with t.no_grad() biru t_.data.tolist() biru t_.data biru qita
raw_caption = captions.data[:,0].tolist() #
raw_caption = ''.join([ix2word[i] for i in raw_caption])
# vis.img('raw_img', raw_img, caption=raw_caption)
info = '<br>'.join([ix2id[indexes[0]],raw_caption])
vis.log(u'raw_caption', info, False)
results, scores = model.generate(img=imgs.data[0], eos_id=eos_id)
cap_sentences = [ ''.join([ix2word[ix.item()] for ix in sentence]) for sentence in results]
info = '<br>'.join(cap_sentences)
info = '<br>'.join([ix2id[indexes[0]],info])
vis.log(u'val', info, False)
model.save()
import torch
import argparse
import torch.nn as nn
import torch.nn.functional as F # 各中层函数的实现，与层类型对应，如：卷积函数、池化函数、归一化函数等等
import torch.optim as optim # 实现各种优化算法的包
from torchvision import transforms
from torchnet import meter
from data.dataset import Dataset
from utils import Visualizer
import models
vis = Visualizer(env='my_wind') #为了可视化增加的内容
train_loss_meter = meter.AverageValueMeter() #为了可视化增加的内容
test_acc_meter = meter.AverageValueMeter() #为了可视化增加的内容
def train(args, model, device, train_loader, optimizer,
epoch): # 还可添加loss_func等参数
model.train() # 必备，将模型设置为训练模式
train_loss_meter.reset() # 为了可视化增加的内容
for batch_idx, (data,
target) in enumerate(train_loader): # 从数据加载器迭代一个batch的数据
data, target = data.to(device), target.to(device) # 将数据存储CPU或者GPU
optimizer.zero_grad() # 清除所有优化的梯度
output = model(data) # 喂入数据并前向传播获取输出
"""
pytorch中CrossEntropyLoss是通过两个步骤计算出来的:
第一步是计算log softmax，第二步是计算cross entropy（或者说是negative log likehood），
CrossEntropyLoss不需要在网络的最后一层添加softmax和log层，直接输出全连接层即可。
而NLLLoss则需要在定义网络的时候在最后一层添加log_softmax层(softmax和log层)
if args.mode == "train":
isTraining = True
else:
isTraining = False
database = build_dataset(args.dataset,
args.data_dir,
channel=args.input_nc,
isTraining=isTraining,
crop_size=(args.crop_size, args.crop_size),
scale_size=(args.scale_size, args.scale_size))
sub_dir = args.dataset + "/second_stage_v2" # + args.model + "/" + args.loss
if isTraining: # train
NAME = args.dataset + "_second_stage_v2-2nd" # + args.model + "_" + args.loss
viz = Visualizer(env=NAME)
writer = SummaryWriter(args.logs_dir + "/" + sub_dir)
mkdir(args.models_dir + "/" +
sub_dir) # two stage时可以创建first_stage和second_stage这两个子文件夹
# 加载数据集
train_dataloader = DataLoader(database,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True)
val_database = build_dataset(args.dataset,
args.data_dir,
channel=args.input_nc,
isTraining=False,
crop_size=(args.crop_size, args.crop_size),
scale_size=(args.scale_size, args.scale_size))
def multitask_train(**kwargs):
config.parse(kwargs)
vis = Visualizer(port=2333, env=config.env)
# prepare data
train_data = MultiLabel_Dataset(config.data_root,
config.train_paths,
phase='train',
balance=config.data_balance)
val_data = MultiLabel_Dataset(config.data_root,
config.test_paths,
phase='val',
balance=config.data_balance)
print('Training Images:', train_data.__len__(), 'Validation Images:',
val_data.__len__())
train_dataloader = DataLoader(train_data,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
val_dataloader = DataLoader(val_data,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.num_workers)
# prepare model
model = MultiTask_DenseNet121(num_classes=2) # 每一个分支都是2分类
# model = CheXPre_MultiTask_DenseNet121(num_classes=2) # 每一个分支都是2分类
if config.load_model_path:
model.load(config.load_model_path)
if config.use_gpu:
model.cuda()
model.train()
# criterion and optimizer
# F, T1, T2 = 3500, 3078, 3565 # 权重，分别是没病，成骨型，溶骨型的图片数量
# weight_1 = torch.FloatTensor([T1/(F+T1+T2), (F+T2)/(F+T1+T2)]).cuda() # weight也需要用cuda的
# weight_2 = torch.FloatTensor([T2/(F+T1+T2), (F+T1)/(F+T1+T2)]).cuda()
# criterion_1 = torch.nn.CrossEntropyLoss(weight=weight_1)
# criterion_2 = torch.nn.CrossEntropyLoss(weight=weight_2)
criterion_1 = torch.nn.CrossEntropyLoss()
criterion_2 = torch.nn.CrossEntropyLoss()
lr = config.lr
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=config.weight_decay)
# metrics
softmax = functional.softmax
loss_meter_1 = meter.AverageValueMeter()
loss_meter_2 = meter.AverageValueMeter()
loss_meter_total = meter.AverageValueMeter()
train_cm_1 = meter.ConfusionMeter(2) # 每个支路都是二分类，整体是三分类
train_cm_2 = meter.ConfusionMeter(2)
train_cm_total = meter.ConfusionMeter(3)
previous_loss = 100
previous_acc = 0
# train
if not os.path.exists(os.path.join('checkpoints', model.model_name)):
os.mkdir(os.path.join('checkpoints', model.model_name))
for epoch in range(config.max_epoch):
loss_meter_1.reset()
loss_meter_2.reset()
loss_meter_total.reset()
train_cm_1.reset()
train_cm_2.reset()
train_cm_total.reset()
# train
for i, (image, label_1, label_2, label,
image_path) in tqdm(enumerate(train_dataloader)):
# prepare input
img = Variable(image)
target_1 = Variable(label_1)
target_2 = Variable(label_2)
target = Variable(label)
if config.use_gpu:
img = img.cuda()
target_1 = target_1.cuda()
target_2 = target_2.cuda()
target = target.cuda()
# go through the model
score_1, score_2 = model(img)
# backpropagate
optimizer.zero_grad()
loss_1 = criterion_1(score_1, target_1)
loss_2 = criterion_2(score_2, target_2)
loss = loss_1 + loss_2
# loss.backward()
# optimizer.step()
loss_1.backward(
retain_graph=True) # 这里将两个loss相加后回传的效果不太好，反而是分别回传效果更好
optimizer.step() # 可能的原因是分别回传时的momentum算了两次，更容易突破局部最优解
loss_2.backward()
optimizer.step()
# calculate loss and confusion matrix
loss_meter_1.add(loss_1.data[0])
loss_meter_2.add(loss_2.data[0])
loss_meter_total.add(loss.data[0])
p_1, p_2 = softmax(score_1, dim=1), softmax(score_2, dim=1)
c = []
# -----------------------------------------------------------------------
for j in range(p_1.data.size()[0]): # 将两个支路合并得到最终的预测结果
if p_1.data[j][1] < 0.5 and p_2.data[j][1] < 0.5:
c.append([1, 0, 0])
else:
if p_1.data[j][1] > p_2.data[j][1]:
c.append([0, 1, 0])
else:
c.append([0, 0, 1])
# -----------------------------------------------------------------------
train_cm_1.add(p_1.data, target_1.data)
train_cm_2.add(p_2.data, target_2.data)
train_cm_total.add(torch.FloatTensor(c), target.data)
if i % config.print_freq == config.print_freq - 1:
vis.plot_many({
'loss_1': loss_meter_1.value()[0],
'loss_2': loss_meter_2.value()[0],
'loss_total': loss_meter_total.value()[0]
})
print('loss_1:',
loss_meter_1.value()[0], 'loss_2:',
loss_meter_2.value()[0], 'loss_total:',
loss_meter_total.value()[0])
# print result
train_accuracy_1 = 100. * sum(
[train_cm_1.value()[c][c]
for c in range(2)]) / train_cm_1.value().sum()
train_accuracy_2 = 100. * sum(
[train_cm_2.value()[c][c]
for c in range(2)]) / train_cm_2.value().sum()
train_accuracy_total = 100. * sum(
[train_cm_total.value()[c][c]
for c in range(3)]) / train_cm_total.value().sum()
val_cm_1, val_accuracy_1, val_loss_1, val_cm_2, val_accuracy_2, val_loss_2, val_cm_total, val_accuracy_total, val_loss_total = multitask_val(
model, val_dataloader)
if val_accuracy_total > previous_acc:
if config.save_model_name:
model.save(
os.path.join('checkpoints', model.model_name,
config.save_model_name))
else:
model.save(
os.path.join('checkpoints', model.model_name,
model.model_name + '_best_model.pth'))
previous_acc = val_accuracy_total
vis.plot_many({
'train_accuracy_1': train_accuracy_1,
'val_accuracy_1': val_accuracy_1,
'train_accuracy_2': train_accuracy_2,
'val_accuracy_2': val_accuracy_2,
'total_train_accuracy': train_accuracy_total,
'total_val_accuracy': val_accuracy_total
})
vis.log(
"epoch: [{epoch}/{total_epoch}], lr: {lr}, loss_1: {loss_1}, loss_2: {loss_2}, loss_total: {loss_total}"
.format(epoch=epoch + 1,
total_epoch=config.max_epoch,
lr=lr,
loss_1=loss_meter_1.value()[0],
loss_2=loss_meter_2.value()[0],
loss_total=loss_meter_total.value()[0]))
vis.log('train_cm_1:' + str(train_cm_1.value()) + ' train_cm_2:' +
str(train_cm_2.value()) + ' train_cm_total:' +
str(train_cm_total.value()))
vis.log('val_cm_1:' + str(val_cm_1.value()) + ' val_cm_2:' +
str(val_cm_2.value()) + ' val_cm_total:' +
str(val_cm_total.value()))
print('train_accuracy_1:', train_accuracy_1, 'val_accuracy_1:',
val_accuracy_1, 'train_accuracy_2:', train_accuracy_2,
'val_accuracy_2:', val_accuracy_2, 'total_train_accuracy:',
train_accuracy_total, 'total_val_accuracy:', val_accuracy_total)
print(
"epoch: [{epoch}/{total_epoch}], lr: {lr}, loss_1: {loss_1}, loss_2: {loss_2}, loss_total: {loss_total}"
.format(epoch=epoch + 1,
total_epoch=config.max_epoch,
lr=lr,
loss_1=loss_meter_1.value()[0],
loss_2=loss_meter_2.value()[0],
loss_total=loss_meter_total.value()[0]))
print('train_cm_1:\n' + str(train_cm_1.value()) + '\ntrain_cm_2:\n' +
str(train_cm_2.value()) + '\ntrain_cm_total:\n' +
str(train_cm_total.value()))
print('val_cm_1:\n' + str(val_cm_1.value()) + '\nval_cm_2:\n' +
str(val_cm_2.value()) + '\nval_cm_total:\n' +
str(val_cm_total.value()))
# update learning rate
if loss_meter_total.value()[0] > previous_loss: # 可以考虑分别用两支的loss来判断
lr = lr * config.lr_decay
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter_total.value()[0]
generator.cuda()
discriminator.cuda()
feature_extractor.cuda()
content_criterion.cuda()
adversarial_criterion.cuda()
ones_const = ones_const.cuda()
optim_generator = optim.Adam(generator.parameters(), lr=opt.generatorLR)
optim_discriminator = optim.Adam(discriminator.parameters(),
lr=opt.discriminatorLR)
configure('logs/' + opt.dataset + '-' + str(opt.batchSize) + '-' +
str(opt.generatorLR) + '-' + str(opt.discriminatorLR),
flush_secs=5)
visualizer = Visualizer(save_dir=opt.samples_dir,
show_step=opt.sample_freq,
image_size=opt.imageSize * opt.upSampling)
low_res = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize)
# Pre-train generator using raw MSE loss
print 'Generator pre-training'
for epoch in range(2):
mean_generator_content_loss = 0.0
# enumerate(tqdm(dataloader, desc='G pre-training', leave=False), 0)
for i, data in enumerate(dataloader):
# Generate data
high_res_real = data
# Downsample images to low resolution
for j in range(opt.batchSize):
import random
import PIL
from PIL import Image, ImageOps
import math
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision
import time
import copy
from utils import Visualizer
vis = Visualizer(env='lxg')
path = '/home/lxg/codedata/ice/'
def iso(arr):
p = np.reshape(np.array(arr), [75, 75]) > (np.mean(np.array(arr)) +
2 * np.std(np.array(arr)))
return p * np.reshape(np.array(arr), [75, 75])
def size(arr):
return float(np.sum(arr < -5)) / (75 * 75)
data = pd.read_json(path + 'train.json')
def train(**kwargs):
opt = Config()
for k, v in kwargs.items():
setattr(opt, k, v)
device=t.device('cuda') if opt.use_gpu else t.device('cpu')
opt.caption_data_path = 'caption.pth' # 原始数据
opt.test_img = '' # 输入图片
# opt.model_ckpt='caption_0914_1947' # 预训练的模型
# 数据
vis = Visualizer(env=opt.env)
dataloader = get_dataloader(opt)
_data = dataloader.dataset._data
word2ix, ix2word = _data['word2ix'], _data['ix2word']
# 模型
model = CaptionModel(opt, word2ix, ix2word)
if opt.model_ckpt:
model.load(opt.model_ckpt)
optimizer = model.get_optimizer(opt.lr)
criterion = t.nn.CrossEntropyLoss()
model.to(device)
# 统计
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for ii, (imgs, (captions, lengths), indexes) in tqdm.tqdm(enumerate(dataloader)):
# 训练
optimizer.zero_grad()
imgs = imgs.to(device)
captions = captions.to(device)
input_captions = captions[:-1]
target_captions = pack_padded_sequence(captions, lengths)[0]
score, _ = model(imgs, input_captions, lengths)
loss = criterion(score, target_captions)
loss.backward()
optimizer.step()
loss_meter.add(loss.item())
# 可视化
if (ii + 1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
# 可视化原始图片 + 可视化人工的描述语句
raw_img = _data['ix2id'][indexes[0]]
img_path = opt.img_path + raw_img
raw_img = Image.open(img_path).convert('RGB')
raw_img = tv.transforms.ToTensor()(raw_img)
raw_caption = captions.data[:, 0]
raw_caption = ''.join([_data['ix2word'][ii] for ii in raw_caption])
vis.text(raw_caption, u'raw_caption')
vis.img('raw', raw_img, caption=raw_caption)
# 可视化网络生成的描述语句
results = model.generate(imgs.data[0])
vis.text('</br>'.join(results), u'caption')
model.save()
def train(**kwargs):
print(kwargs)
start = time.time()
# 根据命令行参数更新配置
vis = Visualizer(opt.env)
opt.parse(kwargs)
# 加载词向量
print("Loading word vectors...Please wait.")
vector = KeyedVectors.load_word2vec_format(
os.path.join(os.path.dirname(os.path.realpath(opt.train_data_root)), 'vector.txt')
)
print("Successfully loaded word vectors.")
# step1: 模型
model = getattr(models, opt.model)(input_size=vector.vector_size+2, output_size=opt.class_num)
if opt.load_model_path:
model.load(opt.load_model_path) # 预加载
if opt.use_gpu and t.cuda.is_available():
model = model.cuda()
print(f"Structure of {model.model_name}:\n{model}\n")
# step2: 数据
train_data = Sentence(root=opt.train_data_root,
relations=opt.relations,
max_length=opt.max_length,
vector=vector,
train=True) # 训练集
train_dataloader = DataLoader(train_data, opt.batch_size, shuffle=True)
val_data = Sentence(opt.train_data_root, opt.relations, opt.max_length, vector, train=False) # 验证集
val_dataloader = DataLoader(val_data, opt.batch_size, shuffle=True)
# step3: 目标函数和优化器
loss_fn = t.nn.CrossEntropyLoss()
lr = opt.lr
optimizer = t.optim.Adam(params=model.parameters(),
lr=lr,
weight_decay = opt.weight_decay)
# step4: 统计指标：平滑处理之后的损失，还有混淆矩阵
loss_meter = meter.AverageValueMeter()
confusion_matrix = meter.ConfusionMeter(opt.class_num)
previous_loss = 1e100
# 训练
for epoch in range(opt.max_epoch):
loss_meter.reset()
confusion_matrix.reset()
for ii, (data, label) in enumerate(train_dataloader):
# 训练模型参数
input = data
target = label
if opt.use_gpu:
input = input.cuda()
target = target.cuda()
optimizer.zero_grad()
prediction = model(input)
loss = loss_fn(prediction, target)
loss.backward()
optimizer.step()
# 更新统计指标以及可视化
loss_meter.add(loss.item())
confusion_matrix.add(prediction.data, target.data)
# if ii % opt.print_freq == opt.print_freq - 1:
# vis.plot('train loss', loss_meter.value()[0])
# 如果需要的话，进入debug模式
# if os.path.exists(opt.debug_file):
# import ipdb;
# ipdb.set_trace()
cm_value = confusion_matrix.value()
correct = 0
for i in range(cm_value.shape[0]):
correct += cm_value[i][i]
accuracy = 100. * correct / (cm_value.sum())
vis.plot('train loss', loss_meter.value()[0])
vis.plot('train accuracy', accuracy)
if epoch % opt.save_epoch == opt.save_epoch -1:
model.save()
# 计算验证集上的指标及可视化
val_lm, val_cm, val_accuracy = val(model, val_dataloader)
vis.plot('val loss', val_lm.value()[0])
vis.plot('val accuracy', val_accuracy)
print("epoch:{epoch}, lr:{lr}, loss:{loss}\ntrain_cm:\n{train_cm}\nval_cm:\n{val_cm}"
.format(epoch=epoch,
loss=loss_meter.value()[0],
val_cm=str(val_cm.value()),
train_cm=str(confusion_matrix.value()),
lr=lr)
)
# 如果损失不再下降，则降低学习率
if loss_meter.value()[0] > previous_loss:
lr = lr * opt.lr_decay
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()[0]
cost = int(time.time()) - int(start)
print(f"Cost {int(cost/60)}min{cost%60}s.")
if opt.cuda:
generator.cuda()
discriminator.cuda()
feature_extractor.cuda()
content_criterion.cuda()
adversarial_criterion.cuda()
ones_const = ones_const.cuda()
optim_generator = optim.Adam(generator.parameters(), lr=opt.generatorLR)
optim_discriminator = optim.Adam(discriminator.parameters(),
lr=opt.discriminatorLR)
configure('logs/' + opt.dataset + '-' + str(opt.batchSize) + '-' +
str(opt.generatorLR) + '-' + str(opt.discriminatorLR),
flush_secs=5)
visualizer = Visualizer(image_size=opt.imageSize * opt.upSampling)
low_res = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize)
# Pre-train generator using raw MSE loss
print('Generator pre-training')
for epoch in range(2):
mean_generator_content_loss = 0.0
for i, data in enumerate(dataloader):
# Generate data
high_res_real, _ = data
# Downsample images to low resolution
for j in range(opt.batchSize):
low_res[j] = scale(high_res_real[j])
from mxnet.gluon.data import vision
import numpy as np
from mxnet.gluon import nn
from matplotlib import pyplot as plt
from utils import Visualizer
train_dir = 'train'
test_dir = 'test'
batch_size = 128
data_dir = '/media/yijie/娱乐/tmp/kaggle_cifar10'
label_file = 'D:/dataset/gluon/train_valid_test/trainLabels.csv'
input_dir = 'D:/dataset/gluon/train_valid_test'
valid_ratio = 0.1
pre_deal_flag = True
vis = Visualizer(env='CIFAR10')
# sorting the dataset and transform
if not pre_deal_flag:
pre_deal.reorg_cifar10_data(data_dir, label_file, train_dir, test_dir,
input_dir, valid_ratio)
input_str = input_dir + '/'
# 读取原始图像文件。flag=1说明输入图像有三个通道（彩色）。
train_ds = vision.ImageFolderDataset(input_str + 'train',
flag=1,
transform=pre_deal.transform_train)
valid_ds = vision.ImageFolderDataset(input_str + 'valid',
flag=1,
transform=pre_deal.transform_test)
def train(**kwargs):
for k, v in kwargs.items():
setattr(opt, k, v)
vis = Visualizer(env=opt.env)
# 获取数据
data, word2ix, ix2word = get_data(opt)
data = t.from_numpy(data)
dataloader = t.utils.data.DataLoader(data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
# 模型定义
model = PoetryModel(len(word2ix), 128, 256)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.use_gpu:
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for ii, data_ in tqdm.tqdm(enumerate(dataloader)):
# 训练
data_ = data_.long().transpose(1, 0).contiguous()
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()
input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :])
output, _ = model(input_)
loss = criterion(output, target.view(-1))
loss.backward()
optimizer.step()
loss_meter.add(loss.data[0])
# 可视化
if (1 + ii) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
# 诗歌原文
poetrys = [[ix2word[_word] for _word in data_[:, _iii]]
for _iii in range(data_.size(1))][:16]
vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]), win=u'origin_poem')
gen_poetries = []
# 分别以这几个字作为诗歌的第一个字，生成8首诗
for word in list(u'春江花月夜凉如水'):
gen_poetry = ''.join(generate(model, word, ix2word, word2ix))
gen_poetries.append(gen_poetry)
vis.text('</br>'.join([''.join(poetry) for poetry in gen_poetries]), win=u'gen_poem')
t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs):
opt = Config()
for k, v in kwargs.items():
setattr(opt, k, v)
vis = Visualizer(env=opt.env)
dataloader = get_dataloader(opt)
_data = dataloader.dataset._data
word2ix, ix2word = _data['word2ix'], _data['ix2word']
# cnn = tv.models.resnet50(True)
model = CaptionModel(opt, None, word2ix, ix2word)
if opt.model_ckpt:
model.load(opt.model_ckpt)
optimizer = model.get_optimizer(opt.lr1)
criterion = t.nn.CrossEntropyLoss()
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
perplexity = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
perplexity.reset()
for ii, (imgs, (captions, lengths),
indexes) in tqdm.tqdm(enumerate(dataloader)):
optimizer.zero_grad()
input_captions = captions[:-1]
imgs = imgs.cuda()
captions = captions.cuda()
imgs = Variable(imgs)
captions = Variable(captions)
input_captions = captions[:-1]
target_captions = pack_padded_sequence(captions, lengths)[0]
score, _ = model(imgs, input_captions, lengths)
loss = criterion(score, target_captions)
loss.backward()
# clip_grad_norm(model.rnn.parameters(),opt.grad_clip)
optimizer.step()
loss_meter.add(loss.data[0])
perplexity.add(t.exp(loss.data)[0])
# 可视化
if (ii + 1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
vis.plot('perplexity', perplexity.value()[0])
# 可视化原始图片
raw_img = _data['train']['ix2id'][indexes[0]]
img_path = '/data/image/ai_cha/caption/ai_challenger_caption_train_20170902/caption_train_images_20170902/' + raw_img
raw_img = Image.open(img_path).convert('RGB')
raw_img = tv.transforms.ToTensor()(raw_img)
vis.img('raw', raw_img)
# raw_img = (imgs.data[0]*0.25+0.45).clamp(max=1,min=0)
# vis.img('raw',raw_img)
# 可视化人工的描述语句
raw_caption = captions.data[:, 0]
raw_caption = ''.join(
[_data['ix2word'][ii] for ii in raw_caption])
vis.text(raw_caption, u'raw_caption')
# 可视化网络生成的描述语句
results = model.generate(imgs.data[0])
vis.text('</br>'.join(results), u'caption')
if (epoch + 1) % 100 == 0:
model.save()
def train(**kwargs):
for k, v in kwargs.items():
setattr(opt, k, v)
vis = Visualizer(env=opt.env)
#获取数据
data, word2ix, ix2word = get_data(opt)
data = t.from_numpy(data)
dataloader = t.utils.data.DataLoader(data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=1)
#模型定义
model = PoetryModel(len(word2ix), 128, 256)
optimizer = t.optim.Adam(model.parameters(), lr=opt.lr)
criterion = nn.CrossEntropyLoss()
if opt.model_path:
model.load_state_dict(t.load(opt.model_path))
if opt.use_gpu:
model.cuda()
criterion.cuda()
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for li, data_ in tqdm.tqdm(enumerate(dataloader)):
#训练
data_ = data_.long().transpose(1, 0).contiguous()
if opt.use_gpu: data_ = data_.cuda()
optimizer.zero_grad()
##输入和目标错开
input_, target = Variable(data_[:-1, :]), Variable(data_[1:, :])
output, _ = model(input_)
loss = criterion(output, target.view(-1))
loss.backward()
optimizer.step()
loss_meter.add(loss.data[0])
# 可视化
if (1 + ii) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
#诗歌原文
poetrys = [[ix2word[_word] for _word in data_[:, -iii]]
for _iii in range(data_.size(1))][:16]
vis.text('</br>'.join([''.join(poetry) for poetry in poetrys]),
win=u'origin_poem')
gen_poetries = []
#分别以这几个字作为诗歌的第一个字，生成8首诗
for word in list(u'春江花月夜凉如水'):
gen_poetry = ''.join(
generate(model, word, ix2word, word2ix))
gen_poetries.append(gen_poetry)
vis.text('</br>'.join(
[''.join(poetry) for poetry in gen_poetries]),
win=u'gen_poem')
t.save(model.state_dict(), '%s_%s.pth' % (opt.model_prefix, epoch))
def train(**kwargs):
opt.parse(kwargs)
if opt.vis_env:
vis = Visualizer(opt.vis_env, port=opt.vis_port)
if opt.device is None or opt.device is 'cpu':
opt.device = torch.device('cpu')
else:
opt.device = torch.device(opt.device)
images, tags, labels = load_data(opt.data_path, type=opt.dataset)
train_data = Dataset(opt, images, tags, labels)
train_dataloader = DataLoader(train_data, batch_size=opt.batch_size, shuffle=True)
L = train_data.get_labels()
L = L.to(opt.device)
# test
i_query_data = Dataset(opt, images, tags, labels, test='image.query')
i_db_data = Dataset(opt, images, tags, labels, test='image.db')
t_query_data = Dataset(opt, images, tags, labels, test='text.query')
t_db_data = Dataset(opt, images, tags, labels, test='text.db')
i_query_dataloader = DataLoader(i_query_data, opt.batch_size, shuffle=False)
i_db_dataloader = DataLoader(i_db_data, opt.batch_size, shuffle=False)
t_query_dataloader = DataLoader(t_query_data, opt.batch_size, shuffle=False)
t_db_dataloader = DataLoader(t_db_data, opt.batch_size, shuffle=False)
query_labels, db_labels = i_query_data.get_labels()
query_labels = query_labels.to(opt.device)
db_labels = db_labels.to(opt.device)
pretrain_model = load_pretrain_model(opt.pretrain_model_path)
generator = GEN(opt.dropout, opt.image_dim, opt.text_dim, opt.hidden_dim, opt.bit, opt.num_label, pretrain_model=pretrain_model).to(opt.device)
discriminator = DIS(opt.hidden_dim//4, opt.hidden_dim//8, opt.bit).to(opt.device)
optimizer = Adam([
# {'params': generator.cnn_f.parameters()}, ## froze parameters of cnn_f
{'params': generator.image_module.parameters()},
{'params': generator.text_module.parameters()},
{'params': generator.hash_module.parameters()}
], lr=opt.lr, weight_decay=0.0005)
optimizer_dis = {
'feature': Adam(discriminator.feature_dis.parameters(), lr=opt.lr, betas=(0.5, 0.9), weight_decay=0.0001),
'hash': Adam(discriminator.hash_dis.parameters(), lr=opt.lr, betas=(0.5, 0.9), weight_decay=0.0001)
}
tri_loss = TripletLoss(opt, reduction='sum')
loss = []
max_mapi2t = 0.
max_mapt2i = 0.
max_average = 0.
mapt2i_list = []
mapi2t_list = []
train_times = []
B_i = torch.randn(opt.training_size, opt.bit).sign().to(opt.device)
B_t = B_i
H_i = torch.zeros(opt.training_size, opt.bit).to(opt.device)
H_t = torch.zeros(opt.training_size, opt.bit).to(opt.device)
for epoch in range(opt.max_epoch):
t1 = time.time()
e_loss = 0
for i, (ind, img, txt, label) in tqdm(enumerate(train_dataloader)):
imgs = img.to(opt.device)
txt = txt.to(opt.device)
labels = label.to(opt.device)
batch_size = len(ind)
h_i, h_t, f_i, f_t = generator(imgs, txt)
H_i[ind, :] = h_i.data
H_t[ind, :] = h_t.data
h_t_detach = generator.generate_txt_code(txt)
#####
# train feature discriminator
#####
D_real_feature = discriminator.dis_feature(f_i.detach())
D_real_feature = -opt.gamma * torch.log(torch.sigmoid(D_real_feature)).mean()
# D_real_feature = -D_real_feature.mean()
optimizer_dis['feature'].zero_grad()
D_real_feature.backward()
# train with fake
D_fake_feature = discriminator.dis_feature(f_t.detach())
D_fake_feature = -opt.gamma * torch.log(torch.ones(batch_size).to(opt.device) - torch.sigmoid(D_fake_feature)).mean()
# D_fake_feature = D_fake_feature.mean()
D_fake_feature.backward()
# train with gradient penalty
alpha = torch.rand(batch_size, opt.hidden_dim//4).to(opt.device)
interpolates = alpha * f_i.detach() + (1 - alpha) * f_t.detach()
interpolates.requires_grad_()
disc_interpolates = discriminator.dis_feature(interpolates)
gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
grad_outputs=torch.ones(disc_interpolates.size()).to(opt.device),
create_graph=True, retain_graph=True, only_inputs=True)[0]
gradients = gradients.view(gradients.size(0), -1)
# 10 is gradient penalty hyperparameter
feature_gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10
feature_gradient_penalty.backward()
optimizer_dis['feature'].step()
#####
# train hash discriminator
#####
D_real_hash = discriminator.dis_hash(h_i.detach())
D_real_hash = -opt.gamma * torch.log(torch.sigmoid(D_real_hash)).mean()
optimizer_dis['hash'].zero_grad()
D_real_hash.backward()
# train with fake
D_fake_hash = discriminator.dis_hash(h_t.detach())
D_fake_hash = -opt.gamma * torch.log(torch.ones(batch_size).to(opt.device) - torch.sigmoid(D_fake_hash)).mean()
D_fake_hash.backward()
# train with gradient penalty
alpha = torch.rand(batch_size, opt.bit).to(opt.device)
interpolates = alpha * h_i.detach() + (1 - alpha) * h_t.detach()
interpolates.requires_grad_()
disc_interpolates = discriminator.dis_hash(interpolates)
gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
grad_outputs=torch.ones(disc_interpolates.size()).to(opt.device),
create_graph=True, retain_graph=True, only_inputs=True)[0]
gradients = gradients.view(gradients.size(0), -1)
hash_gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10
hash_gradient_penalty.backward()
optimizer_dis['hash'].step()
loss_G_txt_feature = -torch.log(torch.sigmoid(discriminator.dis_feature(f_t))).mean()
loss_adver_feature = loss_G_txt_feature
loss_G_txt_hash = -torch.log(torch.sigmoid(discriminator.dis_hash(h_t_detach))).mean()
loss_adver_hash = loss_G_txt_hash
tri_i2t = tri_loss(h_i, labels, target=h_t, margin=opt.margin)
tri_t2i = tri_loss(h_t, labels, target=h_i, margin=opt.margin)
weighted_cos_tri = tri_i2t + tri_t2i
i_ql = torch.sum(torch.pow(B_i[ind, :] - h_i, 2))
t_ql = torch.sum(torch.pow(B_t[ind, :] - h_t, 2))
loss_quant = i_ql + t_ql
err = opt.alpha * weighted_cos_tri + \
opt.beta * loss_quant + opt.gamma * (loss_adver_feature + loss_adver_hash)
optimizer.zero_grad()
err.backward()
optimizer.step()
e_loss = err + e_loss
P_i = torch.inverse(
L.t() @ L + opt.lamb * torch.eye(opt.num_label, device=opt.device)) @ L.t() @ B_i
P_t = torch.inverse(
L.t() @ L + opt.lamb * torch.eye(opt.num_label, device=opt.device)) @ L.t() @ B_t
B_i = (L @ P_i + opt.mu * H_i).sign()
B_t = (L @ P_t + opt.mu * H_t).sign()
loss.append(e_loss.item())
print('...epoch: %3d, loss: %3.3f' % (epoch + 1, loss[-1]))
delta_t = time.time() - t1
if opt.vis_env:
vis.plot('loss', loss[-1])
# validate
if opt.valid and (epoch + 1) % opt.valid_freq == 0:
mapi2t, mapt2i = valid(generator, i_query_dataloader, i_db_dataloader, t_query_dataloader, t_db_dataloader,
query_labels, db_labels)
print('...epoch: %3d, valid MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (epoch + 1, mapi2t, mapt2i))
mapi2t_list.append(mapi2t)
mapt2i_list.append(mapt2i)
train_times.append(delta_t)
if 0.5 * (mapi2t + mapt2i) > max_average:
max_mapi2t = mapi2t
max_mapt2i = mapt2i
max_average = 0.5 * (mapi2t + mapt2i)
save_model(generator)
if opt.vis_env:
vis.plot('mapi2t', mapi2t)
vis.plot('mapt2i', mapt2i)
if epoch % 100 == 0:
for params in optimizer.param_groups:
params['lr'] = max(params['lr'] * 0.8, 1e-6)
if not opt.valid:
save_model(generator)
print('...training procedure finish')
if opt.valid:
print(' max MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (max_mapi2t, max_mapt2i))
else:
mapi2t, mapt2i = valid(generator, i_query_dataloader, i_db_dataloader, t_query_dataloader, t_db_dataloader,
query_labels, db_labels)
print(' max MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (mapi2t, mapt2i))
path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit)
with open(os.path.join(path, 'result.pkl'), 'wb') as f:
pickle.dump([train_times, mapi2t_list, mapt2i_list], f)
def train(args):
# gpu init
multi_gpus = False
if len(args.gpus.split(',')) > 1:
multi_gpus = True
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# log init
save_dir = os.path.join(args.save_dir, datetime.now().date().strftime('%Y%m%d'))
if not os.path.exists(save_dir):
#raise NameError('model dir exists!')
os.makedirs(save_dir)
logging = init_log(save_dir)
_print = logging.info
# summary(net.to(config.device), (3,112,112))
#define tranform
transform = transforms.Compose([
transforms.Resize((112, 112)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), # range [0, 255] -> [0.0,1.0]
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) # range [0.0, 1.0] -> [-1.0,1.0]
])
net = EfficientNet.from_name('efficientnet-b0', num_classes=2)
# validation dataset
trainset = ANTI(train_root="/mnt/sda3/data/FASD", file_list = "train.txt", transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size = 2,
shuffle=True, num_workers=8, drop_last=False)
# define optimizers for different layer
criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer_ft = optim.SGD([
{'params': net.parameters(), 'weight_decay': 5e-4},
], lr=0.001, momentum=0.9, nesterov=True)
exp_lr_scheduler = lr_scheduler.MultiStepLR(optimizer_ft, milestones= [6, 10, 30], gamma=0.1)
if multi_gpus:
net = DataParallel(net).to(device)
else:
net = net.to(device)
total_iters = 1
vis = Visualizer(env= "effiction")
for epoch in range(1, args.total_epoch + 1):
exp_lr_scheduler.step()
_print('Train Epoch: {}/{} ...'.format(epoch, args.total_epoch))
net.train()
since = time.time()
for data in trainloader:
img, label = data[0].to(device), data[1].to(device)
optimizer_ft.zero_grad()
raw_logits = net(img)
total_loss = criterion(raw_logits, label)
total_loss.backward()
optimizer_ft.step()
# print train information
if total_iters % 200 == 0:
# current training accuracy
_, predict = torch.max(raw_logits.data, 1)
total = label.size(0)
correct = (np.array(predict) == np.array(label.data)).sum()
time_cur = (time.time() - since) / 100
since = time.time()
vis.plot_curves({'softmax loss': total_loss.item()}, iters=total_iters, title='train loss',
xlabel='iters', ylabel='train loss')
vis.plot_curves({'train accuracy': correct / total}, iters=total_iters, title='train accuracy', xlabel='iters',
ylabel='train accuracy')
print("Iters: {:0>6d}/[{:0>2d}], loss: {:.4f}, train_accuracy: {:.4f}, time: {:.2f} s/iter, learning rate: {}".format(total_iters, epoch, total_loss.item(), correct/total, time_cur, exp_lr_scheduler.get_lr()[0]))
# save model
if total_iters % args.save_freq == 0:
msg = 'Saving checkpoint: {}'.format(total_iters)
_print(msg)
if multi_gpus:
net_state_dict = net.module.state_dict()
else:
net_state_dict = net.state_dict()
if not os.path.exists(save_dir):
os.mkdir(save_dir)
torch.save({
'iters': total_iters,
'net_state_dict': net_state_dict},
os.path.join(save_dir, 'Iter_%06d_net.ckpt' % total_iters))
# test accuracy
if total_iters % args.test_freq == 0 and args.has_test:
# test model on lfw
net.eval()
_print('LFW Ave Accuracy: {:.4f}'.format(np.mean(lfw_accs) * 100))
net.train()
total_iters += 1
print('finishing training')
action_dim,
gamma=0.99,
hidden_dim=hidden_dim,
policy_lr=policy_lr,
baseline_lr=baseline_lr,
d2c=d2c_converter)
# --- run algorithm ---
if load_flag:
reinforce.load_models(model_name=exp_count)
stats = reinforce.train(env=env, episodes=episodes, time_steps=timesteps)
reinforce.save_models(model_name=exp_count)
# --- visualize the results ---
result_folder = Path(__file__).resolve().parent / 'results'
viz = Visualizer(result_path=result_folder)
viz.plot_episode_length(stats, plot_name=f'r_d_episodes_{exp_count}')
viz.plot_reward(stats, plot_name=f'r_d_rewards_{exp_count}')
# --- animation ---
if render_flag:
with contextlib.closing(ContinuousCartPoleEnv()) as env:
for _ in range(2):
s = env.reset()
for _ in range(500):
env.render()
a, _ = reinforce.get_action(s)
s, _, d, _ = env.step(a)
if d:
break
print("Use " + str(device))
#创建一个网络
if (data_set == 'cifar10'):
net = lenet()
#net = googlenet.GoogLeNet()
print("use lenet")
elif (data_set == 'mnist'):
net = lenet_mnist()
print("use mnist lenet ")
#net = lenet()
net = net.to(device)
#可视化环境
env = 'loss and test acc'
vis = Visualizer(env)
if device == 'cuda:0':
net = t.nn.DataParallel(net)
cudnn.benchmark = True
print("CUDA is ready")
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = t.load('./checkpoint/ckpt.t7')
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
