def verify():
model = load_SPRNET()
torch.cuda.set_device(FLAGS["devices_id"][0])
model = nn.DataParallel(model, device_ids=FLAGS["devices_id"]).cuda()
pretrained_weights = torch.load(FLAGS["model"], map_location='cuda:0')
model.load_state_dict(pretrained_weights['state_dict'])
dataset = AFLWTestDataset(root_dir=FLAGS["data_path"],
filelists=FLAGS["data_list"],
augmentation=False,
transform=transforms.Compose([ToTensor()]))
data_loader = DataLoader(dataset,
batch_size=FLAGS["batch_size"],
num_workers=FLAGS["workers"],
shuffle=False,
pin_memory=True,
drop_last=True)
cudnn.benchmark = True
model.eval()
# summary(model.module.fw, (3, 256, 256))
NME3DError_List = []
# NME2DError = getErrorFunction("NME2D")
NME3DError = getErrorFunction("NME3D")
with torch.no_grad():
for i, (img, target) in tqdm(enumerate(data_loader)):
target.requires_grad = False
target = target.cuda(non_blocking=True)
gens = model(img, isTrain=False)[:, :, 1:257, 1:257]
for i in range(gens.shape[0]):
inp = img[i].cpu().numpy().transpose(1, 2, 0)
prd = gens[i].cpu().numpy().transpose(1, 2, 0) * 280.0
grt = target[i].cpu().numpy().transpose(1, 2, 0) * 280.0
# NME2DError_List.append(NME2DError(grt, prd))
NME3DError_List.append(NME3DError(grt, prd))
show_img = (255.0 * (inp - np.min(inp)) / np.ptp(inp)).astype(
np.uint8)
# P, pose, (s, R, t) = estimate_pose(get_vertices(show_img_uv))
# if pose[0] >= - 30.0 / 180.0 * pi and pose[0] <= 30.0 / 180.0 * pi:
# NME3D_30_Error_List.append(NME3DError(grt, prd))
# elif (pose[0] >= - 60.0 / 180.0 * pi and pose[0] < - 30.0 / 180.0 * pi) or (pose[0] > 30.0 / 180.0 * pi and pose[0] <= 60.0 / 180.0 * pi):
# NME3D_60_Error_List.append(NME3DError(grt, prd))
# else:
# NME3D_90_Error_List.append(NME3DError(grt, prd))
tex = cv2.remap(show_img,
prd[:, :, :2].astype(np.float32),
None,
interpolation=cv2.INTER_CUBIC,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
mesh_info = UVmap2Mesh(uv_position_map=prd, uv_texture_map=tex)
showMesh(mesh_info, tex, show_img)
# show_uv_mesh(show_img, prd)
# show_kpt_result(show_img, prd, grt)
# print(np.mean(NME2DError_List))
print(np.mean(NME3DError_List))
return
def gen():
model = load_SPRNET()
torch.cuda.set_device(FLAGS["devices_id"][0])
model = nn.DataParallel(model, device_ids=FLAGS["devices_id"]).cuda()
pretrained_weights = torch.load(FLAGS["model"], map_location='cuda:0')
model.load_state_dict(pretrained_weights['state_dict'])
dataset = USRTestDataset(
root_dir = FLAGS["data_path"],
transform = transforms.Compose([ToTensor()])
)
data_loader = DataLoader(
dataset,
batch_size = FLAGS["batch_size"],
num_workers = FLAGS["workers"],
shuffle = False,
pin_memory = True,
drop_last = False
)
cudnn.benchmark = True
model.eval()
with torch.no_grad():
for i, (img) in enumerate(data_loader):
gens = model(img, isTrain=False)[:, :, 1:257, 1:257]
prd = gens[0].cpu().numpy().transpose(1, 2, 0) * 280.0
np.save('{}/{}'.format("/home/viet/Projects/Pycharm/SPRNet/result/usr", "image1.jpg".replace('jpg', 'npy')), prd)
def train(self, triplet=True):
# generate dataset for PyTorch
from data.dataset import TruncatedInputFromMFB, ToTensor, SpeakerDataset
from torchvision import transforms
from torch.utils.data import DataLoader
import torch
transform = transforms.Compose([TruncatedInputFromMFB(), ToTensor()])
if Config.MODEL_TYPE == 'cnn3d':
from data.dataset3d import SpeakerDataset3D
initial_dataset = SpeakerDataset(transform=transform)
train_dataset = SpeakerDataset3D(initial_dataset)
else:
train_dataset = SpeakerDataset(transform=transform)
# instantiate a model
if Config.MODEL_TYPE == 'rescnn':
from models.rescnn import ResNet
model_ = ResNet(layers=Config.RESCNN_LAYERS,
num_classes=Config.NUM_CLASSES)
elif Config.MODEL_TYPE == 'gru':
from models.gru import GRU
model_ = GRU(layers=Config.GRU_LAYERS,
num_classes=Config.NUM_CLASSES)
elif Config.MODEL_TYPE == 'cnn3d':
from models.cnn3d import CNN3D
model_ = CNN3D(num_classes=Config.NUM_CLASSES)
from utils.train import Trainer
model_ = model_.cuda()
epoch = Config.SOFTMAX_TRAINING_EPOCH
for i in range(epoch):
optimizer = torch.optim.Adam(model_.parameters())
train_loader = DataLoader(train_dataset,
batch_size=Config.PRETRAIN_BATCH_SIZE,
shuffle=True)
Trainer.train(train_loader, model_, optimizer, i)
if triplet:
from copy import deepcopy
model_tri = deepcopy(model_)
model_tri = model_tri.cuda()
epoch_ = Config.TRIPLET_TRAINING_EPOCH
for i in range(epoch_):
optimizer_ = torch.optim.SGD(model_tri.parameters(),
lr=Config.TRIPLET_LR -
i * Config.TRIPLET_LR_DECAY,
momentum=Config.TRIPLET_MOMENTUM)
train_loader = DataLoader(
train_dataset,
batch_size=Config.FINETUNE_BATCH_SIZE,
shuffle=True)
Trainer.train_tri(train_loader,
model_tri,
optimizer_,
i,
semi_hard=True,
triplet_margin=Config.TRIPLET_MARGIN)
def vis_images(sample, net, vis, title):
"""
生成一组visdom中显示的images
:param sample:
:return: tuple(np.ndarray, np.ndarray), 两张图片前一张是有标记的， 后一张是原图
"""
tsf = transforms.Compose([ZeroMean(), ToTensor()])
img = sample['image']
vis.image(img[np.newaxis, ...], win='ori_img' + title, opts={'title': title})
img_input = tsf(sample)['image']
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
net.eval()
pred_heatmaps = net.forward(img_input.unsqueeze(0).to(opt.device))
pred_heatmaps_numpy = pred_heatmaps.cpu().detach().numpy()
pred_heatmaps_numpy = pred_heatmaps_numpy.squeeze()
draw_points(img, pred_heatmaps_numpy)
vis.image(img.transpose(2, 0, 1), win='dotted_img' + title, opts={'title': title})
def visualize_prediction(dataset=UnityEyeDataset(data_dir=opt.val_data_dir), contrain_eyeball_center=False):
net = ELGNetwork(output_shape=(17, 96, 160))
checkpoint = torch.load('weights/ELG_best_so_far.pth')
net.load_state_dict(checkpoint['net_state_dict'])
net.cuda()
net.eval()
tsf = transforms.Compose([ZeroMean(), ToTensor()])
for sample in dataset:
img = sample["image"]
img_input = tsf(sample)["image"]
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
pred_heatmaps = net.forward(img_input.unsqueeze(0).to(opt.device))
pred_heatmaps_numpy = pred_heatmaps.cpu().detach().numpy()
pred_heatmaps_numpy = pred_heatmaps_numpy.squeeze()
draw_points(img, pred_heatmaps_numpy, contain_eyeball_center=contrain_eyeball_center)
cv2.imshow('img', img)
if cv2.waitKey(0) & 0xff == ord('q'):
break
cv2.destroyAllWindows()
def train():
vis = visdom.Visdom(env='ElgNet')
tsf = transforms.Compose([ZeroMean(), ToTensor()])
train_dataset = UnityEyeDataset(data_dir=opt.dev_data_dir, eye_size=(160, 96), transform=tsf)
train_dataset_for_vis = UnityEyeDataset(data_dir=opt.dev_data_dir, eye_size=(160, 96))
sample_num = len(train_dataset)
iter_per_epoch = math.ceil(sample_num / opt.batch_size)
train_dataloader = DataLoader(dataset=train_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=4)
val_dataset = UnityEyeDataset(data_dir=opt.dev_data_dir, eye_size=(160, 96))
net = ELGNetwork(HGNet_num=3, input_shape=(1, 96, 160), output_shape=(18, 96, 160), feature_channels=64)
if opt.checkpoint_path:
checkpoint = torch.load(opt.checkpoint_path)
net.load_state_dict(checkpoint['net_state_dict'])
net.float().to(opt.device)
net.train()
criterion = MSELoss()
optimizer = optim.Adam(net.parameters(), lr=opt.lr, weight_decay=opt.weight_decay)
best_epoch_loss = 10000
v_max, v_min = 0.0, 0.0
for epoch in range(opt.epochs):
epoch_loss = 0.0
for itr, batch in enumerate(train_dataloader):
inputs = batch["image"].to(opt.device)
heatmaps_targets = batch["heatmaps"].to(opt.device)
optimizer.zero_grad()
net.train()
heatmaps_pred = net.forward(inputs)
# 多loss 回传, 效果不好放弃使用了
loss = criterion(heatmaps_targets, heatmaps_pred)
loss.backward()
optimizer.step()
epoch_loss = (epoch_loss * itr + loss.item()) / (itr + 1)
# 统计最大值和最小值
v_max = torch.max(heatmaps_pred)
v_min = torch.min(heatmaps_pred)
print('[Epoch {} / {}, iter {} / {}] train_loss: {:.6f} max: {:.4f} min: {:.4f}'.format(
epoch, opt.epochs, itr * opt.batch_size, sample_num, loss, v_max, v_min
))
# 可视化
vis.line(Y=np.array([loss.item()]), X=np.array([(epoch * iter_per_epoch) + (itr + 1)]), win='Train_loss',
update='append' if (epoch + 1) * (itr + 1) > 0 else None, opts=dict(title='train_loss'))
vis.line(Y=np.array([v_max.item()]), X=np.array([(epoch * iter_per_epoch) + (itr + 1)]), win='v_max',
update='append' if (epoch + 1) * (itr + 1) != 0 else None, opts=dict(title='v_max'))
vis.line(Y=np.array([v_min.item()]), X=np.array([(epoch * iter_per_epoch) + (itr + 1)]), win="v_min",
update='append' if (epoch + 1) * (itr + 1) != 0 else None, opts=dict(title='v_min'))
# 图片显示
if itr % opt.plot_every_iter == 0:
random_val_idx = np.random.randint(0, len(val_dataset))
random_train_idx = np.random.randint(0, len(train_dataset_for_vis))
vis_val_sample = val_dataset[random_val_idx]
vis_train_sample = train_dataset_for_vis[random_train_idx]
vis_images(vis_val_sample, net, vis, title='val_sample')
vis_images(vis_train_sample, net, vis, title='train_sample')
vis.line(Y=np.array([epoch_loss]), X=np.array([epoch]),
win='epoch_loss', update='append' if epoch > 0 else None)
print('epoch_loss: {:.5f} old_best_epoch_loss: {:.5f}'.format(epoch_loss, best_epoch_loss))
if epoch_loss < best_epoch_loss:
print('epoch loss < best_epoch_loss, save these weights.')
best_epoch_loss = epoch_loss
torch.save({
'epoch': epoch,
'epoch_loss': epoch_loss,
'v_max': v_max,
'v_min': v_min,
'net_state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, opt.weight_save_dir+'ELG_epoch{}.pth'.format(epoch))
def main(root_dir):
### Step1: Define the model structure
model = load_SPRNET()
torch.cuda.set_device(FLAGS["devices_id"][0])
model = nn.DataParallel(model, device_ids=FLAGS["devices_id"]).cuda()
if FLAGS["resume"]:
pretrained_weights = torch.load(
os.path.join(root_dir, "train_log",
"_checkpoint_epoch_19.pth.tar"))
model.load_state_dict(pretrained_weights['state_dict'])
FLAGS["start_epoch"] = int(pretrained_weights['epoch']) + 1
### Step2: Loss and optimization method
criterion = Loss()
optimizer = torch.optim.SGD(model.parameters(),
lr=FLAGS["base_lr"],
momentum=FLAGS["momentum"],
weight_decay=FLAGS["weight_decay"],
nesterov=True)
### Step3: Load 300WLP Augmentation Dataset
data_dir = os.path.join(root_dir, "data")
train_dataset = SiaTrainDataset(root_dir=data_dir,
filelists=os.path.join(
root_dir, "train.configs",
"label_train_aug_120x120.list.train"),
augmentation=True,
transform=transforms.Compose([ToTensor()]))
val_dataset = SiaValDataset(root_dir=data_dir,
filelists=os.path.join(
root_dir, "train.configs",
"label_train_aug_120x120.list.val"),
augmentation=True,
transform=transforms.Compose([ToTensor()]))
train_loader = DataLoader(train_dataset,
batch_size=FLAGS["batch_size"],
num_workers=FLAGS["workers"],
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(val_dataset,
batch_size=FLAGS["batch_size"],
num_workers=FLAGS["workers"],
shuffle=True,
pin_memory=True,
drop_last=True)
cudnn.benchmark = True
### Step4: Setup TensorBoard
train_writer = SummaryWriter("runs/train")
val_writer = SummaryWriter("runs/val")
x = torch.rand(1, 3, 256, 256).cuda()
y = torch.rand(1, 3, 256, 256).cuda()
train_writer.add_graph(model.module.fw, x)
### Step5: Show CUDA Info
os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS["devices_id"][0])
print(torch.cuda.is_available(), torch.cuda.device_count(),
torch.cuda.current_device(), torch.cuda.get_device_name(0))
for epoch in range(FLAGS["start_epoch"], FLAGS["target_epoch"]):
print("[Epoch]: %d" % epoch)
## Adjust learning rate
adjust_lr(optimizer, FLAGS["base_lr"], epoch, FLAGS["target_epoch"],
FLAGS["milestones"])
## Train
train(train_loader, model, criterion, optimizer, epoch, train_writer)
## Validate
validate(val_loader, model, criterion, optimizer, epoch, val_writer)
## Step5: Save model parameters
if (epoch) % 5 == 0:
filename = f'{snapshot}_checkpoint_epoch_{epoch}.pth.tar'
save_checkpoint({
'epoch': epoch,
'state_dict': model.state_dict()
}, filename)
train_writer.close()
val_writer.close()