def validation(model, dataloaders, criterion, phase, use_gpu, device):
model.eval()
pred = np.array([])
true = np.array([])
losses = np.array([])
with torch.no_grad():
for data in tqdm(dataloaders[phase]):
inputs, labels = data
if use_gpu:
inputs = Variable(inputs).to(device)
labels = Variable(labels).to(device)
else:
inputs, labels = Variable(inputs), Variable(labels)
outputs = model(inputs)
loss = criterion(
outputs.type(torch.cuda.FloatTensor)[:, 1],
labels.type(torch.cuda.FloatTensor))
#loss = nn.BCEWithLogitsLoss(reduction='sum',
# weight=torch.ones(outputs.shape[0]).to(device) / 5)(outputs.type(torch.cuda.FloatTensor)[:, 1],
# labels.type(torch.cuda.FloatTensor))
true = np.append(true, to_numpy(labels, use_gpu))
pred = np.append(pred, sigmoid(to_numpy(outputs[:, 1], use_gpu)))
losses = np.append(losses, to_numpy(loss, use_gpu))
del inputs, labels, loss
model.train()
return true, pred, np.sum(losses)
def main():
utils.fix_randomness()
opt = options.set_end2end_optim_options()
assert opt.iou_space == 'part_and_whole'
test_dataset = aligned_dataset.AlignedDatasetFactory.get_aligned_dataset(
opt, 'test')
test_loader = DataLoader(
test_dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=0,
)
e2e = end_2_end_optimization.End2EndOptimFactory.get_end_2_end_optimization_model(
opt)
iou = metrics.IOU(opt)
orig_iou_list = []
optim_iou_list = []
original_homography_list = []
optim_homography_list = []
gt_homography_list = []
t0 = time.time()
for i, data_batch in enumerate(test_loader):
frame, _, gt_homography = data_batch
orig_homography, optim_homography = e2e.optim(frame,
test_dataset.template)
orig_iou = iou(orig_homography, gt_homography)
optim_iou = iou(optim_homography, gt_homography)
orig_iou_list.append(orig_iou)
optim_iou_list.append(optim_iou)
original_homography_list.append(utils.to_numpy(orig_homography.data))
optim_homography_list.append(utils.to_numpy(optim_homography.data))
gt_homography_list.append(utils.to_numpy(gt_homography.data))
t1 = time.time()
orig_iou_list = np.array(orig_iou_list)
orig_iou_part_list = np.concatenate(orig_iou_list[:, 0])
orig_iou_whole_list = np.concatenate(orig_iou_list[:, 1])
print('----- Summary -----')
print('original IOU part mean:', orig_iou_part_list.mean())
print('original IOU part median:', np.median(orig_iou_part_list))
print('original IOU whole mean:', orig_iou_whole_list.mean())
print('original IOU whole median:', np.median(orig_iou_whole_list))
optim_iou_list = np.array(optim_iou_list)
optim_iou_part_list = np.concatenate(optim_iou_list[:, 0])
optim_iou_whole_list = np.concatenate(optim_iou_list[:, 1])
print('optimized IOU part mean:', optim_iou_part_list.mean())
print('optimized IOU part median:', np.median(optim_iou_part_list))
print('optimized IOU whole mean:', optim_iou_whole_list.mean())
print('optimized IOU whole median:', np.median(optim_iou_whole_list))
print('----- -----')
print('spent {0} seconds for {1} images'.format(
(t1 - t0), (optim_iou_whole_list.shape[0])))
print('{0} seconds per single image'.format(
(t1 - t0) / (optim_iou_whole_list.shape[0])))
print('----- End -----')
def render_results(y_2d, y_2d_):
y_2d = to_numpy(y_2d)
y_2d_ = to_numpy(y_2d_)
point_pairs = np.column_stack((y_2d, y_2d_))
im_to_show = point_pairs[1::4]
res = np.concatenate(im_to_show, axis=1)
res = np.expand_dims(res, axis=0)
return res
def _eval_generator_and_save_images(self, epoch_idx):
results = self._eval_generator()
save_images(
results,
to_numpy(self.fixed_inp, False),
to_numpy(self.fixed_target, False),
path=os.path.join(self.test_images_path,
"testimage_{:02d}.png".format(epoch_idx + 1)),
)
def get_model_part_iou_rasterization(self, output, target):
if output.shape == (3, 3):
output = output[None]
if target.shape == (3, 3):
target = target[None]
assert output.shape == target.shape, 'output shape does not match target shape'
batch_size = output.shape[0]
fake_frame = torch.ones(
[batch_size, 1, self.frame_height, self.frame_width],
device=output.device)
fake_template = torch.ones(
[batch_size, 1, self.template_height, self.template_width],
device=output.device)
output_mask = warp.warp_image(fake_frame,
output.inverse(),
out_shape=fake_template.shape[-2:])
target_mask = warp.warp_image(fake_frame,
target.inverse(),
out_shape=fake_template.shape[-2:])
output_mask[output_mask > 0] = 1
target_mask[target_mask > 0] = 1
intersection_mask = output_mask * target_mask
output = output_mask.sum(dim=[1, 2, 3])
target = target_mask.sum(dim=[1, 2, 3])
intersection = intersection_mask.sum(dim=[1, 2, 3])
union = output + target - intersection
iou = intersection / (union + self.epsilon)
return utils.to_numpy(iou)
def main(args):
test_x, test_y = load_image(args.image_path)
test_inp = to_tensor(test_x.astype(np.float32))
test_target = to_tensor(test_y.astype(np.float32))
generator = Generator().to("cuda")
start_t = time.time()
pretrain_model = flow.load(args.model_path)
generator.load_state_dict(pretrain_model)
end_t = time.time()
print("load params time : {}".format(end_t - start_t))
start_t = time.time()
generator.eval()
with flow.no_grad():
gout = to_numpy(generator(test_inp), False)
end_t = time.time()
print("infer time : {}".format(end_t - start_t))
# save images
save_images(
gout,
test_inp.numpy(),
test_target.numpy(),
path=os.path.join("./testimage.png"),
plot_size=1,
)
def train_discriminator(self, input, target, label0, label1):
g_out = self.netG(input)
# Fake; stop backprop to the generator by detaching fake_B
fake_AB = flow.cat([input, g_out.detach()], 1)
pred_fake = self.netD(fake_AB)
d_fake_loss = self.criterionGAN(pred_fake, label0)
# Real
real_AB = flow.cat([input, target], 1)
pred_real = self.netD(real_AB)
d_real_loss = self.criterionGAN(pred_real, label1)
# combine loss and calculate gradients
d_loss = (d_fake_loss + d_real_loss) * 0.5
d_loss.backward()
self.optimizerD.step()
self.optimizerD.zero_grad()
return to_numpy(d_fake_loss), to_numpy(d_real_loss), to_numpy(d_loss)
def get_frame_space_scaling_homography():
src_4pts = utils.to_torch(utils.FULL_CANON4PTS_NP())
dest_4pts = utils.to_torch(
np.array([[0, 0], [0, self.frame_h],
[self.frame_w, self.frame_h], [self.frame_w, 0]],
dtype=np.float32))
scaling_transformation = utils.get_perspective_transform(
src_4pts[None], dest_4pts[None])
scaling_transformation = utils.to_numpy(scaling_transformation[0])
return scaling_transformation
def train_generator(self, input, target, label1):
g_out = self.netG(input)
# First, G(A) should fake the discriminator
fake_AB = flow.cat([input, g_out], 1)
pred_fake = self.netD(fake_AB)
gan_loss = self.criterionGAN(pred_fake, label1)
# Second, G(A) = B
l1_loss = self.criterionL1(g_out, target)
# combine loss and calculate gradients
g_loss = gan_loss + self.LAMBDA * l1_loss
g_loss.backward()
self.optimizerG.step()
self.optimizerG.zero_grad()
return (
to_numpy(gan_loss),
to_numpy(self.LAMBDA * l1_loss),
to_numpy(g_loss),
to_numpy(g_out, False),
)
def squash_image(image):
'''squash image to [0, 1]
'''
min_val = image.min()
max_val = image.max()
if isinstance(image, np.ndarray):
image = np.interp(image, (min_val, max_val), (0, 1))
return image
elif isinstance(image, torch.Tensor):
image_np = utils.to_numpy(image)
image_np = np.interp(image_np, (min_val, max_val), (0, 1))
image_torch = utils.to_torch(image_np)
return image_torch
else:
raise ValueError('unsupported data type: {0}'.format(type(image)))
def evaluate(self):
for subset in ['val']:
for i, _ in enumerate(tqdm(self.loaders["PIC"]["val"], 0)):
for net_id in self.models.keys():
data_type = net_id[-3:]
sample = self.get_samples(data_type, subset, i)
mse_maps, uv_maps_ = self.get_mse_loss(
self.models[net_id], sample)
l1, mse = self.get_uv_errors(uv_maps_, sample)
self.evaluation_results[subset][net_id]['mse_maps'].append(
to_numpy(mse_maps))
self.evaluation_results[subset][net_id]['l1'].append(
to_numpy(l1))
self.evaluation_results[subset][net_id]['mse'].append(
to_numpy(mse))
saves_path = '/'.join(
[self.save_to, f'evaluation_{subset}.pickle'])
with open(saves_path, 'wb') as handle:
pickle.dump(self.evaluation_results[subset],
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def evaluate(self):
for subset in self.subsets:
for i, _ in enumerate(tqdm(self.loaders["PIC"]["val"], 0)):
for net_id in self.models.keys():
is_vid = net_id.endswith('VID')
data_type = net_id[-3:]
sample = self.get_samples(data_type, subset)
mse, l_1, renorm_loss = self.get_losses(
self.models[net_id], sample, is_vid)
self.evaluation_results[subset][net_id]['mse'].append(
to_numpy(mse))
self.evaluation_results[subset][net_id]['l_1'].append(
to_numpy(l_1))
self.evaluation_results[subset][net_id][
'renorm_loss'].append(to_numpy(renorm_loss))
saves_path = '/'.join(
[self.save_to, f'evaluation_{subset}.pickle'])
with open(saves_path, 'wb') as handle:
pickle.dump(self.evaluation_results[subset],
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def _eval_agent(self):
success_rate = 0
for _ in range(self.config.test_rollouts):
observation = self.env.reset()
obs = observation['observation']
goal = observation['desired_goal']
obs, goal = self._preproc_inputs(obs, goal)
for _ in range(self.env_params['max_timesteps']):
if self.config.render:
self.env.render()
with torch.no_grad():
action = self.actor(obs, goal)
new_obs, _, _, info = self.env.step(to_numpy(action[0]))
obs, goal = self._preproc_inputs(new_obs['observation'],
new_obs['desired_goal'])
success_rate += info['is_success']
return success_rate / self.config.test_rollouts
def get_model_whole_iou_rasterization(self, output, target):
'''calculate the model whole iou
'''
if output.shape == (3, 3):
output = output[None]
if target.shape == (3, 3):
target = target[None]
assert output.shape == target.shape, 'output shape does not match target shape'
ZOOM_OUT_SCALE = 4
batch_size = output.shape[0]
fake_template = torch.ones([
batch_size, 1, self.template_height * ZOOM_OUT_SCALE,
self.template_width * ZOOM_OUT_SCALE
],
device=output.device)
scaling_mat = torch.eye(3,
device=output.device).repeat(batch_size, 1, 1)
scaling_mat[:, 0, 0] = scaling_mat[:, 1, 1] = ZOOM_OUT_SCALE
target_mask = warp.warp_image(fake_template,
scaling_mat,
out_shape=fake_template.shape[-2:])
mapping_mat = torch.matmul(output, scaling_mat)
mapping_mat = torch.matmul(target.inverse(), mapping_mat)
output_mask = warp.warp_image(fake_template,
mapping_mat,
out_shape=fake_template.shape[-2:])
output_mask = (output_mask >= 0.5).float()
target_mask = (target_mask >= 0.5).float()
intersection_mask = output_mask * target_mask
output = output_mask.sum(dim=[1, 2, 3])
target = target_mask.sum(dim=[1, 2, 3])
intersection = intersection_mask.sum(dim=[1, 2, 3])
union = output + target - intersection
iou = intersection / (union + self.epsilon)
return utils.to_numpy(iou)
def log_validation(self,
losses_dict,
model,
target,
prediction,
iteration,
target_alignments=None):
self._log_scalar_dict(losses_dict, iteration, "val")
# plot distribution of parameters
for tag, value in model.named_parameters():
tag = tag.replace('.', '/')
self.add_histogram(tag, value.data.cpu().numpy(), iteration)
# plot alignment target and predicted, mel target and predicted, gate target and predicted
log_image = partial(self.add_image,
global_step=iteration,
dataformats="HWC")
idx = random.randint(0, prediction.alignments.size(0) - 1)
if target_alignments is not None:
log_image(
"alignment/target",
plot_alignment_to_numpy(to_numpy(target_alignments[idx]).T))
log_image(
"alignment/predicted",
plot_alignment_to_numpy(to_numpy(prediction.alignments[idx].T)))
log_image("mel/target",
plot_spectrogram_to_numpy(to_numpy(target.mels[idx])))
log_image("mel/predicted",
plot_spectrogram_to_numpy(to_numpy(prediction.mels[idx])))
log_image(
"gate",
plot_gate_outputs_to_numpy(
to_numpy(target.gate[idx]),
to_numpy(torch.sigmoid(prediction.gate[idx]))))
def _eval_generator(self):
self.netG.eval()
with flow.no_grad():
g_out = self.netG(self.fixed_inp)
return to_numpy(g_out, False)
def train_model(model,
criterion,
optimizer,
scheduler,
dataloaders,
dataset_sizes,
model_path,
device,
fold_name,
use_gpu=True,
num_epochs=25,
plot_res=True,
predict_test=True,
save_val=True,
best='loss'):
since = time.time()
torch.save(model.state_dict(), os.path.join(model_path, '_temp'))
best_f1 = 0.0
best_loss = np.inf
best_epoch = 0
train_metrics = []
val_metrics = []
print('=' * 10)
print('VAL data {}'.format(fold_name))
print('=' * 10)
for epoch in range(num_epochs):
model.train(True)
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
running_loss = 0.0
pred = np.array([])
true = np.array([])
phase = 'train'
for data in tqdm(dataloaders[phase]):
optimizer.zero_grad()
inputs, labels = data
if use_gpu:
inputs = Variable(inputs).to(device)
labels = Variable(labels).to(device)
else:
inputs, labels = Variable(inputs), Variable(labels)
outputs = model(inputs)
loss = criterion(
outputs.type(torch.cuda.FloatTensor)[:, 1],
labels.type(torch.cuda.FloatTensor))
#loss = nn.BCEWithLogitsLoss(reduction='sum',
# weight=torch.ones(outputs.shape[0]).to(device) / 5)(outputs.type(torch.cuda.FloatTensor)[:, 1],
# labels.type(torch.cuda.FloatTensor))
loss.backward()
optimizer.step()
running_loss += loss.item()
true = np.append(true, to_numpy(labels, use_gpu))
pred = np.append(pred, sigmoid(to_numpy(outputs[:, 1], use_gpu)))
epoch_loss = running_loss / dataset_sizes[phase]
pr_rec_auc, roc_auc, f1_05, f1_max, f1_auc, metr = calc_metrics(
true, pred)
val_true, val_pred, val_loss = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='val')
pr_rec_auc_val, roc_auc_val, f1_05_val, f1_max_val, f1_auc_val, metr_val = calc_metrics(
val_true, val_pred)
if scheduler is not None:
scheduler.step((val_loss) / dataset_sizes['val'])
clear()
clear_output()
print('=' * 10)
print('VAL data {}'.format(fold_name))
print('=' * 10)
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
print_results(phase=phase,
loss=epoch_loss,
roc_auc=roc_auc,
pr_rec_auc=pr_rec_auc,
f1_max=f1_max,
f1_05=f1_05,
f1_auc=f1_auc)
print('\n')
print_results(phase='val',
loss=(val_loss) / dataset_sizes['val'],
roc_auc=roc_auc_val,
pr_rec_auc=pr_rec_auc_val,
f1_max=f1_max_val,
f1_05=f1_05_val,
f1_auc=f1_auc_val)
train_metrics.append(metr + [epoch_loss])
val_metrics.append(metr_val + [(val_loss) / dataset_sizes['val']])
if f1_05_val > best_f1:
best_f1 = f1_05_val
if best == 'metric':
best_epoch = epoch
torch.save(model.state_dict(),
os.path.join(model_path, '_temp'))
if (val_loss) / dataset_sizes['val'] < best_loss:
best_loss = (val_loss) / dataset_sizes['val']
if best == 'loss':
best_epoch = epoch
torch.save(model.state_dict(),
os.path.join(model_path, '_temp'))
time_elapsed = time.time() - since
print('Elapsed {:.0f}m {:.0f}s\n'.format(time_elapsed // 60,
time_elapsed % 60))
np.save(os.path.join(model_path, 'val_metrics_' + fold_name),
val_metrics)
np.save(os.path.join(model_path, 'train_metrics_' + fold_name),
train_metrics)
print('Current best: {:4f}, epoch {}'.format(best_f1, best_epoch))
if plot_res:
vizualize(model_path,
fold_name,
show=False,
save=True,
save_format='.png')
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val f1_05: {:4f}'.format(best_f1))
print('Best val loss: {:4f}'.format(best_loss))
if best is not None:
model.load_state_dict(torch.load(os.path.join(model_path, '_temp')))
vizualize(model_path, fold_name, show=False, save=True, save_format='.pdf')
if predict_test:
results = pd.DataFrame()
test_true, test_pred, _ = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='test')
results['pred'] = test_pred
results['pseudo_true'] = test_true
results['file'] = [
sample[0] for sample in dataloaders['test'].dataset.samples
]
results.to_csv(os.path.join(model_path,
'test_results_' + fold_name + '.csv'),
index=False)
if save_val:
results = pd.DataFrame()
val_true, val_pred, _ = validation(model,
dataloaders,
criterion,
device=device,
use_gpu=use_gpu,
phase='val')
results['true'] = val_true
results['pred'] = val_pred
results['file'] = [
sample[0] for sample in dataloaders['val'].dataset.samples
]
results.to_csv(os.path.join(model_path,
'val_results_' + fold_name + '.csv'),
index=False)
del inputs, labels, loss
return model, best_f1
import matplotlib.pyplot as plt
from configs.config_m0 import CONFIG_M0
from datasets.frei_dataset_u0 import FreiHandDataset
from transforms.to_tensor_u0 import ToTensor
data_path = CONFIG_U0['dataset_path']
build_mode = CONFIG_U0['build_mode']
frames_generator = FramesGenerator(data_path, build_mode)
oredered_sequence = frames_generator.order_frames('xyz', 2345, 340)
build_id = CONFIG_M0['build_id']
dataset = FreiHandDataset(CONFIG_M0['dataset_path'],
CONFIG_M0['data_version'],
transform=ToTensor())
xyz_array = frames_generator.get_xyz_array()
K_array = frames_generator.get_K_array()
uv_array = frames_generator.get_uv_array(xyz_array, K_array)
for idx in oredered_sequence:
# uv_21x2 = np.array(uv_list).reshape(32560, 21, 2)
img_tensor = dataset[idx]['img']
img_np = to_numpy(img_tensor).transpose((1, 2, 0))
im_rgb = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)
cv2.imshow('df', im_rgb)
cv2.waitKey(-1)
dataset = FreiHandDataset(CONFIG_U0['dataset_path'],
CONFIG_U0['data_version'],
transform=ToTensor())
dataloader = DataLoader(dataset)
model = ModelU0()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
checkpoint = torch.load(f'results/{build_id}.pt',
map_location=torch.device(device))
model.load_state_dict(checkpoint['model_state_dict'])
model.to(device)
model.eval()
sample = next(iter(dataloader))
img = sample['img']
y_2d = to_numpy(sample['y_2d'])
uv = to_numpy(sample['uv'])
y_2d_ = model(img)
y_2d_ = to_numpy(y_2d_)
img = to_numpy(img)
img = img[0].transpose((1, 2, 0))
print(img.shape)
fig, ax = plt.subplots()
ax.imshow(img)
plot_hand(ax, uv[0], order='uv')
plt.show()
map_location=torch.device(device))
model.load_state_dict(checkpoint['model_state_dict'])
g_step = checkpoint['g_step'] + 1
running_loss = checkpoint['running_loss']
print(
f'Model "{build_id}" loaded. g_step: {g_step}; running_loss: {running_loss}'
)
model.to(device)
model.eval()
# x = hand_joints[0].reshape([1, -1])
xyz_ = model(x)
xyz_ = to_numpy(xyz_).reshape([21, 3])
# # print(xyz_)
#
# y = sample['xyz']
K = sample['K']
img = sample['img']
# # print("yb", y)
# # y= y.reshape([CONFIG_M0['batch_size'], -1])
# # print("ya", y.reshape([21, 3]))
#
# y = to_numpy(y)
# # print(y.shape)
# MATRIX WAY
# x = [[1.0692e-03, -2.9279e-04, 1.8849e-04, 3.7177e-01],
