def get_pascal_synthetic(batch_size, train_all, use_augmentation, voc_train):
dataset_real = Pascal3D.Pascal3D(dataset_dir, train_all=train_all, use_warp=True, voc_train=voc_train)
train_real = dataset_real.get_train(use_augmentation)
real_sampler = torch.utils.data.sampler.RandomSampler(train_real, replacement=False)
dataset_rendered = Pascal3D_render.Pascal3DRendered(dataset_dir)
rendered_size = int(0.2*len(dataset_rendered)) # use 20% of synthetic data for training per epoch
rendered_sampler = dataloader_utils.RandomSubsetSampler(dataset_rendered, rendered_size)
dataset_train, sampler_train = dataloader_utils.get_concatenated_dataset([(train_real, real_sampler), (dataset_rendered, rendered_sampler)])
dataloader_train = torch.utils.data.DataLoader(
dataset_train,
sampler=sampler_train,
batch_size=batch_size,
num_workers=8,
worker_init_fn=lambda _: np.random.seed(torch.utils.data.get_worker_info().seed % (2**32)),
pin_memory=True,
drop_last=True)
dataloader_eval = torch.utils.data.DataLoader(
dataset_real.get_eval(),
batch_size=batch_size,
shuffle=False,
num_workers=8,
worker_init_fn=lambda _: np.random.seed(torch.utils.data.get_worker_info().seed % (2**32)),
pin_memory=True,
drop_last=False)
return dataloader_train, dataloader_eval
def main():
real = Pascal3D.Pascal3D(use_warp=False)
ds_path = 'datasets' # TODO change to where datasets are stored
syn = Pascal3D_render.Pascal3DRendered(ds_path, 224)
real_sampler = torch.utils.data.sampler.RandomSampler(real.get_train(),
replacement=False)
syn_size = int(0.2 * len(syn))
syn_sampler = dataloader_utils.RandomSubsetSampler(syn, syn_size)
dataset, sampler = dataloader_utils.get_concatenated_dataset([
(real.get_train(), real_sampler), (syn, syn_sampler)
])
# for idx in sampler:
# use index 121 for visualization of warp vs non-warp
ds = real.get_train()
for idx in range(5502, len(ds)):
print(idx)
sample = ds[idx]
image, extrinsic, class_idx, hard, intrinsic, cad_idx = sample
print(intrinsic)
print(extrinsic[:3, :3])
print(extrinsic)
image = image.transpose(1, 2, 0)
points = np.array([[0.0, 0, 0, 1], [0.1, 0, 0, 1], [0, 0.1, 0, 1],
[0, 0, 0.1, 1]]).transpose()
points = np.matmul(extrinsic, points)
points = points / points[2]
points = points[:3, :]
points = np.matmul(intrinsic, points)
plt.imshow(image)
for p, c in zip(points[:, 1:].transpose(), ['r', 'g', 'b']):
x = [points[0, 0], p[0]]
y = [points[1, 0], p[1]]
plt.plot(x, y, c, linewidth=5)
plt.show()
# render cads
for image, extrinsic, class_idx, hard, intrinsic, cad_idx in tqdm.tqdm(
x.get_train(True)):
points = np.matmul(ext, points)
points = points / points[2]
points = points[:3, :]
print(points)
points = np.matmul(intr, points)
plt.imshow(im)
print(points)
for p, c in zip(points[:, 1:].transpose(), ['r', 'g', 'b']):
x = [points[0, 0], p[0]]
y = [points[1, 0], p[1]]
plt.plot(x, y, c)
plt.show()
nodes, _ = x.get_cad(class_idx, cad_idx)
nodes_homo = np.ones((4, len(nodes)))
nodes_homo[:3, :] = nodes.transpose()
model = np.matmul(extrinsic, nodes_homo)
model = model[:3, :]
model /= model[2].reshape(1, -1)
mod_proj = np.matmul(intrinsic, model)
plt.imshow(image.transpose(1, 2, 0))
plt.plot(mod_proj[0], mod_proj[1])
plt.show()
def visualize_random_errors():
net_path = 'logs/pascal/pascal_new_norm_full'
image_dir_out = 'plots/random_errors'
dataset_location = 'datasets' # TODO update with dataset path
device = torch.device('cpu')
dataset = Pascal3D.Pascal3D(train_all=True)
dataset_vis = dataset.get_eval()
base = resnet101()
model = ResnetHead(base, 13, 32, 512, 9)
loggers = logger.Logger(net_path, Pascal3D.PascalClasses, load=True)
loggers.load_network_weights(119, model, device)
model.eval()
if not os.path.exists(image_dir_out):
os.makedirs(image_dir_out)
np.random.seed(9001)
idx = np.arange(len(dataset_vis))
np.random.shuffle(idx)
sampler = ListSampler(idx)
dataloader = torch.utils.data.DataLoader(dataset_vis,
sampler=sampler,
batch_size=1,
drop_last=False)
fig, axs = plt.subplots(3, 4, figsize=(10, 7.5), dpi=100 * 4)
for i, batch in enumerate(dataloader):
if i == 12:
break
ax = axs[i // 4][i % 4]
image, extrinsic, class_idx_cpu, hard, intrinsic, _ = batch
extrinsic_np = extrinsic[0].numpy()
intrinsic_np = intrinsic[0].numpy()
im_np = image[0].numpy().transpose(1, 2, 0)
R_gt = extrinsic[0, :3, :3].numpy()
out = model(image, class_idx_cpu).view(-1, 3, 3)
R_est = loss.batch_torch_A_to_R(out).detach().cpu().view(3, 3).numpy()
err = loss.angle_error_np(R_gt, R_est)
extr_est = np.copy(extrinsic_np)
extr_est[:3, :3] = R_est
points_est = proj_axis(extr_est, intrinsic_np)
points_true = proj_axis(extrinsic_np, intrinsic_np)
ax.imshow(im_np)
for p, c in zip(points_est[:, 1:].transpose(), ['r', 'g', 'b']):
x = [points_est[0, 0], p[0]]
y = [points_est[1, 0], p[1]]
ax.plot(x, y, c, linewidth=5)
for p, c in zip(points_true[:, 1:].transpose(), ['m', 'y', 'c']):
x = [points_true[0, 0], p[0]]
y = [points_true[1, 0], p[1]]
ax.plot(x, y, c, linewidth=2)
ax.title.set_text("error: {:3.2f}".format(err))
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
image_out = os.path.join(image_dir_out, 'im_errors.pdf')
plt.savefig(image_out)
def get_pascal_no_warp_loaders(batch_size, train_all, voc_train):
dataset = Pascal3D.Pascal3D(dataset_dir, train_all=train_all, use_warp=False, voc_train=voc_train)
dataloader_train = torch.utils.data.DataLoader(
dataset.get_train(False),
batch_size=batch_size,
shuffle=True,
num_workers=8,
worker_init_fn=lambda _: np.random.seed(torch.utils.data.get_worker_info().seed % (2**32)),
pin_memory=True,
drop_last=True)
dataloader_eval = torch.utils.data.DataLoader(
dataset.get_eval(),
batch_size=batch_size,
shuffle=False,
num_workers=8,
worker_init_fn=lambda _: np.random.seed(torch.utils.data.get_worker_info().seed % (2**32)),
pin_memory=True,
drop_last=False)
return dataloader_train, dataloader_eval
def get_cpu_stats():
net_path = 'logs/pascal/pascal_new_norm_full'
dataset_location = 'datasets' # TODO update with dataset path
device = torch.device('cpu')
dataset = Pascal3D.Pascal3D(train_all=True)
dataset_vis = dataset.get_eval()
base = resnet101()
model = ResnetHead(base, 13, 32, 512, 9)
loggers = logger.Logger(net_path, Pascal3D.PascalClasses, load=True)
loggers.load_network_weights(119, model, device)
model.eval()
np.random.seed(9001)
dataloader = torch.utils.data.DataLoader(dataset_vis,
shuffle=True,
batch_size=1,
drop_last=False)
err_per_class = {}
print(len(dataloader))
for i, batch in enumerate(dataloader):
if i % 10 == 0:
print(i)
image, extrinsic, class_idx_cpu, hard, intrinsic, _ = batch
extrinsic_np = extrinsic[0].numpy()
intrinsic_np = intrinsic[0].numpy()
im_np = image[0].numpy().transpose(1, 2, 0)
R_gt = extrinsic[0, :3, :3].numpy()
out = model(image, class_idx_cpu).view(-1, 3, 3)
R_est = loss.batch_torch_A_to_R(out).detach().cpu().view(3, 3).numpy()
err = loss.angle_error_np(R_gt, R_est)
k = class_idx_cpu.numpy()[0]
errs = err_per_class.get(k, [])
errs.append(err)
err_per_class[k] = errs
for k in range(13):
if k in err_per_class:
print(k)
print(np.median(err_per_class[k]))
def visualize_probs():
net_path = 'logs/pascal/pascal_new_norm_full'
image_dir_out = 'plots/probs'
dataset_location = 'datasets' # TODO update with dataset path
device = torch.device('cpu')
dataset = Pascal3D.Pascal3D(train_all=True)
dataset_vis = dataset.get_eval()
base = resnet101()
model = ResnetHead(base, 13, 32, 512, 9)
loggers = logger.Logger(net_path, Pascal3D.PascalClasses, load=True)
loggers.load_network_weights(119, model, device)
model.eval()
if not os.path.exists(image_dir_out):
os.makedirs(image_dir_out)
np.random.seed(9001)
# idx = np.arange(1000,len(dataset_vis), 1)
idx = [4008, 1126, 9024, 11159]
sampler = ListSampler(idx)
dataloader = torch.utils.data.DataLoader(dataset_vis,
sampler=sampler,
batch_size=1,
drop_last=False)
im_weight = 4
fig = plt.figure(figsize=(10, 3), dpi=100 * 4)
gs = fig.add_gridspec(im_weight + 3, 4)
for i, (idx, batch) in enumerate(zip(idx, dataloader)):
print(i)
if i == 4:
break
im_ax = fig.add_subplot(gs[:im_weight, i])
image, extrinsic, class_idx_cpu, hard, intrinsic, _ = batch
extrinsic_np = extrinsic[0].numpy()
intrinsic_np = intrinsic[0].numpy()
im_np = image[0].numpy().transpose(1, 2, 0)
R_gt = extrinsic[0, :3, :3].numpy()
out = model(image, class_idx_cpu).view(-1, 3, 3)
R_est = loss.batch_torch_A_to_R(out).detach().cpu().view(3, 3).numpy()
j = Image.fromarray((im_np * 255).astype(np.uint8))
image_out = os.path.join(image_dir_out, 'im_prob_{}.png'.format(i))
j.save(image_out)
print("F")
print(out.detach().numpy())
print("gt")
print(R_gt)
print("est")
print(R_est)
err = loss.angle_error_np(R_gt, R_est)
F = out[0].detach().numpy()
extr_est = np.copy(extrinsic_np)
extr_est[:3, :3] = R_est
points_est = proj_axis(extr_est, intrinsic_np)
print("points_est")
print(points_est)
points_true = proj_axis(extrinsic_np, intrinsic_np)
print("points_true")
print(points_true)
im_ax.imshow(im_np)
for p, c in zip(points_est[:, 1:].transpose(), ['r', 'g', 'b']):
x = [points_est[0, 0], p[0]]
y = [points_est[1, 0], p[1]]
im_ax.plot(x, y, c, linewidth=5)
for p, c in zip(points_true[:, 1:].transpose(), ['m', 'y', 'c']):
x = [points_true[0, 0], p[0]]
y = [points_true[1, 0], p[1]]
im_ax.plot(x, y, c, linewidth=2)
im_ax.axes.get_xaxis().set_visible(False)
im_ax.axes.get_yaxis().set_visible(False)
for rot_axis, c in zip(range(3), ['r', 'g', 'b']):
x, y = get_prob(rot_axis, np.matmul(F.transpose(), R_est))
ml = get_axis_max_likelihood(rot_axis,
np.matmul(F.transpose(), R_gt))
ax = fig.add_subplot(gs[rot_axis + im_weight, i])
ax.plot(x, y, c)
ax.plot([ml, ml], [0.0, 1.0], 'k')
plt.show()
def __getitem__(self, idx):
relpath = self.relpaths[idx]
str_components = relpath.split('/')
file_str = str_components[-1]
synsetID, shapeID, a, e, t, d = file_str.split('_')
assert (a[0] == 'a' and e[0] == 'e' and t[0] == 't')
a, e, t = map(lambda x: float(x[1:]), [a, e, t])
# aug start
img_full = Image.open(os.path.join(self.path, relpath))
img_full = np.array(img_full.getdata()).reshape(
img_full.size[1], img_full.size[0], 3).astype(np.float) / 255
current_size = max(img_full.shape[1], img_full.shape[0])
bbox = [0, 0, img_full.shape[1], img_full.shape[0]]
distance = 4.0
a = a
e = e
t = t
cam = 3000
principal_point = np.array(
[img_full.shape[1] / 2, img_full.shape[0] / 2], dtype=np.float)
flip = np.random.randint(2)
if flip:
a = -a
t = -t
img_full = img_full[:, ::-1, :]
bbox[0] = img_full.shape[1] - bbox[0]
bbox[2] = img_full.shape[1] - bbox[2]
principal_point[0] = img_full.shape[1] - principal_point[0]
# # change up direction of warp
desired_up = np.array([3.0, 0.0, 0.0]) + np.random.normal(
0, 0.4, size=(3))
desired_up[2] = 0
desired_up /= np.linalg.norm(desired_up)
# # jitter bounding box
bbox_w = bbox[2] - bbox[0]
bbox_h = bbox[3] - bbox[1]
bbox[0::2] += np.random.uniform(-bbox_w * 0.1, bbox_w * 0.1, size=(2))
bbox[1::2] += np.random.uniform(-bbox_h * 0.1, bbox_h * 0.1, size=(2))
angle = np.array([a, e, -t])
intrinsic, extrinsic = Pascal3D.get_camera_parameters(
cam, principal_point, angle, img_full.shape, distance)
back_proj_bbx = Pascal3D.get_back_proj_bbx(bbox, intrinsic)
desired_imagesize = self.image_out_size
extrinsic_desired_change, intrinsic_new = Pascal3D.get_desired_camera(
desired_imagesize, back_proj_bbx, desired_up)
extrinsic_after = np.matmul(extrinsic_desired_change, extrinsic)
P = np.matmul(
np.matmul(intrinsic_new, extrinsic_desired_change[:3, :3]),
np.linalg.inv(intrinsic))
P /= P[2, 2]
Pinv = np.linalg.inv(P)
transform = skimage.transform.ProjectiveTransform(Pinv)
warped_image = skimage.transform.warp(img_full,
transform,
output_shape=(desired_imagesize,
desired_imagesize),
mode='constant',
cval=0.0)
class_enum = id_to_pascal_class[synsetID]
return warped_image.transpose(2, 0, 1).astype(
np.float32), extrinsic_after.astype(
np.float32), int(class_enum), False, intrinsic_new.astype(
np.float32), 0