def test_lstm(b_t_ic_hc_otf_sctv, dtype_str, tensor_fn, dev_str, call):
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels), 'float32')
init_h = ivy.ones([b, hidden_channels])
init_c = ivy.ones([b, hidden_channels])
kernel = ivy.variable(ivy.ones([input_channels, 4*hidden_channels]))*0.5
recurrent_kernel = ivy.variable(ivy.ones([hidden_channels, 4*hidden_channels]))*0.5
output, state_c = ivy.lstm_update(x, init_h, init_c, kernel, recurrent_kernel)
# type test
assert ivy.is_array(output)
assert ivy.is_array(state_c)
# cardinality test
assert output.shape == (b, t, hidden_channels)
assert state_c.shape == (b, hidden_channels)
# value test
output_true = np.tile(np.asarray(output_true_flat).reshape((b, t, 1)), (1, 1, hidden_channels))
state_c_true = np.ones([b, hidden_channels]) * state_c_true_val
output, state_c = call(ivy.lstm_update, x, init_h, init_c, kernel, recurrent_kernel)
assert np.allclose(output, output_true, atol=1e-6)
assert np.allclose(state_c, state_c_true, atol=1e-6)
# compilation test
if call in [helpers.torch_call]:
# this is not a backend implemented function
pytest.skip()
helpers.assert_compilable(ivy.lstm_update)
def empty_memory(self, batch_size, timesteps):
uniform_pixel_coords = \
ivy_vision.create_uniform_pixel_coords_image(self._sphere_img_dims, [batch_size, timesteps],
dev_str=self._dev_str)[..., 0:2]
empty_memory = {
'mean':
ivy.concatenate([uniform_pixel_coords] + [
ivy.ones([batch_size, timesteps] + self._sphere_img_dims + [1],
dev_str=self._dev_str) * self._sphere_depth_prior_val
] + [
ivy.ones([batch_size, timesteps] + self._sphere_img_dims +
[self._feat_dim],
dev_str=self._dev_str) * self._sphere_feat_prior_val
], -1),
'var':
ivy.concatenate([
ivy.ones([batch_size, timesteps] + self._sphere_img_dims + [2],
dev_str=self._dev_str) * self._ang_pix_prior_var_val
] + [
ivy.ones([batch_size, timesteps] + self._sphere_img_dims + [1],
dev_str=self._dev_str) * self._depth_prior_var_val
] + [
ivy.ones([batch_size, timesteps] + self._sphere_img_dims +
[self._feat_dim],
dev_str=self._dev_str) * self._feat_prior_var_val
], -1)
}
return ESMMemory(**empty_memory)
def train(self):
optimizer = ivy.Adam(self._lr, dev_str=self._dev_str)
for i in range(self._num_iters + 1):
img_i = np.random.randint(self._images.shape[0])
target = self._images[img_i]
cam_geom = self._cam_geoms.slice(img_i)
rays_o, rays_d = self._get_rays(cam_geom)
loss, grads = ivy.execute_with_gradients(
lambda v: self._loss_fn(self._model, rays_o, rays_d, target, v=v), self._model.v)
self._model.v = optimizer.step(self._model.v, grads)
if i % self._log_freq == 0 and self._log_freq != -1:
print('step {}, loss {}'.format(i, ivy.to_numpy(loss).item()))
if i % self._vis_freq == 0 and self._vis_freq != -1:
# Render the holdout view for logging
rays_o, rays_d = self._get_rays(self._test_cam_geom)
rgb, depth = ivy_vision.render_implicit_features_and_depth(
self._model, rays_o, rays_d, near=ivy.ones(self._img_dims, dev_str=self._dev_str) * 2,
far=ivy.ones(self._img_dims, dev_str=self._dev_str)*6, samples_per_ray=self._num_samples)
plt.imsave(os.path.join(self._vis_log_dir, 'img_{}.png'.format(str(i).zfill(3))), ivy.to_numpy(rgb))
print('Completed Training')
def test_lstm_layer(b_t_ic_hc_otf_sctv, with_v, with_initial_state, dtype_str,
tensor_fn, dev_str, call):
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(
ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels),
'float32')
if with_initial_state:
init_h = ivy.ones([b, hidden_channels])
init_c = ivy.ones([b, hidden_channels])
initial_state = ([init_h], [init_c])
else:
initial_state = None
if with_v:
kernel = ivy.variable(
ivy.ones([input_channels, 4 * hidden_channels]) * 0.5)
recurrent_kernel = ivy.variable(
ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5)
v = Container({
'input': {
'layer_0': {
'w': kernel
}
},
'recurrent': {
'layer_0': {
'w': recurrent_kernel
}
}
})
else:
v = None
lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v)
output, (state_h, state_c) = lstm_layer(x, initial_state=initial_state)
# type test
assert ivy.is_array(output)
assert ivy.is_array(state_h[0])
assert ivy.is_array(state_c[0])
# cardinality test
assert output.shape == (b, t, hidden_channels)
assert state_h[0].shape == (b, hidden_channels)
assert state_c[0].shape == (b, hidden_channels)
# value test
if not with_v or not with_initial_state:
return
output_true = np.tile(
np.asarray(output_true_flat).reshape((b, t, 1)),
(1, 1, hidden_channels))
state_c_true = np.ones([b, hidden_channels]) * state_c_true_val
output, (state_h, state_c) = call(lstm_layer,
x,
initial_state=initial_state)
assert np.allclose(output, output_true, atol=1e-6)
assert np.allclose(state_c, state_c_true, atol=1e-6)
# compilation test
if call in [helpers.torch_call]:
# this is not a backend implemented function
pytest.skip()
helpers.assert_compilable(ivy.lstm_update)
def create_trimesh_indices_for_image(batch_shape, image_dims, dev_str='cpu:0'):
"""
Create triangle mesh for image with given image dimensions
:param batch_shape: Shape of batch.
:type batch_shape: sequence of ints
:param image_dims: Image dimensions.
:type image_dims: sequence of ints
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc.
:type dev_str: str, optional
:return: Triangle mesh indices for image *[batch_shape,h*w*some_other_stuff,3]*
"""
# shapes as lists
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# other shape specs
num_batch_dims = len(batch_shape)
tri_dim = 2 * (image_dims[0] - 1) * (image_dims[1] - 1)
flat_shape = [1] * num_batch_dims + [tri_dim] + [3]
tile_shape = batch_shape + [1] * 2
# 1 x W-1
t00_ = _ivy.reshape(_ivy.arange(image_dims[1] - 1, dtype_str='float32', dev_str=dev_str), (1, -1))
# H-1 x 1
k_ = _ivy.reshape(_ivy.arange(image_dims[0] - 1, dtype_str='float32', dev_str=dev_str), (-1, 1)) * image_dims[1]
# H-1 x W-1
t00_ = _ivy.matmul(_ivy.ones((image_dims[0] - 1, 1), dev_str=dev_str), t00_)
k_ = _ivy.matmul(k_, _ivy.ones((1, image_dims[1] - 1), dev_str=dev_str))
# (H-1xW-1) x 1
t00 = _ivy.expand_dims(t00_ + k_, -1)
t01 = t00 + 1
t02 = t00 + image_dims[1]
t10 = t00 + image_dims[1] + 1
t11 = t01
t12 = t02
# (H-1xW-1) x 3
t0 = _ivy.concatenate((t00, t01, t02), -1)
t1 = _ivy.concatenate((t10, t11, t12), -1)
# BS x 2x(H-1xW-1) x 3
return _ivy.tile(_ivy.reshape(_ivy.concatenate((t0, t1), 0),
flat_shape), tile_shape)
def test_linear_layer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str,
call):
# smoke test
batch_shape, input_channels, output_channels, target = bs_ic_oc_target
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
if with_v:
np.random.seed(0)
wlim = (6 / (output_channels + input_channels))**0.5
w = ivy.variable(
ivy.array(
np.random.uniform(-wlim, wlim,
(output_channels, input_channels)),
'float32'))
b = ivy.variable(ivy.zeros([output_channels]))
v = Container({'w': w, 'b': b})
else:
v = None
linear_layer = ivy.Linear(input_channels, output_channels, v=v)
ret = linear_layer(x)
# type test
assert ivy.is_array(ret)
# cardinality test
assert ret.shape == tuple(batch_shape + [output_channels])
# value test
if not with_v:
return
assert np.allclose(call(linear_layer, x), np.array(target))
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(linear_layer)
def weighted_image_smooth(mean, weights, kernel_dim):
"""
Smooth an image using weight values from a weight image of the same size.
:param mean: Image to smooth *[batch_shape,h,w,d]*
:type mean: array
:param weights: Variance image, with the variance values of each pixel in the image *[batch_shape,h,w,d]*
:type weights: array
:param kernel_dim: The dimension of the kernel
:type kernel_dim: int
:return: Image smoothed based on variance image and smoothing kernel.
"""
# shapes as list
kernel_shape = [kernel_dim, kernel_dim]
dim = mean.shape[-1]
# KW x KW x D
kernel = _ivy.ones(kernel_shape + [dim])
# D
kernel_sum = _ivy.reduce_sum(kernel, [0, 1])[0]
# BS x H x W x D
mean_x_weights = mean * weights
mean_x_weights_sum = _ivy.abs(
_ivy.depthwise_conv2d(mean_x_weights, kernel, 1, "VALID"))
sum_of_weights = _ivy.depthwise_conv2d(weights, kernel, 1, "VALID")
new_mean = mean_x_weights_sum / (sum_of_weights + MIN_DENOMINATOR)
new_weights = sum_of_weights / (kernel_sum + MIN_DENOMINATOR)
# BS x H x W x D, # BS x H x W x D
return new_mean, new_weights
def as_identity(batch_shape):
"""
Return camera intrinsics object with array attributes as either zeros or identity matrices.
:param batch_shape: Batch shape for each geometric array attribute
:type batch_shape: sequence of ints
:return: New camera intrinsics object, with each entry as either zeros or identity matrices.
"""
batch_shape = list(batch_shape)
focal_lengths = _ivy.ones(batch_shape + [2])
persp_angles = _ivy.ones(batch_shape + [2])
pp_offsets = _ivy.zeros(batch_shape + [2])
calib_mats = _ivy.identity(3, batch_shape=batch_shape)
inv_calib_mats = _ivy.identity(3, batch_shape=batch_shape)
return __class__(focal_lengths, persp_angles, pp_offsets, calib_mats,
inv_calib_mats)
def as_identity(batch_shape):
"""
Return primitive scene object with array attributes as either zeros or identity matrices.
:param batch_shape: Batch shape for each geometric array attribute
:type batch_shape: sequence of ints
:return: New primitive scene object, with each entry as either zeros or identity matrices.
"""
batch_shape = list(batch_shape)
sphere_positions = _ivy.identity(4, batch_shape=batch_shape)[...,
0:3, :]
sphere_radii = _ivy.ones(batch_shape + [1])
cuboid_ext_mats = _ivy.identity(4, batch_shape=batch_shape)[...,
0:3, :]
cuboid_dims = _ivy.ones(batch_shape + [3])
return __class__(sphere_positions, sphere_radii, cuboid_ext_mats,
cuboid_dims)
def __init__(self, base_inv_ext_mat=None):
a_s = ivy.array([0.5, 0.5])
d_s = ivy.array([0., 0.])
alpha_s = ivy.array([0., 0.])
dh_joint_scales = ivy.ones((2, ))
dh_joint_offsets = ivy.array([-np.pi / 2, 0.])
super().__init__(a_s, d_s, alpha_s, dh_joint_scales,
dh_joint_offsets, base_inv_ext_mat)
def get_fundamental_matrix(full_mat1,
full_mat2,
camera_center1=None,
pinv_full_mat1=None,
batch_shape=None,
dev_str=None):
"""
Compute fundamental matrix :math:`\mathbf{F}\in\mathbb{R}^{3×3}` between two cameras, given their extrinsic
matrices :math:`\mathbf{E}_1\in\mathbb{R}^{3×4}` and :math:`\mathbf{E}_2\in\mathbb{R}^{3×4}`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=262>`_
bottom of page 244, section 9.2.2, equation 9.1
:param full_mat1: Frame 1 full projection matrix *[batch_shape,3,4]*
:type full_mat1: array
:param full_mat2: Frame 2 full projection matrix *[batch_shape,3,4]*
:type full_mat2: array
:param camera_center1: Frame 1 camera center, inferred from full_mat1 if None *[batch_shape,3,1]*
:type camera_center1: array, optional
:param pinv_full_mat1: Frame 1 full projection matrix pseudo-inverse, inferred from full_mat1 if None *[batch_shape,4,3]*
:type pinv_full_mat1: array, optional
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Fundamental matrix connecting frames 1 and 2 *[batch_shape,3,3]*
"""
if batch_shape is None:
batch_shape = full_mat1.shape[:-2]
if dev_str is None:
dev_str = _ivy.dev_str(full_mat1)
# shapes as list
batch_shape = list(batch_shape)
if camera_center1 is None:
inv_full_mat1 = _ivy.inv(
_ivy_mech.make_transformation_homogeneous(full_mat1, batch_shape,
dev_str))[..., 0:3, :]
camera_center1 = _ivy_svg.inv_ext_mat_to_camera_center(inv_full_mat1)
if pinv_full_mat1 is None:
pinv_full_mat1 = _ivy.pinv(full_mat1)
# BS x 4 x 1
camera_center1_homo = _ivy.concatenate(
(camera_center1, _ivy.ones(batch_shape + [1, 1], dev_str=dev_str)), -2)
# BS x 3
e2 = _ivy.matmul(full_mat2, camera_center1_homo)[..., -1]
# BS x 3 x 3
e2_skew_symmetric = _ivy.linalg.vector_to_skew_symmetric_matrix(e2)
# BS x 3 x 3
return _ivy.matmul(e2_skew_symmetric, _ivy.matmul(full_mat2,
pinv_full_mat1))
def test_sgd_optimizer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str,
call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels, target = bs_ic_oc_target
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
if with_v:
np.random.seed(0)
wlim = (6 / (output_channels + input_channels))**0.5
w = ivy.variable(
ivy.array(
np.random.uniform(-wlim, wlim,
(output_channels, input_channels)),
'float32'))
b = ivy.variable(ivy.zeros([output_channels]))
v = Container({'w': w, 'b': b})
else:
v = None
linear_layer = ivy.Linear(input_channels, output_channels, v=v)
def loss_fn(v_):
out = linear_layer(x, v=v_)
return ivy.reduce_mean(out)[0]
# optimizer
optimizer = ivy.SGD()
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, linear_layer.v)
linear_layer.v = optimizer.step(linear_layer.v, grads)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
assert ivy.reduce_max(ivy.abs(grads.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.w)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(loss_fn)
def save_video(self):
if not self._interactive:
return
trans_t = lambda t: ivy.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, t],
[0, 0, 0, 1],
], 'float32')
rot_phi = lambda phi: ivy.array([
[1, 0, 0, 0],
[0, np.cos(phi), -np.sin(phi), 0],
[0, np.sin(phi), np.cos(phi), 0],
[0, 0, 0, 1],
], 'float32')
rot_theta = lambda th_: ivy.array([
[np.cos(th_), 0, -np.sin(th_), 0],
[0, 1, 0, 0],
[np.sin(th_), 0, np.cos(th_), 0],
[0, 0, 0, 1],
], 'float32')
def pose_spherical(theta, phi, radius):
c2w_ = trans_t(radius)
c2w_ = rot_phi(phi / 180. * np.pi) @ c2w_
c2w_ = rot_theta(theta / 180. * np.pi) @ c2w_
c2w_ = ivy.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]], 'float32') @ c2w_
c2w_ = c2w_ @ ivy.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]], 'float32')
return c2w_
frames = []
for th in tqdm(np.linspace(0., 360., 120, endpoint=False)):
c2w = ivy.to_dev(pose_spherical(th, -30., 4.), self._dev_str)
cam_geom = ivy_vision.inv_ext_mat_and_intrinsics_to_cam_geometry_object(
c2w[0:3], self._intrinsics.slice(0))
rays_o, rays_d = self._get_rays(cam_geom)
rgb, depth = ivy_vision.render_implicit_features_and_depth(
self._model, rays_o, rays_d, near=ivy.ones(self._img_dims, dev_str=self._dev_str) * 2,
far=ivy.ones(self._img_dims, dev_str=self._dev_str) * 6, samples_per_ray=self._num_samples)
frames.append((255 * np.clip(ivy.to_numpy(rgb), 0, 1)).astype(np.uint8))
import imageio
vid_filename = 'nerf_video.mp4'
imageio.mimwrite(vid_filename, frames, fps=30, quality=7)
def _get_dummy_obs(batch_size, num_frames, num_cams, image_dims, num_feature_channels, dev_str='cpu', ones=False,
empty=False):
uniform_pixel_coords =\
ivy_vision.create_uniform_pixel_coords_image(image_dims, [batch_size, num_frames], dev_str=dev_str)
img_meas = dict()
for i in range(num_cams):
validity_mask = ivy.ones([batch_size, num_frames] + image_dims + [1], dev_str=dev_str)
if ones:
img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.ones(
[batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1)
img_var = ivy.ones(
[batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str)*1e-3
pose_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str)
pose_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3
else:
img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.random_uniform(
1e-3, 1, [batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1)
img_var = ivy.random_uniform(
1e-3, 1, [batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str)
pose_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str)
pose_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str)
if empty:
img_var = ivy.ones_like(img_var) * 1e12
validity_mask = ivy.zeros_like(validity_mask)
img_meas['dummy_cam_{}'.format(i)] =\
{'img_mean': img_mean,
'img_var': img_var,
'validity_mask': validity_mask,
'pose_mean': pose_mean,
'pose_cov': pose_cov,
'cam_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames], dev_str=dev_str)[..., 0:3, :]}
if ones:
control_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str)
control_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3
else:
control_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str)
control_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str)
return Container({'img_meas': img_meas,
'control_mean': control_mean,
'control_cov': control_cov,
'agent_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames],
dev_str=dev_str)[..., 0:3, :]})
def test_module_w_none_attribute(bs_ic_oc, dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels = bs_ic_oc
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
module = ModuleWithNoneAttribute()
def test_exception(self, dev_str, dtype_str, call):
input_ = ivy.ones((1, 1, 3, 4), dev_str=dev_str, dtype_str=dtype_str)
kernel = ivy.ones((3, 3), dev_str=dev_str, dtype_str=dtype_str)
with pytest.raises(ValueError):
test = ivy.ones((2, 3, 4), dev_str=dev_str, dtype_str=dtype_str)
assert black_hat(test, kernel)
with pytest.raises(ValueError):
test = ivy.ones((2, 3, 4), dev_str=dev_str, dtype_str=dtype_str)
assert black_hat(input_, test)
if call is not helpers.torch_call:
return
with pytest.raises(TypeError):
assert black_hat([0.], kernel)
with pytest.raises(TypeError):
assert black_hat(input_, [0.])
def test_stack_images(shp_n_num_n_ar_n_newshp, dev_str, call):
# smoke test
shape, num, ar, new_shape = shp_n_num_n_ar_n_newshp
xs = [ivy.ones(shape)] * num
ret = ivy.stack_images(xs, ar)
# type test
assert ivy.is_array(ret)
# cardinality test
assert ret.shape == new_shape
# compilation test
helpers.assert_compilable(ivy.stack_images)
def ds_pixel_to_world_coords(ds_pixel_coords, inv_full_mat, batch_shape=None, image_dims=None, dev_str=None):
"""
Get world-centric homogeneous co-ordinates image :math:`\mathbf{X}_w\in\mathbb{R}^{h×w×4}` from depth scaled
homogeneous pixel co-ordinates image :math:`\mathbf{X}_p\in\mathbb{R}^{h×w×3}`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=173>`_
combination of page 155, matrix inverse of equation 6.3, and matrix inverse of page 156, equation 6.6
:param ds_pixel_coords: Depth scaled homogeneous pixel co-ordinates image: *[batch_shape,h,w,3]*
:type ds_pixel_coords: array
:param inv_full_mat: Inverse full projection matrix *[batch_shape,3,4]*
:type inv_full_mat: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: World-centric homogeneous co-ordinates image *[batch_shape,h,w,4]*
"""
if batch_shape is None:
batch_shape = ds_pixel_coords.shape[:-3]
if image_dims is None:
image_dims = ds_pixel_coords.shape[-3:-1]
if dev_str is None:
dev_str = _ivy.dev_str(ds_pixel_coords)
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# BS x H x W x 4
ds_pixel_coords = _ivy.concatenate((ds_pixel_coords, _ivy.ones(batch_shape + image_dims + [1], dev_str=dev_str)), -1)
# BS x H x W x 3
world_coords = _ivy_pg.transform(ds_pixel_coords, inv_full_mat, batch_shape, image_dims)
# BS x H x W x 4
return _ivy.concatenate((world_coords, _ivy.ones(batch_shape + image_dims + [1], dev_str=dev_str)), -1)
def test_jit(self, dev_str, dtype_str, call):
op = top_hat
if call in [helpers.jnp_call, helpers.torch_call]:
# compiled jax tensors do not have device_buffer attribute, preventing device info retrieval,
# pytorch scripting does not support .type() casting, nor Union or Numbers for type hinting
pytest.skip()
op_compiled = ivy.compile_fn(op)
input_ = ivy.cast(ivy.random_uniform(shape=(1, 2, 7, 7), dev_str=dev_str), dtype_str)
kernel = ivy.ones((3, 3), dev_str=dev_str, dtype_str=dtype_str)
actual = call(op_compiled, input_, kernel)
expected = call(op, input_, kernel)
assert np.allclose(actual, expected)
def _create_variables(self, dev_str):
vars_dict = dict()
wlim = (6 / (2 * self._memory_vector_dim)) ** 0.5
vars_dict['read_weights'] =\
dict(zip(['w_' + str(i) for i in range(self._read_head_num)],
[ivy.variable(ivy.random_uniform(-wlim, wlim, [self._memory_vector_dim, ], dev_str=dev_str))
for _ in range(self._read_head_num)]))
wlim = (6 / (2 * self._memory_size)) ** 0.5
vars_dict['write_weights'] =\
dict(zip(['w_' + str(i) for i in range(self._read_head_num + self._write_head_num)],
[ivy.variable(ivy.random_uniform(-wlim, wlim, [self._memory_size, ], dev_str=dev_str))
for _ in range(self._read_head_num + self._write_head_num)]))
vars_dict['memory'] = ivy.variable(
ivy.ones([self._memory_size, self._memory_vector_dim], dev_str=dev_str) * self._init_value)
return vars_dict
def test_module_training(bs_ic_oc, dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels = bs_ic_oc
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
module = TrainableModule(input_channels, output_channels)
def loss_fn(v_):
out = module(x, v=v_)
return ivy.reduce_mean(out)[0]
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, module.v)
module.v = ivy.gradient_descent_update(module.v, grads, 1e-3)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
assert ivy.reduce_max(ivy.abs(grads.linear0.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear0.w)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear1.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear1.w)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear2.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear2.w)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not support **kwargs
return
helpers.assert_compilable(loss_fn)
def ds_pixel_to_ds_pixel_coords(ds_pixel_coords1,
cam1to2_full_mat,
batch_shape=None,
image_dims=None,
dev_str=None):
"""
Transform depth scaled homogeneous pixel co-ordinates image in first camera frame
:math:`\mathbf{X}_{p1}\in\mathbb{R}^{h×w×3}` to depth scaled homogeneous pixel co-ordinates image in second camera
frame :math:`\mathbf{X}_{p2}\in\mathbb{R}^{h×w×3}`, given camera to camera projection matrix
:math:`\mathbf{P}_{1→2}\in\mathbb{R}^{3×4}`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=174>`_
:param ds_pixel_coords1: Depth scaled homogeneous pixel co-ordinates image in frame 1 *[batch_shape,h,w,3]*
:type ds_pixel_coords1: array
:param cam1to2_full_mat: Camera1-to-camera2 full projection matrix *[batch_shape,3,4]*
:type cam1to2_full_mat: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Depth scaled homogeneous pixel co-ordinates image in frame 2 *[batch_shape,h,w,3]*
"""
if batch_shape is None:
batch_shape = ds_pixel_coords1.shape[:-3]
if image_dims is None:
image_dims = ds_pixel_coords1.shape[-3:-1]
if dev_str is None:
dev_str = _ivy.dev_str(ds_pixel_coords1)
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# BS x H x W x 4
pixel_coords_homo = _ivy.concatenate(
(ds_pixel_coords1,
_ivy.ones(batch_shape + image_dims + [1], dev_str=dev_str)), -1)
# BS x H x W x 3
return _ivy_pg.transform(pixel_coords_homo, cam1to2_full_mat, batch_shape,
image_dims)
def cam_to_cam_coords(cam_coords1,
cam1to2_ext_mat,
batch_shape=None,
image_dims=None,
dev_str=None):
"""
Transform camera-centric homogeneous co-ordinates image for camera 1 :math:`\mathbf{X}_{c1}\in\mathbb{R}^{h×w×4}` to
camera-centric homogeneous co-ordinates image for camera 2 :math:`\mathbf{X}_{c2}\in\mathbb{R}^{h×w×4}`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=174>`_
:param cam_coords1: Camera-centric homogeneous co-ordinates image in frame 1 *[batch_shape,h,w,4]*
:type cam_coords1: array
:param cam1to2_ext_mat: Camera1-to-camera2 extrinsic projection matrix *[batch_shape,3,4]*
:type cam1to2_ext_mat: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Depth scaled homogeneous pixel co-ordinates image in frame 2 *[batch_shape,h,w,3]*
"""
if batch_shape is None:
batch_shape = cam_coords1.shape[:-3]
if image_dims is None:
image_dims = cam_coords1.shape[-3:-1]
if dev_str is None:
dev_str = _ivy.dev_str(cam_coords1)
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# BS x H x W x 3
cam_coords2 = _ivy_pg.transform(cam_coords1, cam1to2_ext_mat, batch_shape,
image_dims)
# BS x H x W x 4
return _ivy.concatenate(
(cam_coords2, _ivy.ones(batch_shape + image_dims + [1],
dev_str=dev_str)), -1)
def focal_lengths_and_pp_offsets_to_calib_mat(focal_lengths, pp_offsets, batch_shape=None, dev_str=None):
"""
Compute calibration matrix :math:`\mathbf{K}\in\mathbb{R}^{3×3}` from focal lengths :math:`f_x, f_y` and
principal-point offsets :math:`p_x, p_y`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=173>`_
page 155, section 6.1, equation 6.4
:param focal_lengths: Focal lengths *[batch_shape,2]*
:type focal_lengths: array
:param pp_offsets: Principal-point offsets *[batch_shape,2]*
:type pp_offsets: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Calibration matrix *[batch_shape,3,3]*
"""
if batch_shape is None:
batch_shape = focal_lengths.shape[:-1]
if dev_str is None:
dev_str = _ivy.dev_str(focal_lengths)
# shapes as list
batch_shape = list(batch_shape)
# BS x 1 x 1
zeros = _ivy.zeros(batch_shape + [1, 1], dev_str=dev_str)
ones = _ivy.ones(batch_shape + [1, 1], dev_str=dev_str)
# BS x 2 x 1
focal_lengths_reshaped = _ivy.expand_dims(focal_lengths, -1)
pp_offsets_reshaped = _ivy.expand_dims(pp_offsets, -1)
# BS x 1 x 3
row1 = _ivy.concatenate((focal_lengths_reshaped[..., 0:1, :], zeros, pp_offsets_reshaped[..., 0:1, :]), -1)
row2 = _ivy.concatenate((zeros, focal_lengths_reshaped[..., 1:2, :], pp_offsets_reshaped[..., 1:2, :]), -1)
row3 = _ivy.concatenate((zeros, zeros, ones), -1)
# BS x 3 x 3
return _ivy.concatenate((row1, row2, row3), -2)
def sphere_to_cam_coords(sphere_coords, forward_facing_z=True, batch_shape=None, image_dims=None, dev_str=None):
"""
Convert camera-centric ego-sphere polar co-ordinates image :math:`\mathbf{S}_c\in\mathbb{R}^{h×w×3}` to
camera-centric homogeneous cartesian co-ordinates image :math:`\mathbf{X}_c\in\mathbb{R}^{h×w×4}`.\n
`[reference] <https://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates>`_
:param sphere_coords: Camera-centric ego-sphere polar co-ordinates image *[batch_shape,h,w,3]*
:type sphere_coords: array
:param forward_facing_z: Whether to use reference frame so z is forward facing. Default is False.
:type forward_facing_z: bool, optional
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: *Camera-centric homogeneous cartesian co-ordinates image *[batch_shape,h,w,4]*
"""
if batch_shape is None:
batch_shape = sphere_coords.shape[:-3]
if image_dims is None:
image_dims = sphere_coords.shape[-3:-1]
if dev_str is None:
dev_str = _ivy.dev_str(sphere_coords)
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# BS x H x W x 3
cam_coords = _ivy_mec.polar_to_cartesian_coords(sphere_coords)
if forward_facing_z:
cam_coords = _ivy.concatenate(
(cam_coords[..., 1:2], cam_coords[..., 2:3], cam_coords[..., 0:1]), -1)
# BS x H x W x 4
return _ivy.concatenate((cam_coords, _ivy.ones(batch_shape + image_dims + [1], dev_str=dev_str)), -1)
def __init__(self, base_inv_ext_mat=None):
"""
Initialize FRANKA EMIKA Panda robot manipulator instance.
Denavit–Hartenberg parameters inferred from FRANKA EMIKA online API.
Screenshot included in this module for reference.
:param base_inv_ext_mat: Inverse extrinsic matrix of the robot base *[3,4]*
:type base_inv_ext_mat: array, optional
"""
# dh params
# panda_DH_params.png
# taken from https://frankaemika.github.io/docs/control_parameters.html
a_s = _ivy.array([0., 0., 0.0825, -0.0825, 0., 0.088, 0.])
d_s = _ivy.array([0.333, 0., 0.316, 0., 0.384, 0., 0.107])
alpha_s = _ivy.array(
[-_math.pi / 2, _math.pi / 2, _math.pi / 2, -_math.pi / 2, _math.pi / 2, _math.pi / 2, 0.])
dh_joint_scales = _ivy.ones((7,))
dh_joint_offsets = _ivy.zeros((7,))
super().__init__(a_s, d_s, alpha_s, dh_joint_scales, dh_joint_offsets, base_inv_ext_mat)
def cam_to_world_coords(coords_wrt_cam, inv_ext_mat, batch_shape=None, image_dims=None, dev_str=None):
"""
Get world-centric homogeneous co-ordinates image :math:`\mathbf{X}_w\in\mathbb{R}^{h×w×4}` from camera-centric
homogeneous co-ordinates image :math:`\mathbf{X}_c\in\mathbb{R}^{h×w×4}`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=174>`_
matrix inverse of page 156, equation 6.6
:param coords_wrt_cam: Camera-centric homogeneous co-ordinates image *[batch_shape,h,w,4]*
:type coords_wrt_cam: array
:param inv_ext_mat: Inverse extrinsic matrix *[batch_shape,3,4]*
:type inv_ext_mat: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: World-centric homogeneous co-ordinates image *[batch_shape,h,w,4]*
"""
if batch_shape is None:
batch_shape = coords_wrt_cam.shape[:-3]
if image_dims is None:
image_dims = coords_wrt_cam.shape[-3:-1]
if dev_str is None:
dev_str = _ivy.dev_str(coords_wrt_cam)
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
# BS x H x W x 3
world_coords = _ivy_pg.transform(coords_wrt_cam, inv_ext_mat, batch_shape, image_dims)
# BS x H x W x 4
return _ivy.concatenate((world_coords, _ivy.ones(batch_shape + image_dims + [1], dev_str=dev_str)), -1)
def __init__(self, data_loader_spec):
super().__init__(data_loader_spec)
# dataset size
self._num_examples = self._spec.dataset_spec.num_examples
# counter
self._i = 0
# load vector data
vector_dim = self._spec.dataset_spec.vector_dim
self._targets = ivy.zeros((self._num_examples, vector_dim, 1))
# load image data
image_dims = self._spec.dataset_spec.image_dims
self._input = ivy.ones(
(self._num_examples, image_dims[0], image_dims[1], 3))
self._training_data = ivy.Container(targets=self._targets,
input=self._input)
self._validation_data = ivy.Container(targets=self._targets,
input=self._input)
self._data = ivy.Container(training=self._training_data,
validation=self._validation_data)
def test_cardinality(self, dev_str, dtype_str, call, shape, kernel):
img = ivy.ones(shape, dtype_str=dtype_str, dev_str=dev_str)
krnl = ivy.ones(kernel, dtype_str=dtype_str, dev_str=dev_str)
assert black_hat(img, krnl).shape == shape
def test_lstm_layer_training(b_t_ic_hc_otf_sctv, with_v, dtype_str, tensor_fn,
dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(
ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels),
'float32')
if with_v:
kernel = ivy.variable(
ivy.ones([input_channels, 4 * hidden_channels]) * 0.5)
recurrent_kernel = ivy.variable(
ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5)
v = Container({
'input': {
'layer_0': {
'w': kernel
}
},
'recurrent': {
'layer_0': {
'w': recurrent_kernel
}
}
})
else:
v = None
lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v)
def loss_fn(v_):
out, (state_h, state_c) = lstm_layer(x, v=v_)
return ivy.reduce_mean(out)[0]
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, lstm_layer.v)
lstm_layer.v = ivy.gradient_descent_update(lstm_layer.v, grads, 1e-3)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
for key, val in grads.to_iterator():
assert ivy.reduce_max(ivy.abs(val)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(loss_fn)
