def test_linear_layer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str,
call):
# smoke test
batch_shape, input_channels, output_channels, target = bs_ic_oc_target
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
if with_v:
np.random.seed(0)
wlim = (6 / (output_channels + input_channels))**0.5
w = ivy.variable(
ivy.array(
np.random.uniform(-wlim, wlim,
(output_channels, input_channels)),
'float32'))
b = ivy.variable(ivy.zeros([output_channels]))
v = Container({'w': w, 'b': b})
else:
v = None
linear_layer = ivy.Linear(input_channels, output_channels, v=v)
ret = linear_layer(x)
# type test
assert ivy.is_array(ret)
# cardinality test
assert ret.shape == tuple(batch_shape + [output_channels])
# value test
if not with_v:
return
assert np.allclose(call(linear_layer, x), np.array(target))
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(linear_layer)
def _fit_spline(train_points, train_values, order):
# shapes
train_points_shape = train_points.shape
batch_shape = list(train_points_shape[:-2])
num_batch_dims = len(batch_shape)
n = train_points_shape[-2]
pd = train_values.shape[-1]
# BS x N x 1
c = train_points
# BS x N x PD
f_ = train_values
# BS x N x N
matrix_a = _phi(_pairwise_distance(c, c), order)
# BS x N x 1
ones = _ivy.ones_like(c[..., :1])
# BS x N x 2
matrix_b = _ivy.concatenate([c, ones], -1)
# BS x 2 x N
matrix_b_trans = _ivy.transpose(
matrix_b,
list(range(num_batch_dims)) + [num_batch_dims + 1, num_batch_dims])
# BS x N+2 x N
left_block = _ivy.concatenate([matrix_a, matrix_b_trans], -2)
# BS x 2 x 2
lhs_zeros = _ivy.zeros(batch_shape + [2, 2])
# BS x N+2 x 2
right_block = _ivy.concatenate([matrix_b, lhs_zeros], -2)
# BS x N+2 x N+2
lhs = _ivy.concatenate([left_block, right_block], -1)
# BS x 2 x PD
rhs_zeros = _ivy.zeros(batch_shape + [2, pd])
# BS x N+2 x PD
rhs = _ivy.concatenate([f_, rhs_zeros], -2)
# BS x N+2 x PD
w_v = _ivy.matmul(_ivy.pinv(lhs), rhs)
# BS x N x PD
w = w_v[..., :n, :]
# BS x 2 x PD
v = w_v[..., n:, :]
# BS x N x PD, BS x 2 x PD
return w, v
def test_sgd_optimizer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str,
call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels, target = bs_ic_oc_target
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
if with_v:
np.random.seed(0)
wlim = (6 / (output_channels + input_channels))**0.5
w = ivy.variable(
ivy.array(
np.random.uniform(-wlim, wlim,
(output_channels, input_channels)),
'float32'))
b = ivy.variable(ivy.zeros([output_channels]))
v = Container({'w': w, 'b': b})
else:
v = None
linear_layer = ivy.Linear(input_channels, output_channels, v=v)
def loss_fn(v_):
out = linear_layer(x, v=v_)
return ivy.reduce_mean(out)[0]
# optimizer
optimizer = ivy.SGD()
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, linear_layer.v)
linear_layer.v = optimizer.step(linear_layer.v, grads)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
assert ivy.reduce_max(ivy.abs(grads.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.w)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(loss_fn)
def get_initial_state(self, batch_shape):
"""
Get the initial state of the hidden and cell states, if not provided explicitly
"""
batch_shape = list(batch_shape)
return ([
ivy.zeros((batch_shape + [self._output_channels]))
for i in range(self._num_layers)
], [
ivy.zeros((batch_shape + [self._output_channels]))
for i in range(self._num_layers)
])
def test_lstm(b_t_ic_hc_otf_sctv, dtype_str, tensor_fn, dev_str, call):
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels), 'float32')
init_h = ivy.ones([b, hidden_channels])
init_c = ivy.ones([b, hidden_channels])
kernel = ivy.variable(ivy.ones([input_channels, 4*hidden_channels]))*0.5
recurrent_kernel = ivy.variable(ivy.ones([hidden_channels, 4*hidden_channels]))*0.5
output, state_c = ivy.lstm_update(x, init_h, init_c, kernel, recurrent_kernel)
# type test
assert ivy.is_array(output)
assert ivy.is_array(state_c)
# cardinality test
assert output.shape == (b, t, hidden_channels)
assert state_c.shape == (b, hidden_channels)
# value test
output_true = np.tile(np.asarray(output_true_flat).reshape((b, t, 1)), (1, 1, hidden_channels))
state_c_true = np.ones([b, hidden_channels]) * state_c_true_val
output, state_c = call(ivy.lstm_update, x, init_h, init_c, kernel, recurrent_kernel)
assert np.allclose(output, output_true, atol=1e-6)
assert np.allclose(state_c, state_c_true, atol=1e-6)
# compilation test
if call in [helpers.torch_call]:
# this is not a backend implemented function
pytest.skip()
helpers.assert_compilable(ivy.lstm_update)
def test_lstm_layer(b_t_ic_hc_otf_sctv, with_v, with_initial_state, dtype_str,
tensor_fn, dev_str, call):
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(
ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels),
'float32')
if with_initial_state:
init_h = ivy.ones([b, hidden_channels])
init_c = ivy.ones([b, hidden_channels])
initial_state = ([init_h], [init_c])
else:
initial_state = None
if with_v:
kernel = ivy.variable(
ivy.ones([input_channels, 4 * hidden_channels]) * 0.5)
recurrent_kernel = ivy.variable(
ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5)
v = Container({
'input': {
'layer_0': {
'w': kernel
}
},
'recurrent': {
'layer_0': {
'w': recurrent_kernel
}
}
})
else:
v = None
lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v)
output, (state_h, state_c) = lstm_layer(x, initial_state=initial_state)
# type test
assert ivy.is_array(output)
assert ivy.is_array(state_h[0])
assert ivy.is_array(state_c[0])
# cardinality test
assert output.shape == (b, t, hidden_channels)
assert state_h[0].shape == (b, hidden_channels)
assert state_c[0].shape == (b, hidden_channels)
# value test
if not with_v or not with_initial_state:
return
output_true = np.tile(
np.asarray(output_true_flat).reshape((b, t, 1)),
(1, 1, hidden_channels))
state_c_true = np.ones([b, hidden_channels]) * state_c_true_val
output, (state_h, state_c) = call(lstm_layer,
x,
initial_state=initial_state)
assert np.allclose(output, output_true, atol=1e-6)
assert np.allclose(state_c, state_c_true, atol=1e-6)
# compilation test
if call in [helpers.torch_call]:
# this is not a backend implemented function
pytest.skip()
helpers.assert_compilable(ivy.lstm_update)
def _get_dummy_obs(batch_size, num_frames, num_cams, image_dims, num_feature_channels, dev_str='cpu', ones=False,
empty=False):
uniform_pixel_coords =\
ivy_vision.create_uniform_pixel_coords_image(image_dims, [batch_size, num_frames], dev_str=dev_str)
img_meas = dict()
for i in range(num_cams):
validity_mask = ivy.ones([batch_size, num_frames] + image_dims + [1], dev_str=dev_str)
if ones:
img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.ones(
[batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1)
img_var = ivy.ones(
[batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str)*1e-3
pose_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str)
pose_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3
else:
img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.random_uniform(
1e-3, 1, [batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1)
img_var = ivy.random_uniform(
1e-3, 1, [batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str)
pose_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str)
pose_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str)
if empty:
img_var = ivy.ones_like(img_var) * 1e12
validity_mask = ivy.zeros_like(validity_mask)
img_meas['dummy_cam_{}'.format(i)] =\
{'img_mean': img_mean,
'img_var': img_var,
'validity_mask': validity_mask,
'pose_mean': pose_mean,
'pose_cov': pose_cov,
'cam_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames], dev_str=dev_str)[..., 0:3, :]}
if ones:
control_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str)
control_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3
else:
control_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str)
control_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str)
return Container({'img_meas': img_meas,
'control_mean': control_mean,
'control_cov': control_cov,
'agent_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames],
dev_str=dev_str)[..., 0:3, :]})
def test_module_w_none_attribute(bs_ic_oc, dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels = bs_ic_oc
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
module = ModuleWithNoneAttribute()
def _create_variables(self, dev_str):
"""
Create internal variables for the Linear layer
"""
# ToDo: support other initialization mechanisms, via class constructor options
# ToDo: tidy the construction of these variables, with helper functions
wlim = (6 / (self._output_channels + self._input_channels))**0.5
w = ivy.variable(
ivy.random_uniform(-wlim,
wlim,
(self._output_channels, self._input_channels),
dev_str=dev_str))
b = ivy.variable(ivy.zeros([self._output_channels], dev_str=dev_str))
return {'w': w, 'b': b}
def _addressing(self, k, beta, g, s, gamma, prev_M, prev_w):
# Sec 3.3.1 Focusing by Content
# Cosine Similarity
k = ivy.expand_dims(k, axis=2)
inner_product = ivy.matmul(prev_M, k)
k_norm = ivy.reduce_sum(k**2, axis=1, keepdims=True)**0.5
M_norm = ivy.reduce_sum(prev_M**2, axis=2, keepdims=True)**0.5
norm_product = M_norm * k_norm
K = ivy.squeeze(inner_product / (norm_product + 1e-8)) # eq (6)
# Calculating w^c
K_amplified = ivy.exp(ivy.expand_dims(beta, axis=1) * K)
w_c = K_amplified / ivy.reduce_sum(K_amplified, axis=1,
keepdims=True) # eq (5)
if self._addressing_mode == 'content': # Only focus on content
return w_c
# Sec 3.3.2 Focusing by Location
g = ivy.expand_dims(g, axis=1)
w_g = g * w_c + (1 - g) * prev_w # eq (7)
s = ivy.concatenate([
s[:, :self._shift_range + 1],
ivy.zeros(
[s.shape[0], self._memory_size -
(self._shift_range * 2 + 1)]), s[:, -self._shift_range:]
],
axis=1)
t = ivy.concatenate([ivy.flip(s, axis=[1]),
ivy.flip(s, axis=[1])],
axis=1)
s_matrix = ivy.stack([
t[:, self._memory_size - i - 1:self._memory_size * 2 - i - 1]
for i in range(self._memory_size)
],
axis=1)
w_ = ivy.reduce_sum(ivy.expand_dims(w_g, axis=1) * s_matrix,
axis=2) # eq (8)
w_sharpen = w_**ivy.expand_dims(gamma, axis=1)
w = w_sharpen / ivy.reduce_sum(w_sharpen, axis=1,
keepdims=True) # eq (9)
return w
def as_identity(batch_shape):
"""
Return camera intrinsics object with array attributes as either zeros or identity matrices.
:param batch_shape: Batch shape for each geometric array attribute
:type batch_shape: sequence of ints
:return: New camera intrinsics object, with each entry as either zeros or identity matrices.
"""
batch_shape = list(batch_shape)
focal_lengths = _ivy.ones(batch_shape + [2])
persp_angles = _ivy.ones(batch_shape + [2])
pp_offsets = _ivy.zeros(batch_shape + [2])
calib_mats = _ivy.identity(3, batch_shape=batch_shape)
inv_calib_mats = _ivy.identity(3, batch_shape=batch_shape)
return __class__(focal_lengths, persp_angles, pp_offsets, calib_mats,
inv_calib_mats)
def as_identity(batch_shape):
"""
Return camera extrinsics object with array attributes as either zeros or identity matrices.
:param batch_shape: Batch shape for each geometric array attribute.
:type batch_shape: sequence of ints
:return: New camera extrinsics object, with each entry as either zeros or identity matrices.
"""
batch_shape = list(batch_shape)
cam_centers = _ivy.zeros(batch_shape + [3, 1])
Rs = _ivy.identity(3, batch_shape=batch_shape)
inv_Rs = _ivy.identity(3, batch_shape=batch_shape)
ext_mats_homo = _ivy.identity(4, batch_shape=batch_shape)
inv_ext_mats_homo = _ivy.identity(4, batch_shape=batch_shape)
return __class__(cam_centers, Rs, inv_Rs, ext_mats_homo,
inv_ext_mats_homo)
def test_module_training(bs_ic_oc, dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
batch_shape, input_channels, output_channels = bs_ic_oc
x = ivy.cast(
ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape),
input_channels), 'float32')
module = TrainableModule(input_channels, output_channels)
def loss_fn(v_):
out = module(x, v=v_)
return ivy.reduce_mean(out)[0]
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, module.v)
module.v = ivy.gradient_descent_update(module.v, grads, 1e-3)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
assert ivy.reduce_max(ivy.abs(grads.linear0.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear0.w)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear1.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear1.w)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear2.b)) > 0
assert ivy.reduce_max(ivy.abs(grads.linear2.w)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not support **kwargs
return
helpers.assert_compilable(loss_fn)
def __init__(self, controller, controller_proj, output_proj, output_dim, ctrl_input_size, ctrl_output_size,
total_parameter_num, memory_size, memory_vector_dim, read_head_num, write_head_num, v=None, usage=None,
addressing_mode='content_and_location', shift_range=1, clip_value=20, init_value=1e-6,
sequential_writing=False, retroactive_updates=False, retroactive_discount=0.96, with_erase=True,
seed=0):
# vanilla ntm
self._memory_size = memory_size
self._memory_vector_dim = memory_vector_dim
self._read_head_num = read_head_num
self._write_head_num = write_head_num
self._init_value = init_value
self._addressing_mode = addressing_mode
self._clip_value = clip_value
self._output_dim = output_dim
self._shift_range = shift_range
self._num_parameters_per_head = self._memory_vector_dim + 1 + 1 + (self._shift_range * 2 + 1) + 1
self._num_heads = self._read_head_num + self._write_head_num
ivy.seed(seed)
# fns + classes
self._controller = controller
self._controller_proj = controller_proj
self._output_proj = output_proj
# usage
if usage is not None:
self._usage = usage
else:
self._usage = ivy.zeros([memory_size, ])
# step
self._step = 0
# MERLIN changes
self._sequential_writing = sequential_writing
self._retroactive_updates = retroactive_updates
self._retroactive_discount = retroactive_discount
self._with_erase = with_erase
# variables
ivy.Module.__init__(self, 'cpu')
def focal_lengths_and_pp_offsets_to_calib_mat(focal_lengths, pp_offsets, batch_shape=None, dev_str=None):
"""
Compute calibration matrix :math:`\mathbf{K}\in\mathbb{R}^{3×3}` from focal lengths :math:`f_x, f_y` and
principal-point offsets :math:`p_x, p_y`.\n
`[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=173>`_
page 155, section 6.1, equation 6.4
:param focal_lengths: Focal lengths *[batch_shape,2]*
:type focal_lengths: array
:param pp_offsets: Principal-point offsets *[batch_shape,2]*
:type pp_offsets: array
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Calibration matrix *[batch_shape,3,3]*
"""
if batch_shape is None:
batch_shape = focal_lengths.shape[:-1]
if dev_str is None:
dev_str = _ivy.dev_str(focal_lengths)
# shapes as list
batch_shape = list(batch_shape)
# BS x 1 x 1
zeros = _ivy.zeros(batch_shape + [1, 1], dev_str=dev_str)
ones = _ivy.ones(batch_shape + [1, 1], dev_str=dev_str)
# BS x 2 x 1
focal_lengths_reshaped = _ivy.expand_dims(focal_lengths, -1)
pp_offsets_reshaped = _ivy.expand_dims(pp_offsets, -1)
# BS x 1 x 3
row1 = _ivy.concatenate((focal_lengths_reshaped[..., 0:1, :], zeros, pp_offsets_reshaped[..., 0:1, :]), -1)
row2 = _ivy.concatenate((zeros, focal_lengths_reshaped[..., 1:2, :], pp_offsets_reshaped[..., 1:2, :]), -1)
row3 = _ivy.concatenate((zeros, zeros, ones), -1)
# BS x 3 x 3
return _ivy.concatenate((row1, row2, row3), -2)
def __init__(self, base_inv_ext_mat=None):
"""
Initialize FRANKA EMIKA Panda robot manipulator instance.
Denavit–Hartenberg parameters inferred from FRANKA EMIKA online API.
Screenshot included in this module for reference.
:param base_inv_ext_mat: Inverse extrinsic matrix of the robot base *[3,4]*
:type base_inv_ext_mat: array, optional
"""
# dh params
# panda_DH_params.png
# taken from https://frankaemika.github.io/docs/control_parameters.html
a_s = _ivy.array([0., 0., 0.0825, -0.0825, 0., 0.088, 0.])
d_s = _ivy.array([0.333, 0., 0.316, 0., 0.384, 0., 0.107])
alpha_s = _ivy.array(
[-_math.pi / 2, _math.pi / 2, _math.pi / 2, -_math.pi / 2, _math.pi / 2, _math.pi / 2, 0.])
dh_joint_scales = _ivy.ones((7,))
dh_joint_offsets = _ivy.zeros((7,))
super().__init__(a_s, d_s, alpha_s, dh_joint_scales, dh_joint_offsets, base_inv_ext_mat)
def __init__(self, data_loader_spec):
super().__init__(data_loader_spec)
# dataset size
self._num_examples = self._spec.dataset_spec.num_examples
# counter
self._i = 0
# load vector data
vector_dim = self._spec.dataset_spec.vector_dim
self._targets = ivy.zeros((self._num_examples, vector_dim, 1))
# load image data
image_dims = self._spec.dataset_spec.image_dims
self._input = ivy.ones(
(self._num_examples, image_dims[0], image_dims[1], 3))
self._training_data = ivy.Container(targets=self._targets,
input=self._input)
self._validation_data = ivy.Container(targets=self._targets,
input=self._input)
self._data = ivy.Container(training=self._training_data,
validation=self._validation_data)
def quantize_to_image(pixel_coords,
final_image_dims,
feat=None,
feat_prior=None,
with_db=False,
pixel_coords_var=1e-3,
feat_var=1e-3,
pixel_coords_prior_var=1e12,
feat_prior_var=1e12,
var_threshold=(1e-3, 1e12),
uniform_pixel_coords=None,
batch_shape=None,
dev_str=None):
"""
Quantize pixel co-ordinates with d feature channels (for depth, rgb, normals etc.), from
images :math:`\mathbf{X}\in\mathbb{R}^{input\_images\_shape×(2+d)}`, which may have been reprojected from a host of
different cameras (leading to non-integer pixel values), to a new quantized pixel co-ordinate image with the same
feature channels :math:`\mathbf{X}\in\mathbb{R}^{h×w×(2+d)}`, and with integer pixel co-ordinates.
Duplicates during the quantization are either probabilistically fused based on variance, or the minimum depth is
chosen when using depth buffer mode.
:param pixel_coords: Pixel co-ordinates *[batch_shape,input_size,2]*
:type pixel_coords: array
:param final_image_dims: Image dimensions of the final image.
:type final_image_dims: sequence of ints
:param feat: Features (i.e. depth, rgb, encoded), default is None. *[batch_shape,input_size,d]*
:type feat: array, optional
:param feat_prior: Prior feature image mean, default is None. *[batch_shape,input_size,d]*
:type feat_prior: array or float to fill with
:param with_db: Whether or not to use depth buffer in rendering, default is false
:type with_db: bool, optional
:param pixel_coords_var: Pixel coords variance *[batch_shape,input_size,2]*
:type pixel_coords_var: array or float to fill with
:param feat_var: Feature variance *[batch_shape,input_size,d]*
:type feat_var: array or float to fill with
:param pixel_coords_prior_var: Pixel coords prior variance *[batch_shape,h,w,2]*
:type pixel_coords_prior_var: array or float to fill with
:param feat_prior_var: Features prior variance *[batch_shape,h,w,3]*
:type feat_prior_var: array or float to fill with
:param var_threshold: Variance threshold, for projecting valid coords and clipping *[batch_shape,2+d,2]*
:type var_threshold: array or sequence of floats to fill with
:param uniform_pixel_coords: Homogeneous uniform (integer) pixel co-ordinate images, inferred from final_image_dims
if None *[batch_shape,h,w,3]*
:type uniform_pixel_coords: array, optional
:param batch_shape: Shape of batch. Assumed no batches if None.
:type batch_shape: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Quantized pixel co-ordinates image with d feature channels (for depth, rgb, normals etc.) *[batch_shape,h,w,2+d]*,
maybe the quantized variance, *[batch_shape,h,w,2+d]*, and scatter counter image *[batch_shape,h,w,1]*
"""
# ToDo: make variance fully optional. If not specified,
# then do not compute and scatter during function call for better efficiency.
# config
if batch_shape is None:
batch_shape = pixel_coords.shape[:-2]
if dev_str is None:
dev_str = _ivy.dev_str(pixel_coords)
if feat is None:
d = 0
else:
d = feat.shape[-1]
min_depth_diff = _ivy.array([MIN_DEPTH_DIFF], dev_str=dev_str)
red = 'min' if with_db else 'sum'
# shapes as list
batch_shape = list(batch_shape)
final_image_dims = list(final_image_dims)
num_batch_dims = len(batch_shape)
# variance threshold
if isinstance(var_threshold, tuple) or isinstance(var_threshold, list):
ones = _ivy.ones(batch_shape + [1, 2 + d, 1])
var_threshold = _ivy.concatenate(
(ones * var_threshold[0], ones * var_threshold[1]), -1)
else:
var_threshold = _ivy.reshape(var_threshold,
batch_shape + [1, 2 + d, 2])
# uniform pixel coords
if uniform_pixel_coords is None:
uniform_pixel_coords =\
_ivy_svg.create_uniform_pixel_coords_image(final_image_dims, batch_shape, dev_str=dev_str)
uniform_pixel_coords = uniform_pixel_coords[..., 0:2]
# Extract Values #
feat_prior = _ivy.ones_like(feat) * feat_prior if isinstance(
feat_prior, float) else feat_prior
pixel_coords_var = _ivy.ones_like(pixel_coords) * pixel_coords_var\
if isinstance(pixel_coords_var, float) else pixel_coords_var
feat_var = _ivy.ones_like(feat) * feat_var if isinstance(
feat_var, float) else feat_var
pixel_coords_prior_var = _ivy.ones(batch_shape + final_image_dims + [2]) * pixel_coords_prior_var\
if isinstance(pixel_coords_prior_var, float) else pixel_coords_prior_var
feat_prior_var = _ivy.ones(batch_shape + final_image_dims + [d]) * feat_prior_var\
if isinstance(feat_prior_var, float) else feat_prior_var
# Quantize #
# BS x N x 2
quantized_pixel_coords = _ivy.reshape(
_ivy.cast(_ivy.round(pixel_coords), 'int32'), batch_shape + [-1, 2])
# Combine #
# BS x N x (2+D)
pc_n_feat = _ivy.reshape(_ivy.concatenate((pixel_coords, feat), -1),
batch_shape + [-1, 2 + d])
pc_n_feat_var = _ivy.reshape(
_ivy.concatenate((pixel_coords_var, feat_var), -1),
batch_shape + [-1, 2 + d])
# BS x H x W x (2+D)
prior = _ivy.concatenate((uniform_pixel_coords, feat_prior), -1)
prior_var = _ivy.concatenate((pixel_coords_prior_var, feat_prior_var), -1)
# Validity Mask #
# BS x N x 1
var_validity_mask = \
_ivy.reduce_sum(_ivy.cast(pc_n_feat_var < var_threshold[..., 1], 'int32'), -1, keepdims=True) == 2+d
bounds_validity_mask = _ivy.logical_and(
_ivy.logical_and(quantized_pixel_coords[..., 0:1] >= 0,
quantized_pixel_coords[..., 1:2] >= 0),
_ivy.logical_and(
quantized_pixel_coords[..., 0:1] <= final_image_dims[1] - 1,
quantized_pixel_coords[..., 1:2] <= final_image_dims[0] - 1))
validity_mask = _ivy.logical_and(var_validity_mask, bounds_validity_mask)
# num_valid_indices x len(BS)+2
validity_indices = _ivy.reshape(
_ivy.cast(_ivy.indices_where(validity_mask), 'int32'),
[-1, num_batch_dims + 2])
num_valid_indices = validity_indices.shape[-2]
if num_valid_indices == 0:
return _ivy.concatenate((uniform_pixel_coords[..., 0:2], feat_prior), -1), \
_ivy.concatenate((pixel_coords_prior_var, feat_prior_var), -1),\
_ivy.zeros_like(feat[..., 0:1], dev_str=dev_str)
# Depth Based Scaling #
mean_depth_min = None
mean_depth_range = None
pc_n_feat_wo_depth_range = None
pc_n_feat_wo_depth_min = None
var_vals_range = None
var_vals_min = None
if with_db:
# BS x N x 1
mean_depth = pc_n_feat[..., 2:3]
# BS x 1 x 1
mean_depth_min = _ivy.reduce_min(mean_depth, -2, keepdims=True)
mean_depth_max = _ivy.reduce_max(mean_depth, -2, keepdims=True)
mean_depth_range = mean_depth_max - mean_depth_min
# BS x N x 1
scaled_depth = (mean_depth - mean_depth_min) / (
mean_depth_range * min_depth_diff + MIN_DENOMINATOR)
if d == 1:
# BS x 1 x 1+D
pc_n_feat_wo_depth_min = _ivy.zeros(batch_shape + [1, 0],
dev_str=dev_str)
pc_n_feat_wo_depth_range = _ivy.ones(batch_shape + [1, 0],
dev_str=dev_str)
else:
# feat without depth
# BS x N x 1+D
pc_n_feat_wo_depth = _ivy.concatenate(
(pc_n_feat[..., 0:2], pc_n_feat[..., 3:]), -1)
# find the min and max of each value
# BS x 1 x 1+D
pc_n_feat_wo_depth_max = _ivy.reduce_max(
pc_n_feat_wo_depth, -2, keepdims=True) + 1
pc_n_feat_wo_depth_min = _ivy.reduce_min(
pc_n_feat_wo_depth, -2, keepdims=True) - 1
pc_n_feat_wo_depth_range = pc_n_feat_wo_depth_max - pc_n_feat_wo_depth_min
# BS x N x 1+D
normed_pc_n_feat_wo_depth = (pc_n_feat_wo_depth - pc_n_feat_wo_depth_min) / \
(pc_n_feat_wo_depth_range + MIN_DENOMINATOR)
# combine with scaled depth
# BS x N x 1+D
pc_n_feat_wo_depth_scaled = normed_pc_n_feat_wo_depth + scaled_depth
# BS x N x (2+D)
pc_n_feat = _ivy.concatenate(
(pc_n_feat_wo_depth_scaled[..., 0:2], mean_depth,
pc_n_feat_wo_depth_scaled[..., 2:]), -1)
# scale variance
# BS x 1 x (2+D)
var_vals_max = _ivy.reduce_max(pc_n_feat_var, -2, keepdims=True) + 1
var_vals_min = _ivy.reduce_min(pc_n_feat_var, -2, keepdims=True) - 1
var_vals_range = var_vals_max - var_vals_min
# BS x N x (2+D)
normed_var_vals = (pc_n_feat_var - var_vals_min) / (var_vals_range +
MIN_DENOMINATOR)
pc_n_feat_var = normed_var_vals + scaled_depth
# ready for later reversal with full image dimensions
# BS x 1 x 1 x D
var_vals_min = _ivy.expand_dims(var_vals_min, -2)
var_vals_range = _ivy.expand_dims(var_vals_range, -2)
# Validity Pruning #
# num_valid_indices x (2+D)
pc_n_feat = _ivy.gather_nd(pc_n_feat,
validity_indices[..., 0:num_batch_dims + 1])
pc_n_feat_var = _ivy.gather_nd(pc_n_feat_var,
validity_indices[..., 0:num_batch_dims + 1])
# num_valid_indices x 2
quantized_pixel_coords = _ivy.gather_nd(
quantized_pixel_coords, validity_indices[..., 0:num_batch_dims + 1])
if with_db:
means_to_scatter = pc_n_feat
vars_to_scatter = pc_n_feat_var
else:
# num_valid_indices x (2+D)
vars_to_scatter = 1 / (pc_n_feat_var + MIN_DENOMINATOR)
means_to_scatter = pc_n_feat * vars_to_scatter
# Scatter #
# num_valid_indices x 1
counter = _ivy.ones_like(pc_n_feat[..., 0:1], dev_str=dev_str)
if with_db:
counter *= -1
# num_valid_indices x 2(2+D)+1
values_to_scatter = _ivy.concatenate(
(means_to_scatter, vars_to_scatter, counter), -1)
# num_valid_indices x (num_batch_dims + 2)
all_indices = _ivy.flip(quantized_pixel_coords, -1)
if num_batch_dims > 0:
all_indices = _ivy.concatenate(
(validity_indices[..., :-2], all_indices), -1)
# BS x H x W x (2(2+D) + 1)
quantized_img = _ivy.scatter_nd(
_ivy.reshape(all_indices, [-1, num_batch_dims + 2]),
_ivy.reshape(values_to_scatter, [-1, 2 * (2 + d) + 1]),
batch_shape + final_image_dims + [2 * (2 + d) + 1],
reduction='replace' if _ivy.backend == 'mxnd' else red)
# BS x H x W x 1
quantized_counter = quantized_img[..., -1:]
if with_db:
invalidity_mask = quantized_counter != -1
else:
invalidity_mask = quantized_counter == 0
if with_db:
# BS x H x W x (2+D)
quantized_mean_scaled = quantized_img[..., 0:2 + d]
quantized_var_scaled = quantized_img[..., 2 + d:2 * (2 + d)]
# BS x H x W x 1
quantized_depth_mean = quantized_mean_scaled[..., 2:3]
# BS x 1 x 1 x 1
mean_depth_min = _ivy.expand_dims(mean_depth_min, -2)
mean_depth_range = _ivy.expand_dims(mean_depth_range, -2)
# BS x 1 x 1 x (1+D)
pc_n_feat_wo_depth_min = _ivy.expand_dims(pc_n_feat_wo_depth_min, -2)
pc_n_feat_wo_depth_range = _ivy.expand_dims(pc_n_feat_wo_depth_range,
-2)
# BS x 1 x 1 x (2+D) x 2
var_threshold = _ivy.expand_dims(var_threshold, -3)
# BS x H x W x (1+D)
quantized_mean_wo_depth_scaled = _ivy.concatenate(
(quantized_mean_scaled[..., 0:2], quantized_mean_scaled[..., 3:]),
-1)
quantized_mean_wo_depth_normed = quantized_mean_wo_depth_scaled - (quantized_depth_mean - mean_depth_min) / \
(mean_depth_range * min_depth_diff + MIN_DENOMINATOR)
quantized_mean_wo_depth = quantized_mean_wo_depth_normed * pc_n_feat_wo_depth_range + pc_n_feat_wo_depth_min
prior_wo_depth = _ivy.concatenate((prior[..., 0:2], prior[..., 3:]),
-1)
quantized_mean_wo_depth = _ivy.where(invalidity_mask, prior_wo_depth,
quantized_mean_wo_depth)
# BS x H x W x (2+D)
quantized_mean = _ivy.concatenate(
(quantized_mean_wo_depth[..., 0:2], quantized_depth_mean,
quantized_mean_wo_depth[..., 2:]), -1)
# BS x H x W x (2+D)
quantized_var_normed = quantized_var_scaled - (quantized_depth_mean - mean_depth_min) / \
(mean_depth_range * min_depth_diff + MIN_DENOMINATOR)
quantized_var = _ivy.maximum(
quantized_var_normed * var_vals_range + var_vals_min,
var_threshold[..., 0])
quantized_var = _ivy.where(invalidity_mask, prior_var, quantized_var)
else:
# BS x H x W x (2+D)
quantized_sum_mean_x_recip_var = quantized_img[..., 0:2 + d]
quantized_var_wo_increase = _ivy.where(
invalidity_mask, prior_var,
(1 / (quantized_img[..., 2 + d:2 * (2 + d)] + MIN_DENOMINATOR)))
quantized_var = _ivy.maximum(
quantized_var_wo_increase * quantized_counter,
_ivy.expand_dims(var_threshold[..., 0], -2))
quantized_var = _ivy.where(invalidity_mask, prior_var, quantized_var)
quantized_mean = _ivy.where(
invalidity_mask, prior,
quantized_var_wo_increase * quantized_sum_mean_x_recip_var)
# BS x H x W x (2+D) BS x H x W x (2+D) BS x H x W x 1
return quantized_mean, quantized_var, quantized_counter
def reset(self):
self.urchin_xys = ivy.random_uniform(-1, 1, (self.num_urchins, 2))
self.xy = ivy.random_uniform(-1, 1, (2, ))
self.xy_vel = ivy.zeros((2, ))
self.goal_xy = ivy.random_uniform(-1, 1, (2, ))
return self.get_observation()
def main(interactive=True, try_use_sim=True, f=None):
f = choose_random_framework() if f is None else f
set_framework(f)
with_mxnd = f is ivy.mxnd
if with_mxnd:
print('\nMXnet does not support "sum" or "min" reductions for scatter_nd,\n'
'instead it only supports non-deterministic replacement for duplicates.\n'
'Depth buffer rendering (requies min) or fusion buffer (requies sum) are therefore unsupported.\n'
'The rendering in this demo with MXNet backend exhibits non-deterministic jagged edges as a result.')
sim = Simulator(interactive, try_use_sim)
import matplotlib.pyplot as plt
xyzs = list()
rgbs = list()
iterations = 10 if sim.with_pyrep else 1
for _ in range(iterations):
for cam in sim.cams:
depth, rgb = cam.cap()
xyz = sim.depth_to_xyz(depth, cam.get_inv_ext_mat(), cam.inv_calib_mat, [1024]*2)
xyzs.append(xyz)
rgbs.append(rgb)
xyz = ivy.reshape(ivy.concatenate(xyzs, 1), (-1, 3))
rgb = ivy.reshape(ivy.concatenate(rgbs, 1), (-1, 3))
cam_coords = ivy_vision.world_to_cam_coords(ivy_mech.make_coordinates_homogeneous(ivy.expand_dims(xyz, 1)),
sim.target_cam.get_ext_mat())
ds_pix_coords = ivy_vision.cam_to_ds_pixel_coords(cam_coords, sim.target_cam.calib_mat)
depth = ds_pix_coords[..., -1]
pix_coords = ds_pix_coords[..., 0, 0:2] / depth
final_image_dims = [512]*2
feat = ivy.concatenate((depth, rgb), -1)
rendered_img_no_db, _, _ = ivy_vision.quantize_to_image(
pix_coords, final_image_dims, feat, ivy.zeros(final_image_dims + [4]), with_db=False)
with_db = not with_mxnd
rendered_img_with_db, _, _ = ivy_vision.quantize_to_image(
pix_coords, final_image_dims, feat, ivy.zeros(final_image_dims + [4]), with_db=with_db)
a_img = cv2.resize(ivy.to_numpy(rgbs[0]), (256, 256))
a_img[0:50, 0:50] = np.zeros_like(a_img[0:50, 0:50])
a_img[5:45, 5:45] = np.ones_like(a_img[5:45, 5:45])
cv2.putText(a_img, 'a', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
b_img = cv2.resize(ivy.to_numpy(rgbs[1]), (256, 256))
b_img[0:50, 0:50] = np.zeros_like(b_img[0:50, 0:50])
b_img[5:45, 5:45] = np.ones_like(b_img[5:45, 5:45])
cv2.putText(b_img, 'b', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
c_img = cv2.resize(ivy.to_numpy(rgbs[2]), (256, 256))
c_img[0:50, 0:50] = np.zeros_like(c_img[0:50, 0:50])
c_img[5:45, 5:45] = np.ones_like(c_img[5:45, 5:45])
cv2.putText(c_img, 'c', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
target_img = cv2.resize(ivy.to_numpy(sim.target_cam.cap()[1]), (256, 256))
target_img[0:50, 0:140] = np.zeros_like(target_img[0:50, 0:140])
target_img[5:45, 5:135] = np.ones_like(target_img[5:45, 5:135])
cv2.putText(target_img, 'target', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
msg = 'non-deterministic' if with_mxnd else 'no depth buffer'
width = 360 if with_mxnd else 320
no_db_img = np.copy(ivy.to_numpy(rendered_img_no_db[..., 3:]))
no_db_img[0:50, 0:width+5] = np.zeros_like(no_db_img[0:50, 0:width+5])
no_db_img[5:45, 5:width] = np.ones_like(no_db_img[5:45, 5:width])
cv2.putText(no_db_img, msg, (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
with_db_img = np.copy(ivy.to_numpy(rendered_img_with_db[..., 3:]))
with_db_img[0:50, 0:350] = np.zeros_like(with_db_img[0:50, 0:350])
with_db_img[5:45, 5:345] = np.ones_like(with_db_img[5:45, 5:345])
cv2.putText(with_db_img, 'with depth buffer', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
raw_imgs = np.concatenate((np.concatenate((a_img, b_img), 1),
np.concatenate((c_img, target_img), 1)), 0)
to_concat = (raw_imgs, no_db_img) if with_mxnd else (raw_imgs, no_db_img, with_db_img)
final_img = np.concatenate(to_concat, 1)
if interactive:
print('\nClose the image window when you are ready.\n')
plt.imshow(final_img)
plt.show()
xyzs.clear()
rgbs.clear()
sim.close()
unset_framework()
def test_lstm_layer_training(b_t_ic_hc_otf_sctv, with_v, dtype_str, tensor_fn,
dev_str, call):
# smoke test
if call is helpers.np_call:
# NumPy does not support gradients
pytest.skip()
# smoke test
b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv
x = ivy.cast(
ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels),
'float32')
if with_v:
kernel = ivy.variable(
ivy.ones([input_channels, 4 * hidden_channels]) * 0.5)
recurrent_kernel = ivy.variable(
ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5)
v = Container({
'input': {
'layer_0': {
'w': kernel
}
},
'recurrent': {
'layer_0': {
'w': recurrent_kernel
}
}
})
else:
v = None
lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v)
def loss_fn(v_):
out, (state_h, state_c) = lstm_layer(x, v=v_)
return ivy.reduce_mean(out)[0]
# train
loss_tm1 = 1e12
loss = None
grads = None
for i in range(10):
loss, grads = ivy.execute_with_gradients(loss_fn, lstm_layer.v)
lstm_layer.v = ivy.gradient_descent_update(lstm_layer.v, grads, 1e-3)
assert loss < loss_tm1
loss_tm1 = loss
# type test
assert ivy.is_array(loss)
assert isinstance(grads, ivy.Container)
# cardinality test
if call is helpers.mx_call:
# mxnet slicing cannot reduce dimension to zero
assert loss.shape == (1, )
else:
assert loss.shape == ()
# value test
for key, val in grads.to_iterator():
assert ivy.reduce_max(ivy.abs(val)) > 0
# compilation test
if call is helpers.torch_call:
# pytest scripting does not **kwargs
return
helpers.assert_compilable(loss_fn)
def test_ntm(addressing_mode, batch_shape, dev_str, call):
# ntm config
input_dim = 256
output_dim = 8
ctrl_output_size = 256
ctrl_layers = 2
memory_size = 5
timesteps = 5
memory_vector_dim = 2
read_head_num = 3
write_head_num = 1
shift_range = 0
clip_value = 20
init_value = 1e-6
ctrl_input_size = read_head_num * memory_vector_dim + input_dim
num_heads = read_head_num + write_head_num
num_parameters_per_head = memory_vector_dim + 1 + 1 + (shift_range * 2 +
1) + 1
total_parameter_num = num_parameters_per_head * num_heads + memory_vector_dim * 2 * write_head_num
usage = ivy.zeros([
memory_size,
])
# memory object wo vars
ntm = ivy_mem.NTM(input_dim,
output_dim,
ctrl_output_size,
ctrl_layers,
memory_size,
memory_vector_dim,
read_head_num,
write_head_num,
addressing_mode=addressing_mode,
shift_range=shift_range,
clip_value=clip_value,
sequential_writing=True,
retroactive_updates=False,
with_erase=False)
# test
x = ivy.ones(batch_shape + [timesteps, input_dim])
assert call(ntm, x).shape == tuple(batch_shape + [timesteps, output_dim])
# variables
variables = dict()
variables['ntm_cell'] = dict()
np.random.seed(0)
# lstm
in_wlim = (6 / (ctrl_input_size + 4 * ctrl_output_size))**0.5
rec_wlim = (6 / (ctrl_output_size + 4 * ctrl_output_size))**0.5
variables['ntm_cell']['controller'] = \
{'input': {'layer1': {'w': ivy.array(np.random.uniform(
-in_wlim, in_wlim, size=[ctrl_input_size, 4 * ctrl_output_size]).astype(np.float32))},
'layer2': {'w': ivy.array(np.random.uniform(
-in_wlim, in_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype(np.float32))}},
'recurrent': {'layer1': {'w': ivy.array(np.random.uniform(
-rec_wlim, rec_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype(np.float32))},
'layer2': {'w': ivy.array(np.random.uniform(
-rec_wlim, rec_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype(
np.float32))}}}
# fully connected
proj_wlim = (6 / (total_parameter_num + ctrl_output_size))**0.5
variables['ntm_cell']['controller_proj'] = {
'w':
ivy.array(
np.random.uniform(-proj_wlim,
proj_wlim,
size=[total_parameter_num,
ctrl_output_size]).astype(np.float32)),
'b':
ivy.zeros([total_parameter_num])
}
out_wlim = (6 / (total_parameter_num + ctrl_input_size))**0.5
variables['ntm_cell']['output_proj'] = {
'w':
ivy.array(
np.random.uniform(-out_wlim,
out_wlim,
size=[
output_dim, ctrl_output_size +
read_head_num * memory_vector_dim
]).astype(np.float32)),
'b':
ivy.zeros([output_dim])
}
# memory
wlim = (6 / (2 * memory_vector_dim))**0.5
variables['ntm_cell']['read_weights'] = dict(
zip(['w_' + str(i) for i in range(read_head_num)], [
ivy.variable(
ivy.array(
np.random.uniform(-wlim, wlim, [
memory_vector_dim,
]), 'float32')) for _ in range(read_head_num)
]))
wlim = (6 / (2 * memory_size))**0.5
variables['ntm_cell']['write_weights'] = dict(
zip(['w_' + str(i) for i in range(read_head_num + write_head_num)], [
ivy.variable(
ivy.array(np.random.uniform(-wlim, wlim, [
memory_size,
]), 'float32')) for _ in range(read_head_num + write_head_num)
]))
variables['ntm_cell']['memory'] = ivy.variable(
ivy.ones([memory_size, memory_vector_dim]) * init_value)
# memory object w vars
ntm = ivy_mem.NTM(input_dim,
output_dim,
ctrl_output_size,
ctrl_layers,
memory_size,
memory_vector_dim,
read_head_num,
write_head_num,
Container(variables),
usage,
addressing_mode=addressing_mode,
shift_range=shift_range,
clip_value=clip_value,
init_value=init_value,
sequential_writing=True,
retroactive_updates=False,
with_erase=False)
# test
assert np.allclose(call(ntm, x), td.ntm_return[addressing_mode], atol=1e-6)
# compilation test
if call is helpers.torch_call:
# pytest scripting does not support try-catch statements
return
helpers.assert_compilable(ntm)
def reset(self):
self.x = ivy.random_uniform(-0.9, -0.2, [1])
self.x_vel = ivy.zeros([1])
return self.get_observation()
def __init__(self, a_s, d_s, alpha_s, dh_joint_scales, dh_joint_offsets, base_inv_ext_mat=None):
"""
Initialize robot manipulator instance
:param a_s: Denavit–Hartenberg "a" parameters *[num_joints]*
:type a_s: array
:param d_s: Denavit–Hartenberg "d" parameters *[num_joints]*
:type d_s: array
:param alpha_s: Denavit–Hartenberg "alpha" parameters *[num_joints]*
:type alpha_s: array
:param dh_joint_scales: Scalars to apply to input joints *[num_joints]*
:type dh_joint_scales: array
:param dh_joint_offsets: Scalar offsets to apply to input joints *[num_joints]*
:type dh_joint_offsets: array
:param base_inv_ext_mat: Inverse extrinsic matrix of the robot base *[4,4]*
:type base_inv_ext_mat: array, optional
"""
self._num_joints = a_s.shape[-1]
# ToDo: incorporate the base_inv_ext_mat more elegantly, instead of the hack as in the sample_links method
if base_inv_ext_mat is None:
self._base_inv_ext_mat = _ivy.identity(4)
else:
self._base_inv_ext_mat = base_inv_ext_mat
# NJ
self._dis = d_s
self._dh_joint_scales = dh_joint_scales
self._dh_joint_offsets = dh_joint_offsets
# Forward Kinematics Constant Matrices
# Based on Denavit-Hartenberg Convention
# Using same nomenclature as in:
# Modelling, Planning and Control. Bruno Siciliano, Lorenzo Sciavicco, Luigi Villani, Giuseppe Oriolo
# page 61 - 65
AidashtoAis_list = [_ivy.identity(4, batch_shape=[1])]
# repeated blocks
# 1 x 1 x 3
start_of_top_row = _ivy.array([[[1., 0., 0.]]])
# 1 x 1 x 1
zeros = _ivy.zeros((1, 1, 1))
# 1 x 1 x 4
bottom_row = _ivy.array([[[0., 0., 0., 1.]]])
for i in range(self._num_joints):
# 1 x 1 x 1
a_i = _ivy.reshape(a_s[i], [1] * 3)
cos_alpha = _ivy.reshape(_ivy.cos(alpha_s[i]), [1] * 3)
sin_alpha = _ivy.reshape(_ivy.sin(alpha_s[i]), [1] * 3)
# 1 x 1 x 4
top_row = _ivy.concatenate((start_of_top_row, a_i), -1)
top_middle_row = _ivy.concatenate((zeros, cos_alpha, -sin_alpha, zeros), -1)
bottom_middle_row = _ivy.concatenate((zeros, sin_alpha, cos_alpha, zeros), -1)
# 1 x 4 x 4
AidashtoAi = _ivy.concatenate((top_row, top_middle_row, bottom_middle_row, bottom_row), 1)
# list
AidashtoAis_list.append(AidashtoAi)
# NJ x 4 x 4
self._AidashtoAis = _ivy.concatenate(AidashtoAis_list, 0)
# Constant Jacobian Params
# Using same nomenclature as in:
# Modelling, Planning and Control. Bruno Siciliano, Lorenzo Sciavicco, Luigi Villani, Giuseppe Oriolo
# page 111 - 113
# 1 x 3
self._z0 = _ivy.array([[0],
[0],
[1]])
# 1 x 4
self._p0hat = _ivy.array([[0],
[0],
[0],
[1]])
# link lengths
# NJ
self._link_lengths = (a_s ** 2 + d_s ** 2) ** 0.5
def depth_from_flow_and_cam_mats(flow,
full_mats,
inv_full_mats=None,
camera_centers=None,
uniform_pixel_coords=None,
triangulation_method='cmp',
batch_shape=None,
image_dims=None,
dev_str=None):
"""
Compute depth map :math:`\mathbf{X}\in\mathbb{R}^{h×w×1}` in frame 1 using optical flow
:math:`\mathbf{U}_{1→2}\in\mathbb{R}^{h×w×2}` from frame 1 to 2, and the camera geometry.\n
:param flow: Optical flow from frame 1 to 2 *[batch_shape,h,w,2]*
:type flow: array
:param full_mats: Full projection matrices *[batch_shape,2,3,4]*
:type full_mats: array
:param inv_full_mats: Inverse full projection matrices, inferred from full_mats if None and 'cmp' triangulation method *[batch_shape,2,3,4]*
:type inv_full_mats: array, optional
:param camera_centers: Camera centers, inferred from inv_full_mats if None and 'cmp' triangulation method *[batch_shape,2,3,1]*
:type camera_centers: array, optional
:param uniform_pixel_coords: Homogeneous uniform (integer) pixel co-ordinate images, inferred from image_dims if None *[batch_shape,h,w,3]*
:type uniform_pixel_coords: array, optional
:param triangulation_method: Triangulation method, one of [cmp|dlt], for closest mutual points or homogeneous dlt approach, closest_mutual_points by default
:type triangulation_method: str, optional
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:param image_dims: Image dimensions. Inferred from inputs in None.
:type image_dims: sequence of ints, optional
:param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None.
:type dev_str: str, optional
:return: Depth map in frame 1 *[batch_shape,h,w,1]*
"""
if batch_shape is None:
batch_shape = flow.shape[:-3]
if image_dims is None:
image_dims = flow.shape[-3:-1]
# shapes as list
batch_shape = list(batch_shape)
image_dims = list(image_dims)
if dev_str is None:
dev_str = _ivy.dev_str(flow)
if inv_full_mats is None:
inv_full_mats = _ivy.inv(
_ivy_mech.make_transformation_homogeneous(full_mats,
batch_shape + [2],
dev_str))[..., 0:3, :]
if camera_centers is None:
camera_centers = _ivy_svg.inv_ext_mat_to_camera_center(inv_full_mats)
if uniform_pixel_coords is None:
uniform_pixel_coords = _ivy_svg.create_uniform_pixel_coords_image(
image_dims, batch_shape, dev_str=dev_str)
# BS x H x W x 3
flow_homo = _ivy.concatenate(
(flow, _ivy.zeros(batch_shape + image_dims + [1], dev_str=dev_str)),
-1)
# BS x H x W x 3
transformed_pixel_coords = uniform_pixel_coords + flow_homo
# BS x 2 x H x W x 3
pixel_coords = _ivy.concatenate(
(_ivy.expand_dims(uniform_pixel_coords, -4),
_ivy.expand_dims(transformed_pixel_coords, -4)), -4)
# BS x H x W x 1
return _ivy_tvg.triangulate_depth(pixel_coords, full_mats, inv_full_mats,
camera_centers, triangulation_method,
batch_shape, image_dims)[..., -1:]
def compute_link_matrices(self, joint_angles, link_num, batch_shape=None):
"""
Compute homogeneous transformation matrices relative to base frame, up to link_num of links.
:param joint_angles: Joint angles of the robot *[batch_shape,num_joints]*
:type joint_angles: array
:param link_num: Link number for which to compute matrices up to
:type link_num: int
:param batch_shape: Shape of batch. Inferred from inputs if None.
:type batch_shape: sequence of ints, optional
:return: The link_num matrices, up the link_num *[batch_shape,link_num,4,4]*
"""
if batch_shape is None:
batch_shape = joint_angles.shape[:-1]
batch_shape = list(batch_shape)
num_batch_dims = len(batch_shape)
# BS x 1 x NJ
try:
dh_joint_angles = _ivy.expand_dims(joint_angles * self._dh_joint_scales - self._dh_joint_offsets, -2)
except:
d = 0
# BS x 1 x 4 x 4
A00 = _ivy.identity(4, batch_shape=batch_shape + [1])
Aitoip1dashs = list()
Aiip1s = list()
A0is = [A00]
# repeated blocks
# BS x 1 x NJ
dis = _ivy.tile(_ivy.reshape(self._dis, [1] * num_batch_dims + [1, self._num_joints]),
batch_shape + [1, 1])
# BS x 1 x 4
bottom_row = _ivy.tile(
_ivy.reshape(_ivy.array([0., 0., 0., 1.]), [1] * num_batch_dims + [1, 4]),
batch_shape + [1, 1])
# BS x 1 x 3
start_of_bottom_middle = _ivy.tile(
_ivy.reshape(_ivy.array([0., 0., 1.]), [1] * num_batch_dims + [1, 3]),
batch_shape + [1, 1])
# BS x 1 x 2
zeros = _ivy.zeros(batch_shape + [1, 2])
for i in range(self._num_joints):
# BS x 1 x 4
top_row = _ivy.concatenate((_ivy.cos(dh_joint_angles[..., i:i + 1]),
-_ivy.sin(dh_joint_angles[..., i:i + 1]), zeros), -1)
top_middle_row = _ivy.concatenate((_ivy.sin(dh_joint_angles[..., i:i + 1]),
_ivy.cos(dh_joint_angles[..., i:i + 1]), zeros), -1)
bottom_middle_row = _ivy.concatenate((start_of_bottom_middle, dis[..., i:i + 1]), -1)
# BS x 4 x 4
Aitoip1dash = _ivy.concatenate((top_row, top_middle_row, bottom_middle_row, bottom_row), -2)
# (BSx4) x 4
Aitoip1dash_flat = _ivy.reshape(Aitoip1dash, (-1, 4))
# (BSx4) x 4
Aiip1_flat = _ivy.matmul(Aitoip1dash_flat, self._AidashtoAis[i + 1])
# BS x 4 x 4
Aiip1 = _ivy.reshape(Aiip1_flat, batch_shape + [4, 4])
# BS x 4 x 4
A0ip1 = _ivy.matmul(A0is[-1][..., 0, :, :], Aiip1)
# append term to lists
Aitoip1dashs.append(Aitoip1dash)
Aiip1s.append(Aiip1)
A0is.append(_ivy.expand_dims(A0ip1, -3))
if i + 1 == link_num:
# BS x LN x 4 x 4
return _ivy.concatenate(A0is, -3)
raise Exception('wrong parameter entered for link_num, please enter integer from 1-' + str(self._num_joints))
def main(batch_size=32,
num_train_steps=31250,
compile_flag=True,
num_bits=8,
seq_len=28,
ctrl_output_size=100,
memory_size=128,
memory_vector_dim=28,
overfit_flag=False,
interactive=True,
f=None):
f = choose_random_framework() if f is None else f
set_framework(f)
# train config
lr = 1e-3 if not overfit_flag else 1e-2
batch_size = batch_size if not overfit_flag else 1
num_train_steps = num_train_steps if not overfit_flag else 150
max_grad_norm = 50
# logging config
vis_freq = 250 if not overfit_flag else 1
# optimizer
optimizer = ivy.Adam(lr=lr)
# ntm
ntm = NTM(input_dim=num_bits + 1,
output_dim=num_bits,
ctrl_output_size=ctrl_output_size,
ctrl_layers=1,
memory_size=memory_size,
memory_vector_dim=memory_vector_dim,
read_head_num=1,
write_head_num=1)
# compile loss fn
total_seq_example = ivy.random_uniform(shape=(batch_size, 2 * seq_len + 1,
num_bits + 1))
target_seq_example = total_seq_example[:, 0:seq_len, :-1]
if compile_flag:
loss_fn_maybe_compiled = ivy.compile_fn(
lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn(ntm, v, ttl_sq, trgt_sq,
sq_ln),
dynamic=False,
example_inputs=[
ntm.v, total_seq_example, target_seq_example, seq_len
])
else:
loss_fn_maybe_compiled = lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn(
ntm, v, ttl_sq, trgt_sq, sq_ln)
# init
input_seq_m1 = ivy.cast(
ivy.random_uniform(0., 1., (batch_size, seq_len, num_bits)) > 0.5,
'float32')
mw = None
vw = None
for i in range(num_train_steps):
# sequence to copy
if not overfit_flag:
input_seq_m1 = ivy.cast(
ivy.random_uniform(0., 1.,
(batch_size, seq_len, num_bits)) > 0.5,
'float32')
target_seq = input_seq_m1
input_seq = ivy.concatenate(
(input_seq_m1, ivy.zeros((batch_size, seq_len, 1))), -1)
eos = ivy.ones((batch_size, 1, num_bits + 1))
output_seq = ivy.zeros_like(input_seq)
total_seq = ivy.concatenate((input_seq, eos, output_seq), -2)
# train step
loss, pred_vals = train_step(loss_fn_maybe_compiled, optimizer, ntm,
total_seq, target_seq, seq_len, mw, vw,
ivy.array(i + 1,
'float32'), max_grad_norm)
# log
print('step: {}, loss: {}'.format(i, ivy.to_numpy(loss).item()))
# visualize
if i % vis_freq == 0:
target_to_vis = (ivy.to_numpy(target_seq[0] * 255)).astype(
np.uint8)
target_to_vis = np.transpose(
cv2.resize(target_to_vis, (560, 160),
interpolation=cv2.INTER_NEAREST), (1, 0))
pred_to_vis = (ivy.to_numpy(pred_vals[0] * 255)).astype(np.uint8)
pred_to_vis = np.transpose(
cv2.resize(pred_to_vis, (560, 160),
interpolation=cv2.INTER_NEAREST), (1, 0))
img_to_vis = np.concatenate((pred_to_vis, target_to_vis), 0)
img_to_vis = cv2.resize(img_to_vis, (1120, 640),
interpolation=cv2.INTER_NEAREST)
img_to_vis[0:60, -200:] = 0
img_to_vis[5:55, -195:-5] = 255
cv2.putText(img_to_vis, 'step {}'.format(i), (935, 42),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
img_to_vis[0:60, 0:200] = 0
img_to_vis[5:55, 5:195] = 255
cv2.putText(img_to_vis, 'prediction', (7, 42),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
img_to_vis[320:380, 0:130] = 0
img_to_vis[325:375, 5:125] = 255
cv2.putText(img_to_vis, 'target', (7, 362),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2)
if interactive:
cv2.imshow('prediction_and_target', img_to_vis)
if overfit_flag:
cv2.waitKey(1)
else:
cv2.waitKey(100)
cv2.destroyAllWindows()
