def test_linear_layer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str, call): # smoke test batch_shape, input_channels, output_channels, target = bs_ic_oc_target x = ivy.cast( ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape), input_channels), 'float32') if with_v: np.random.seed(0) wlim = (6 / (output_channels + input_channels))**0.5 w = ivy.variable( ivy.array( np.random.uniform(-wlim, wlim, (output_channels, input_channels)), 'float32')) b = ivy.variable(ivy.zeros([output_channels])) v = Container({'w': w, 'b': b}) else: v = None linear_layer = ivy.Linear(input_channels, output_channels, v=v) ret = linear_layer(x) # type test assert ivy.is_array(ret) # cardinality test assert ret.shape == tuple(batch_shape + [output_channels]) # value test if not with_v: return assert np.allclose(call(linear_layer, x), np.array(target)) # compilation test if call is helpers.torch_call: # pytest scripting does not **kwargs return helpers.assert_compilable(linear_layer)
def _fit_spline(train_points, train_values, order): # shapes train_points_shape = train_points.shape batch_shape = list(train_points_shape[:-2]) num_batch_dims = len(batch_shape) n = train_points_shape[-2] pd = train_values.shape[-1] # BS x N x 1 c = train_points # BS x N x PD f_ = train_values # BS x N x N matrix_a = _phi(_pairwise_distance(c, c), order) # BS x N x 1 ones = _ivy.ones_like(c[..., :1]) # BS x N x 2 matrix_b = _ivy.concatenate([c, ones], -1) # BS x 2 x N matrix_b_trans = _ivy.transpose( matrix_b, list(range(num_batch_dims)) + [num_batch_dims + 1, num_batch_dims]) # BS x N+2 x N left_block = _ivy.concatenate([matrix_a, matrix_b_trans], -2) # BS x 2 x 2 lhs_zeros = _ivy.zeros(batch_shape + [2, 2]) # BS x N+2 x 2 right_block = _ivy.concatenate([matrix_b, lhs_zeros], -2) # BS x N+2 x N+2 lhs = _ivy.concatenate([left_block, right_block], -1) # BS x 2 x PD rhs_zeros = _ivy.zeros(batch_shape + [2, pd]) # BS x N+2 x PD rhs = _ivy.concatenate([f_, rhs_zeros], -2) # BS x N+2 x PD w_v = _ivy.matmul(_ivy.pinv(lhs), rhs) # BS x N x PD w = w_v[..., :n, :] # BS x 2 x PD v = w_v[..., n:, :] # BS x N x PD, BS x 2 x PD return w, v
def test_sgd_optimizer(bs_ic_oc_target, with_v, dtype_str, tensor_fn, dev_str, call): # smoke test if call is helpers.np_call: # NumPy does not support gradients pytest.skip() batch_shape, input_channels, output_channels, target = bs_ic_oc_target x = ivy.cast( ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape), input_channels), 'float32') if with_v: np.random.seed(0) wlim = (6 / (output_channels + input_channels))**0.5 w = ivy.variable( ivy.array( np.random.uniform(-wlim, wlim, (output_channels, input_channels)), 'float32')) b = ivy.variable(ivy.zeros([output_channels])) v = Container({'w': w, 'b': b}) else: v = None linear_layer = ivy.Linear(input_channels, output_channels, v=v) def loss_fn(v_): out = linear_layer(x, v=v_) return ivy.reduce_mean(out)[0] # optimizer optimizer = ivy.SGD() # train loss_tm1 = 1e12 loss = None grads = None for i in range(10): loss, grads = ivy.execute_with_gradients(loss_fn, linear_layer.v) linear_layer.v = optimizer.step(linear_layer.v, grads) assert loss < loss_tm1 loss_tm1 = loss # type test assert ivy.is_array(loss) assert isinstance(grads, ivy.Container) # cardinality test if call is helpers.mx_call: # mxnet slicing cannot reduce dimension to zero assert loss.shape == (1, ) else: assert loss.shape == () # value test assert ivy.reduce_max(ivy.abs(grads.b)) > 0 assert ivy.reduce_max(ivy.abs(grads.w)) > 0 # compilation test if call is helpers.torch_call: # pytest scripting does not **kwargs return helpers.assert_compilable(loss_fn)
def get_initial_state(self, batch_shape): """ Get the initial state of the hidden and cell states, if not provided explicitly """ batch_shape = list(batch_shape) return ([ ivy.zeros((batch_shape + [self._output_channels])) for i in range(self._num_layers) ], [ ivy.zeros((batch_shape + [self._output_channels])) for i in range(self._num_layers) ])
def test_lstm(b_t_ic_hc_otf_sctv, dtype_str, tensor_fn, dev_str, call): # smoke test b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv x = ivy.cast(ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels), 'float32') init_h = ivy.ones([b, hidden_channels]) init_c = ivy.ones([b, hidden_channels]) kernel = ivy.variable(ivy.ones([input_channels, 4*hidden_channels]))*0.5 recurrent_kernel = ivy.variable(ivy.ones([hidden_channels, 4*hidden_channels]))*0.5 output, state_c = ivy.lstm_update(x, init_h, init_c, kernel, recurrent_kernel) # type test assert ivy.is_array(output) assert ivy.is_array(state_c) # cardinality test assert output.shape == (b, t, hidden_channels) assert state_c.shape == (b, hidden_channels) # value test output_true = np.tile(np.asarray(output_true_flat).reshape((b, t, 1)), (1, 1, hidden_channels)) state_c_true = np.ones([b, hidden_channels]) * state_c_true_val output, state_c = call(ivy.lstm_update, x, init_h, init_c, kernel, recurrent_kernel) assert np.allclose(output, output_true, atol=1e-6) assert np.allclose(state_c, state_c_true, atol=1e-6) # compilation test if call in [helpers.torch_call]: # this is not a backend implemented function pytest.skip() helpers.assert_compilable(ivy.lstm_update)
def test_lstm_layer(b_t_ic_hc_otf_sctv, with_v, with_initial_state, dtype_str, tensor_fn, dev_str, call): # smoke test b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv x = ivy.cast( ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels), 'float32') if with_initial_state: init_h = ivy.ones([b, hidden_channels]) init_c = ivy.ones([b, hidden_channels]) initial_state = ([init_h], [init_c]) else: initial_state = None if with_v: kernel = ivy.variable( ivy.ones([input_channels, 4 * hidden_channels]) * 0.5) recurrent_kernel = ivy.variable( ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5) v = Container({ 'input': { 'layer_0': { 'w': kernel } }, 'recurrent': { 'layer_0': { 'w': recurrent_kernel } } }) else: v = None lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v) output, (state_h, state_c) = lstm_layer(x, initial_state=initial_state) # type test assert ivy.is_array(output) assert ivy.is_array(state_h[0]) assert ivy.is_array(state_c[0]) # cardinality test assert output.shape == (b, t, hidden_channels) assert state_h[0].shape == (b, hidden_channels) assert state_c[0].shape == (b, hidden_channels) # value test if not with_v or not with_initial_state: return output_true = np.tile( np.asarray(output_true_flat).reshape((b, t, 1)), (1, 1, hidden_channels)) state_c_true = np.ones([b, hidden_channels]) * state_c_true_val output, (state_h, state_c) = call(lstm_layer, x, initial_state=initial_state) assert np.allclose(output, output_true, atol=1e-6) assert np.allclose(state_c, state_c_true, atol=1e-6) # compilation test if call in [helpers.torch_call]: # this is not a backend implemented function pytest.skip() helpers.assert_compilable(ivy.lstm_update)
def _get_dummy_obs(batch_size, num_frames, num_cams, image_dims, num_feature_channels, dev_str='cpu', ones=False, empty=False): uniform_pixel_coords =\ ivy_vision.create_uniform_pixel_coords_image(image_dims, [batch_size, num_frames], dev_str=dev_str) img_meas = dict() for i in range(num_cams): validity_mask = ivy.ones([batch_size, num_frames] + image_dims + [1], dev_str=dev_str) if ones: img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.ones( [batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1) img_var = ivy.ones( [batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str)*1e-3 pose_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str) pose_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3 else: img_mean = ivy.concatenate((uniform_pixel_coords[..., 0:2], ivy.random_uniform( 1e-3, 1, [batch_size, num_frames] + image_dims + [1 + num_feature_channels], dev_str=dev_str)), -1) img_var = ivy.random_uniform( 1e-3, 1, [batch_size, num_frames] + image_dims + [3 + num_feature_channels], dev_str=dev_str) pose_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str) pose_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str) if empty: img_var = ivy.ones_like(img_var) * 1e12 validity_mask = ivy.zeros_like(validity_mask) img_meas['dummy_cam_{}'.format(i)] =\ {'img_mean': img_mean, 'img_var': img_var, 'validity_mask': validity_mask, 'pose_mean': pose_mean, 'pose_cov': pose_cov, 'cam_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames], dev_str=dev_str)[..., 0:3, :]} if ones: control_mean = ivy.zeros([batch_size, num_frames, 6], dev_str=dev_str) control_cov = ivy.ones([batch_size, num_frames, 6, 6], dev_str=dev_str)*1e-3 else: control_mean = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6], dev_str=dev_str) control_cov = ivy.random_uniform(1e-3, 1, [batch_size, num_frames, 6, 6], dev_str=dev_str) return Container({'img_meas': img_meas, 'control_mean': control_mean, 'control_cov': control_cov, 'agent_rel_mat': ivy.identity(4, batch_shape=[batch_size, num_frames], dev_str=dev_str)[..., 0:3, :]})
def test_module_w_none_attribute(bs_ic_oc, dev_str, call): # smoke test if call is helpers.np_call: # NumPy does not support gradients pytest.skip() batch_shape, input_channels, output_channels = bs_ic_oc x = ivy.cast( ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape), input_channels), 'float32') module = ModuleWithNoneAttribute()
def _create_variables(self, dev_str): """ Create internal variables for the Linear layer """ # ToDo: support other initialization mechanisms, via class constructor options # ToDo: tidy the construction of these variables, with helper functions wlim = (6 / (self._output_channels + self._input_channels))**0.5 w = ivy.variable( ivy.random_uniform(-wlim, wlim, (self._output_channels, self._input_channels), dev_str=dev_str)) b = ivy.variable(ivy.zeros([self._output_channels], dev_str=dev_str)) return {'w': w, 'b': b}
def _addressing(self, k, beta, g, s, gamma, prev_M, prev_w): # Sec 3.3.1 Focusing by Content # Cosine Similarity k = ivy.expand_dims(k, axis=2) inner_product = ivy.matmul(prev_M, k) k_norm = ivy.reduce_sum(k**2, axis=1, keepdims=True)**0.5 M_norm = ivy.reduce_sum(prev_M**2, axis=2, keepdims=True)**0.5 norm_product = M_norm * k_norm K = ivy.squeeze(inner_product / (norm_product + 1e-8)) # eq (6) # Calculating w^c K_amplified = ivy.exp(ivy.expand_dims(beta, axis=1) * K) w_c = K_amplified / ivy.reduce_sum(K_amplified, axis=1, keepdims=True) # eq (5) if self._addressing_mode == 'content': # Only focus on content return w_c # Sec 3.3.2 Focusing by Location g = ivy.expand_dims(g, axis=1) w_g = g * w_c + (1 - g) * prev_w # eq (7) s = ivy.concatenate([ s[:, :self._shift_range + 1], ivy.zeros( [s.shape[0], self._memory_size - (self._shift_range * 2 + 1)]), s[:, -self._shift_range:] ], axis=1) t = ivy.concatenate([ivy.flip(s, axis=[1]), ivy.flip(s, axis=[1])], axis=1) s_matrix = ivy.stack([ t[:, self._memory_size - i - 1:self._memory_size * 2 - i - 1] for i in range(self._memory_size) ], axis=1) w_ = ivy.reduce_sum(ivy.expand_dims(w_g, axis=1) * s_matrix, axis=2) # eq (8) w_sharpen = w_**ivy.expand_dims(gamma, axis=1) w = w_sharpen / ivy.reduce_sum(w_sharpen, axis=1, keepdims=True) # eq (9) return w
def as_identity(batch_shape): """ Return camera intrinsics object with array attributes as either zeros or identity matrices. :param batch_shape: Batch shape for each geometric array attribute :type batch_shape: sequence of ints :return: New camera intrinsics object, with each entry as either zeros or identity matrices. """ batch_shape = list(batch_shape) focal_lengths = _ivy.ones(batch_shape + [2]) persp_angles = _ivy.ones(batch_shape + [2]) pp_offsets = _ivy.zeros(batch_shape + [2]) calib_mats = _ivy.identity(3, batch_shape=batch_shape) inv_calib_mats = _ivy.identity(3, batch_shape=batch_shape) return __class__(focal_lengths, persp_angles, pp_offsets, calib_mats, inv_calib_mats)
def as_identity(batch_shape): """ Return camera extrinsics object with array attributes as either zeros or identity matrices. :param batch_shape: Batch shape for each geometric array attribute. :type batch_shape: sequence of ints :return: New camera extrinsics object, with each entry as either zeros or identity matrices. """ batch_shape = list(batch_shape) cam_centers = _ivy.zeros(batch_shape + [3, 1]) Rs = _ivy.identity(3, batch_shape=batch_shape) inv_Rs = _ivy.identity(3, batch_shape=batch_shape) ext_mats_homo = _ivy.identity(4, batch_shape=batch_shape) inv_ext_mats_homo = _ivy.identity(4, batch_shape=batch_shape) return __class__(cam_centers, Rs, inv_Rs, ext_mats_homo, inv_ext_mats_homo)
def test_module_training(bs_ic_oc, dev_str, call): # smoke test if call is helpers.np_call: # NumPy does not support gradients pytest.skip() batch_shape, input_channels, output_channels = bs_ic_oc x = ivy.cast( ivy.linspace(ivy.zeros(batch_shape), ivy.ones(batch_shape), input_channels), 'float32') module = TrainableModule(input_channels, output_channels) def loss_fn(v_): out = module(x, v=v_) return ivy.reduce_mean(out)[0] # train loss_tm1 = 1e12 loss = None grads = None for i in range(10): loss, grads = ivy.execute_with_gradients(loss_fn, module.v) module.v = ivy.gradient_descent_update(module.v, grads, 1e-3) assert loss < loss_tm1 loss_tm1 = loss # type test assert ivy.is_array(loss) assert isinstance(grads, ivy.Container) # cardinality test if call is helpers.mx_call: # mxnet slicing cannot reduce dimension to zero assert loss.shape == (1, ) else: assert loss.shape == () # value test assert ivy.reduce_max(ivy.abs(grads.linear0.b)) > 0 assert ivy.reduce_max(ivy.abs(grads.linear0.w)) > 0 assert ivy.reduce_max(ivy.abs(grads.linear1.b)) > 0 assert ivy.reduce_max(ivy.abs(grads.linear1.w)) > 0 assert ivy.reduce_max(ivy.abs(grads.linear2.b)) > 0 assert ivy.reduce_max(ivy.abs(grads.linear2.w)) > 0 # compilation test if call is helpers.torch_call: # pytest scripting does not support **kwargs return helpers.assert_compilable(loss_fn)
def __init__(self, controller, controller_proj, output_proj, output_dim, ctrl_input_size, ctrl_output_size, total_parameter_num, memory_size, memory_vector_dim, read_head_num, write_head_num, v=None, usage=None, addressing_mode='content_and_location', shift_range=1, clip_value=20, init_value=1e-6, sequential_writing=False, retroactive_updates=False, retroactive_discount=0.96, with_erase=True, seed=0): # vanilla ntm self._memory_size = memory_size self._memory_vector_dim = memory_vector_dim self._read_head_num = read_head_num self._write_head_num = write_head_num self._init_value = init_value self._addressing_mode = addressing_mode self._clip_value = clip_value self._output_dim = output_dim self._shift_range = shift_range self._num_parameters_per_head = self._memory_vector_dim + 1 + 1 + (self._shift_range * 2 + 1) + 1 self._num_heads = self._read_head_num + self._write_head_num ivy.seed(seed) # fns + classes self._controller = controller self._controller_proj = controller_proj self._output_proj = output_proj # usage if usage is not None: self._usage = usage else: self._usage = ivy.zeros([memory_size, ]) # step self._step = 0 # MERLIN changes self._sequential_writing = sequential_writing self._retroactive_updates = retroactive_updates self._retroactive_discount = retroactive_discount self._with_erase = with_erase # variables ivy.Module.__init__(self, 'cpu')
def focal_lengths_and_pp_offsets_to_calib_mat(focal_lengths, pp_offsets, batch_shape=None, dev_str=None): """ Compute calibration matrix :math:`\mathbf{K}\in\mathbb{R}^{3×3}` from focal lengths :math:`f_x, f_y` and principal-point offsets :math:`p_x, p_y`.\n `[reference] <localhost:63342/ivy/docs/source/references/mvg_textbook.pdf#page=173>`_ page 155, section 6.1, equation 6.4 :param focal_lengths: Focal lengths *[batch_shape,2]* :type focal_lengths: array :param pp_offsets: Principal-point offsets *[batch_shape,2]* :type pp_offsets: array :param batch_shape: Shape of batch. Inferred from inputs if None. :type batch_shape: sequence of ints, optional :param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None. :type dev_str: str, optional :return: Calibration matrix *[batch_shape,3,3]* """ if batch_shape is None: batch_shape = focal_lengths.shape[:-1] if dev_str is None: dev_str = _ivy.dev_str(focal_lengths) # shapes as list batch_shape = list(batch_shape) # BS x 1 x 1 zeros = _ivy.zeros(batch_shape + [1, 1], dev_str=dev_str) ones = _ivy.ones(batch_shape + [1, 1], dev_str=dev_str) # BS x 2 x 1 focal_lengths_reshaped = _ivy.expand_dims(focal_lengths, -1) pp_offsets_reshaped = _ivy.expand_dims(pp_offsets, -1) # BS x 1 x 3 row1 = _ivy.concatenate((focal_lengths_reshaped[..., 0:1, :], zeros, pp_offsets_reshaped[..., 0:1, :]), -1) row2 = _ivy.concatenate((zeros, focal_lengths_reshaped[..., 1:2, :], pp_offsets_reshaped[..., 1:2, :]), -1) row3 = _ivy.concatenate((zeros, zeros, ones), -1) # BS x 3 x 3 return _ivy.concatenate((row1, row2, row3), -2)
def __init__(self, base_inv_ext_mat=None): """ Initialize FRANKA EMIKA Panda robot manipulator instance. Denavit–Hartenberg parameters inferred from FRANKA EMIKA online API. Screenshot included in this module for reference. :param base_inv_ext_mat: Inverse extrinsic matrix of the robot base *[3,4]* :type base_inv_ext_mat: array, optional """ # dh params # panda_DH_params.png # taken from https://frankaemika.github.io/docs/control_parameters.html a_s = _ivy.array([0., 0., 0.0825, -0.0825, 0., 0.088, 0.]) d_s = _ivy.array([0.333, 0., 0.316, 0., 0.384, 0., 0.107]) alpha_s = _ivy.array( [-_math.pi / 2, _math.pi / 2, _math.pi / 2, -_math.pi / 2, _math.pi / 2, _math.pi / 2, 0.]) dh_joint_scales = _ivy.ones((7,)) dh_joint_offsets = _ivy.zeros((7,)) super().__init__(a_s, d_s, alpha_s, dh_joint_scales, dh_joint_offsets, base_inv_ext_mat)
def __init__(self, data_loader_spec): super().__init__(data_loader_spec) # dataset size self._num_examples = self._spec.dataset_spec.num_examples # counter self._i = 0 # load vector data vector_dim = self._spec.dataset_spec.vector_dim self._targets = ivy.zeros((self._num_examples, vector_dim, 1)) # load image data image_dims = self._spec.dataset_spec.image_dims self._input = ivy.ones( (self._num_examples, image_dims[0], image_dims[1], 3)) self._training_data = ivy.Container(targets=self._targets, input=self._input) self._validation_data = ivy.Container(targets=self._targets, input=self._input) self._data = ivy.Container(training=self._training_data, validation=self._validation_data)
def quantize_to_image(pixel_coords, final_image_dims, feat=None, feat_prior=None, with_db=False, pixel_coords_var=1e-3, feat_var=1e-3, pixel_coords_prior_var=1e12, feat_prior_var=1e12, var_threshold=(1e-3, 1e12), uniform_pixel_coords=None, batch_shape=None, dev_str=None): """ Quantize pixel co-ordinates with d feature channels (for depth, rgb, normals etc.), from images :math:`\mathbf{X}\in\mathbb{R}^{input\_images\_shape×(2+d)}`, which may have been reprojected from a host of different cameras (leading to non-integer pixel values), to a new quantized pixel co-ordinate image with the same feature channels :math:`\mathbf{X}\in\mathbb{R}^{h×w×(2+d)}`, and with integer pixel co-ordinates. Duplicates during the quantization are either probabilistically fused based on variance, or the minimum depth is chosen when using depth buffer mode. :param pixel_coords: Pixel co-ordinates *[batch_shape,input_size,2]* :type pixel_coords: array :param final_image_dims: Image dimensions of the final image. :type final_image_dims: sequence of ints :param feat: Features (i.e. depth, rgb, encoded), default is None. *[batch_shape,input_size,d]* :type feat: array, optional :param feat_prior: Prior feature image mean, default is None. *[batch_shape,input_size,d]* :type feat_prior: array or float to fill with :param with_db: Whether or not to use depth buffer in rendering, default is false :type with_db: bool, optional :param pixel_coords_var: Pixel coords variance *[batch_shape,input_size,2]* :type pixel_coords_var: array or float to fill with :param feat_var: Feature variance *[batch_shape,input_size,d]* :type feat_var: array or float to fill with :param pixel_coords_prior_var: Pixel coords prior variance *[batch_shape,h,w,2]* :type pixel_coords_prior_var: array or float to fill with :param feat_prior_var: Features prior variance *[batch_shape,h,w,3]* :type feat_prior_var: array or float to fill with :param var_threshold: Variance threshold, for projecting valid coords and clipping *[batch_shape,2+d,2]* :type var_threshold: array or sequence of floats to fill with :param uniform_pixel_coords: Homogeneous uniform (integer) pixel co-ordinate images, inferred from final_image_dims if None *[batch_shape,h,w,3]* :type uniform_pixel_coords: array, optional :param batch_shape: Shape of batch. Assumed no batches if None. :type batch_shape: sequence of ints, optional :param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None. :type dev_str: str, optional :return: Quantized pixel co-ordinates image with d feature channels (for depth, rgb, normals etc.) *[batch_shape,h,w,2+d]*, maybe the quantized variance, *[batch_shape,h,w,2+d]*, and scatter counter image *[batch_shape,h,w,1]* """ # ToDo: make variance fully optional. If not specified, # then do not compute and scatter during function call for better efficiency. # config if batch_shape is None: batch_shape = pixel_coords.shape[:-2] if dev_str is None: dev_str = _ivy.dev_str(pixel_coords) if feat is None: d = 0 else: d = feat.shape[-1] min_depth_diff = _ivy.array([MIN_DEPTH_DIFF], dev_str=dev_str) red = 'min' if with_db else 'sum' # shapes as list batch_shape = list(batch_shape) final_image_dims = list(final_image_dims) num_batch_dims = len(batch_shape) # variance threshold if isinstance(var_threshold, tuple) or isinstance(var_threshold, list): ones = _ivy.ones(batch_shape + [1, 2 + d, 1]) var_threshold = _ivy.concatenate( (ones * var_threshold[0], ones * var_threshold[1]), -1) else: var_threshold = _ivy.reshape(var_threshold, batch_shape + [1, 2 + d, 2]) # uniform pixel coords if uniform_pixel_coords is None: uniform_pixel_coords =\ _ivy_svg.create_uniform_pixel_coords_image(final_image_dims, batch_shape, dev_str=dev_str) uniform_pixel_coords = uniform_pixel_coords[..., 0:2] # Extract Values # feat_prior = _ivy.ones_like(feat) * feat_prior if isinstance( feat_prior, float) else feat_prior pixel_coords_var = _ivy.ones_like(pixel_coords) * pixel_coords_var\ if isinstance(pixel_coords_var, float) else pixel_coords_var feat_var = _ivy.ones_like(feat) * feat_var if isinstance( feat_var, float) else feat_var pixel_coords_prior_var = _ivy.ones(batch_shape + final_image_dims + [2]) * pixel_coords_prior_var\ if isinstance(pixel_coords_prior_var, float) else pixel_coords_prior_var feat_prior_var = _ivy.ones(batch_shape + final_image_dims + [d]) * feat_prior_var\ if isinstance(feat_prior_var, float) else feat_prior_var # Quantize # # BS x N x 2 quantized_pixel_coords = _ivy.reshape( _ivy.cast(_ivy.round(pixel_coords), 'int32'), batch_shape + [-1, 2]) # Combine # # BS x N x (2+D) pc_n_feat = _ivy.reshape(_ivy.concatenate((pixel_coords, feat), -1), batch_shape + [-1, 2 + d]) pc_n_feat_var = _ivy.reshape( _ivy.concatenate((pixel_coords_var, feat_var), -1), batch_shape + [-1, 2 + d]) # BS x H x W x (2+D) prior = _ivy.concatenate((uniform_pixel_coords, feat_prior), -1) prior_var = _ivy.concatenate((pixel_coords_prior_var, feat_prior_var), -1) # Validity Mask # # BS x N x 1 var_validity_mask = \ _ivy.reduce_sum(_ivy.cast(pc_n_feat_var < var_threshold[..., 1], 'int32'), -1, keepdims=True) == 2+d bounds_validity_mask = _ivy.logical_and( _ivy.logical_and(quantized_pixel_coords[..., 0:1] >= 0, quantized_pixel_coords[..., 1:2] >= 0), _ivy.logical_and( quantized_pixel_coords[..., 0:1] <= final_image_dims[1] - 1, quantized_pixel_coords[..., 1:2] <= final_image_dims[0] - 1)) validity_mask = _ivy.logical_and(var_validity_mask, bounds_validity_mask) # num_valid_indices x len(BS)+2 validity_indices = _ivy.reshape( _ivy.cast(_ivy.indices_where(validity_mask), 'int32'), [-1, num_batch_dims + 2]) num_valid_indices = validity_indices.shape[-2] if num_valid_indices == 0: return _ivy.concatenate((uniform_pixel_coords[..., 0:2], feat_prior), -1), \ _ivy.concatenate((pixel_coords_prior_var, feat_prior_var), -1),\ _ivy.zeros_like(feat[..., 0:1], dev_str=dev_str) # Depth Based Scaling # mean_depth_min = None mean_depth_range = None pc_n_feat_wo_depth_range = None pc_n_feat_wo_depth_min = None var_vals_range = None var_vals_min = None if with_db: # BS x N x 1 mean_depth = pc_n_feat[..., 2:3] # BS x 1 x 1 mean_depth_min = _ivy.reduce_min(mean_depth, -2, keepdims=True) mean_depth_max = _ivy.reduce_max(mean_depth, -2, keepdims=True) mean_depth_range = mean_depth_max - mean_depth_min # BS x N x 1 scaled_depth = (mean_depth - mean_depth_min) / ( mean_depth_range * min_depth_diff + MIN_DENOMINATOR) if d == 1: # BS x 1 x 1+D pc_n_feat_wo_depth_min = _ivy.zeros(batch_shape + [1, 0], dev_str=dev_str) pc_n_feat_wo_depth_range = _ivy.ones(batch_shape + [1, 0], dev_str=dev_str) else: # feat without depth # BS x N x 1+D pc_n_feat_wo_depth = _ivy.concatenate( (pc_n_feat[..., 0:2], pc_n_feat[..., 3:]), -1) # find the min and max of each value # BS x 1 x 1+D pc_n_feat_wo_depth_max = _ivy.reduce_max( pc_n_feat_wo_depth, -2, keepdims=True) + 1 pc_n_feat_wo_depth_min = _ivy.reduce_min( pc_n_feat_wo_depth, -2, keepdims=True) - 1 pc_n_feat_wo_depth_range = pc_n_feat_wo_depth_max - pc_n_feat_wo_depth_min # BS x N x 1+D normed_pc_n_feat_wo_depth = (pc_n_feat_wo_depth - pc_n_feat_wo_depth_min) / \ (pc_n_feat_wo_depth_range + MIN_DENOMINATOR) # combine with scaled depth # BS x N x 1+D pc_n_feat_wo_depth_scaled = normed_pc_n_feat_wo_depth + scaled_depth # BS x N x (2+D) pc_n_feat = _ivy.concatenate( (pc_n_feat_wo_depth_scaled[..., 0:2], mean_depth, pc_n_feat_wo_depth_scaled[..., 2:]), -1) # scale variance # BS x 1 x (2+D) var_vals_max = _ivy.reduce_max(pc_n_feat_var, -2, keepdims=True) + 1 var_vals_min = _ivy.reduce_min(pc_n_feat_var, -2, keepdims=True) - 1 var_vals_range = var_vals_max - var_vals_min # BS x N x (2+D) normed_var_vals = (pc_n_feat_var - var_vals_min) / (var_vals_range + MIN_DENOMINATOR) pc_n_feat_var = normed_var_vals + scaled_depth # ready for later reversal with full image dimensions # BS x 1 x 1 x D var_vals_min = _ivy.expand_dims(var_vals_min, -2) var_vals_range = _ivy.expand_dims(var_vals_range, -2) # Validity Pruning # # num_valid_indices x (2+D) pc_n_feat = _ivy.gather_nd(pc_n_feat, validity_indices[..., 0:num_batch_dims + 1]) pc_n_feat_var = _ivy.gather_nd(pc_n_feat_var, validity_indices[..., 0:num_batch_dims + 1]) # num_valid_indices x 2 quantized_pixel_coords = _ivy.gather_nd( quantized_pixel_coords, validity_indices[..., 0:num_batch_dims + 1]) if with_db: means_to_scatter = pc_n_feat vars_to_scatter = pc_n_feat_var else: # num_valid_indices x (2+D) vars_to_scatter = 1 / (pc_n_feat_var + MIN_DENOMINATOR) means_to_scatter = pc_n_feat * vars_to_scatter # Scatter # # num_valid_indices x 1 counter = _ivy.ones_like(pc_n_feat[..., 0:1], dev_str=dev_str) if with_db: counter *= -1 # num_valid_indices x 2(2+D)+1 values_to_scatter = _ivy.concatenate( (means_to_scatter, vars_to_scatter, counter), -1) # num_valid_indices x (num_batch_dims + 2) all_indices = _ivy.flip(quantized_pixel_coords, -1) if num_batch_dims > 0: all_indices = _ivy.concatenate( (validity_indices[..., :-2], all_indices), -1) # BS x H x W x (2(2+D) + 1) quantized_img = _ivy.scatter_nd( _ivy.reshape(all_indices, [-1, num_batch_dims + 2]), _ivy.reshape(values_to_scatter, [-1, 2 * (2 + d) + 1]), batch_shape + final_image_dims + [2 * (2 + d) + 1], reduction='replace' if _ivy.backend == 'mxnd' else red) # BS x H x W x 1 quantized_counter = quantized_img[..., -1:] if with_db: invalidity_mask = quantized_counter != -1 else: invalidity_mask = quantized_counter == 0 if with_db: # BS x H x W x (2+D) quantized_mean_scaled = quantized_img[..., 0:2 + d] quantized_var_scaled = quantized_img[..., 2 + d:2 * (2 + d)] # BS x H x W x 1 quantized_depth_mean = quantized_mean_scaled[..., 2:3] # BS x 1 x 1 x 1 mean_depth_min = _ivy.expand_dims(mean_depth_min, -2) mean_depth_range = _ivy.expand_dims(mean_depth_range, -2) # BS x 1 x 1 x (1+D) pc_n_feat_wo_depth_min = _ivy.expand_dims(pc_n_feat_wo_depth_min, -2) pc_n_feat_wo_depth_range = _ivy.expand_dims(pc_n_feat_wo_depth_range, -2) # BS x 1 x 1 x (2+D) x 2 var_threshold = _ivy.expand_dims(var_threshold, -3) # BS x H x W x (1+D) quantized_mean_wo_depth_scaled = _ivy.concatenate( (quantized_mean_scaled[..., 0:2], quantized_mean_scaled[..., 3:]), -1) quantized_mean_wo_depth_normed = quantized_mean_wo_depth_scaled - (quantized_depth_mean - mean_depth_min) / \ (mean_depth_range * min_depth_diff + MIN_DENOMINATOR) quantized_mean_wo_depth = quantized_mean_wo_depth_normed * pc_n_feat_wo_depth_range + pc_n_feat_wo_depth_min prior_wo_depth = _ivy.concatenate((prior[..., 0:2], prior[..., 3:]), -1) quantized_mean_wo_depth = _ivy.where(invalidity_mask, prior_wo_depth, quantized_mean_wo_depth) # BS x H x W x (2+D) quantized_mean = _ivy.concatenate( (quantized_mean_wo_depth[..., 0:2], quantized_depth_mean, quantized_mean_wo_depth[..., 2:]), -1) # BS x H x W x (2+D) quantized_var_normed = quantized_var_scaled - (quantized_depth_mean - mean_depth_min) / \ (mean_depth_range * min_depth_diff + MIN_DENOMINATOR) quantized_var = _ivy.maximum( quantized_var_normed * var_vals_range + var_vals_min, var_threshold[..., 0]) quantized_var = _ivy.where(invalidity_mask, prior_var, quantized_var) else: # BS x H x W x (2+D) quantized_sum_mean_x_recip_var = quantized_img[..., 0:2 + d] quantized_var_wo_increase = _ivy.where( invalidity_mask, prior_var, (1 / (quantized_img[..., 2 + d:2 * (2 + d)] + MIN_DENOMINATOR))) quantized_var = _ivy.maximum( quantized_var_wo_increase * quantized_counter, _ivy.expand_dims(var_threshold[..., 0], -2)) quantized_var = _ivy.where(invalidity_mask, prior_var, quantized_var) quantized_mean = _ivy.where( invalidity_mask, prior, quantized_var_wo_increase * quantized_sum_mean_x_recip_var) # BS x H x W x (2+D) BS x H x W x (2+D) BS x H x W x 1 return quantized_mean, quantized_var, quantized_counter
def reset(self): self.urchin_xys = ivy.random_uniform(-1, 1, (self.num_urchins, 2)) self.xy = ivy.random_uniform(-1, 1, (2, )) self.xy_vel = ivy.zeros((2, )) self.goal_xy = ivy.random_uniform(-1, 1, (2, )) return self.get_observation()
def main(interactive=True, try_use_sim=True, f=None): f = choose_random_framework() if f is None else f set_framework(f) with_mxnd = f is ivy.mxnd if with_mxnd: print('\nMXnet does not support "sum" or "min" reductions for scatter_nd,\n' 'instead it only supports non-deterministic replacement for duplicates.\n' 'Depth buffer rendering (requies min) or fusion buffer (requies sum) are therefore unsupported.\n' 'The rendering in this demo with MXNet backend exhibits non-deterministic jagged edges as a result.') sim = Simulator(interactive, try_use_sim) import matplotlib.pyplot as plt xyzs = list() rgbs = list() iterations = 10 if sim.with_pyrep else 1 for _ in range(iterations): for cam in sim.cams: depth, rgb = cam.cap() xyz = sim.depth_to_xyz(depth, cam.get_inv_ext_mat(), cam.inv_calib_mat, [1024]*2) xyzs.append(xyz) rgbs.append(rgb) xyz = ivy.reshape(ivy.concatenate(xyzs, 1), (-1, 3)) rgb = ivy.reshape(ivy.concatenate(rgbs, 1), (-1, 3)) cam_coords = ivy_vision.world_to_cam_coords(ivy_mech.make_coordinates_homogeneous(ivy.expand_dims(xyz, 1)), sim.target_cam.get_ext_mat()) ds_pix_coords = ivy_vision.cam_to_ds_pixel_coords(cam_coords, sim.target_cam.calib_mat) depth = ds_pix_coords[..., -1] pix_coords = ds_pix_coords[..., 0, 0:2] / depth final_image_dims = [512]*2 feat = ivy.concatenate((depth, rgb), -1) rendered_img_no_db, _, _ = ivy_vision.quantize_to_image( pix_coords, final_image_dims, feat, ivy.zeros(final_image_dims + [4]), with_db=False) with_db = not with_mxnd rendered_img_with_db, _, _ = ivy_vision.quantize_to_image( pix_coords, final_image_dims, feat, ivy.zeros(final_image_dims + [4]), with_db=with_db) a_img = cv2.resize(ivy.to_numpy(rgbs[0]), (256, 256)) a_img[0:50, 0:50] = np.zeros_like(a_img[0:50, 0:50]) a_img[5:45, 5:45] = np.ones_like(a_img[5:45, 5:45]) cv2.putText(a_img, 'a', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) b_img = cv2.resize(ivy.to_numpy(rgbs[1]), (256, 256)) b_img[0:50, 0:50] = np.zeros_like(b_img[0:50, 0:50]) b_img[5:45, 5:45] = np.ones_like(b_img[5:45, 5:45]) cv2.putText(b_img, 'b', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) c_img = cv2.resize(ivy.to_numpy(rgbs[2]), (256, 256)) c_img[0:50, 0:50] = np.zeros_like(c_img[0:50, 0:50]) c_img[5:45, 5:45] = np.ones_like(c_img[5:45, 5:45]) cv2.putText(c_img, 'c', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) target_img = cv2.resize(ivy.to_numpy(sim.target_cam.cap()[1]), (256, 256)) target_img[0:50, 0:140] = np.zeros_like(target_img[0:50, 0:140]) target_img[5:45, 5:135] = np.ones_like(target_img[5:45, 5:135]) cv2.putText(target_img, 'target', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) msg = 'non-deterministic' if with_mxnd else 'no depth buffer' width = 360 if with_mxnd else 320 no_db_img = np.copy(ivy.to_numpy(rendered_img_no_db[..., 3:])) no_db_img[0:50, 0:width+5] = np.zeros_like(no_db_img[0:50, 0:width+5]) no_db_img[5:45, 5:width] = np.ones_like(no_db_img[5:45, 5:width]) cv2.putText(no_db_img, msg, (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) with_db_img = np.copy(ivy.to_numpy(rendered_img_with_db[..., 3:])) with_db_img[0:50, 0:350] = np.zeros_like(with_db_img[0:50, 0:350]) with_db_img[5:45, 5:345] = np.ones_like(with_db_img[5:45, 5:345]) cv2.putText(with_db_img, 'with depth buffer', (13, 33), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) raw_imgs = np.concatenate((np.concatenate((a_img, b_img), 1), np.concatenate((c_img, target_img), 1)), 0) to_concat = (raw_imgs, no_db_img) if with_mxnd else (raw_imgs, no_db_img, with_db_img) final_img = np.concatenate(to_concat, 1) if interactive: print('\nClose the image window when you are ready.\n') plt.imshow(final_img) plt.show() xyzs.clear() rgbs.clear() sim.close() unset_framework()
def test_lstm_layer_training(b_t_ic_hc_otf_sctv, with_v, dtype_str, tensor_fn, dev_str, call): # smoke test if call is helpers.np_call: # NumPy does not support gradients pytest.skip() # smoke test b, t, input_channels, hidden_channels, output_true_flat, state_c_true_val = b_t_ic_hc_otf_sctv x = ivy.cast( ivy.linspace(ivy.zeros([b, t]), ivy.ones([b, t]), input_channels), 'float32') if with_v: kernel = ivy.variable( ivy.ones([input_channels, 4 * hidden_channels]) * 0.5) recurrent_kernel = ivy.variable( ivy.ones([hidden_channels, 4 * hidden_channels]) * 0.5) v = Container({ 'input': { 'layer_0': { 'w': kernel } }, 'recurrent': { 'layer_0': { 'w': recurrent_kernel } } }) else: v = None lstm_layer = ivy.LSTM(input_channels, hidden_channels, v=v) def loss_fn(v_): out, (state_h, state_c) = lstm_layer(x, v=v_) return ivy.reduce_mean(out)[0] # train loss_tm1 = 1e12 loss = None grads = None for i in range(10): loss, grads = ivy.execute_with_gradients(loss_fn, lstm_layer.v) lstm_layer.v = ivy.gradient_descent_update(lstm_layer.v, grads, 1e-3) assert loss < loss_tm1 loss_tm1 = loss # type test assert ivy.is_array(loss) assert isinstance(grads, ivy.Container) # cardinality test if call is helpers.mx_call: # mxnet slicing cannot reduce dimension to zero assert loss.shape == (1, ) else: assert loss.shape == () # value test for key, val in grads.to_iterator(): assert ivy.reduce_max(ivy.abs(val)) > 0 # compilation test if call is helpers.torch_call: # pytest scripting does not **kwargs return helpers.assert_compilable(loss_fn)
def test_ntm(addressing_mode, batch_shape, dev_str, call): # ntm config input_dim = 256 output_dim = 8 ctrl_output_size = 256 ctrl_layers = 2 memory_size = 5 timesteps = 5 memory_vector_dim = 2 read_head_num = 3 write_head_num = 1 shift_range = 0 clip_value = 20 init_value = 1e-6 ctrl_input_size = read_head_num * memory_vector_dim + input_dim num_heads = read_head_num + write_head_num num_parameters_per_head = memory_vector_dim + 1 + 1 + (shift_range * 2 + 1) + 1 total_parameter_num = num_parameters_per_head * num_heads + memory_vector_dim * 2 * write_head_num usage = ivy.zeros([ memory_size, ]) # memory object wo vars ntm = ivy_mem.NTM(input_dim, output_dim, ctrl_output_size, ctrl_layers, memory_size, memory_vector_dim, read_head_num, write_head_num, addressing_mode=addressing_mode, shift_range=shift_range, clip_value=clip_value, sequential_writing=True, retroactive_updates=False, with_erase=False) # test x = ivy.ones(batch_shape + [timesteps, input_dim]) assert call(ntm, x).shape == tuple(batch_shape + [timesteps, output_dim]) # variables variables = dict() variables['ntm_cell'] = dict() np.random.seed(0) # lstm in_wlim = (6 / (ctrl_input_size + 4 * ctrl_output_size))**0.5 rec_wlim = (6 / (ctrl_output_size + 4 * ctrl_output_size))**0.5 variables['ntm_cell']['controller'] = \ {'input': {'layer1': {'w': ivy.array(np.random.uniform( -in_wlim, in_wlim, size=[ctrl_input_size, 4 * ctrl_output_size]).astype(np.float32))}, 'layer2': {'w': ivy.array(np.random.uniform( -in_wlim, in_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype(np.float32))}}, 'recurrent': {'layer1': {'w': ivy.array(np.random.uniform( -rec_wlim, rec_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype(np.float32))}, 'layer2': {'w': ivy.array(np.random.uniform( -rec_wlim, rec_wlim, size=[ctrl_output_size, 4 * ctrl_output_size]).astype( np.float32))}}} # fully connected proj_wlim = (6 / (total_parameter_num + ctrl_output_size))**0.5 variables['ntm_cell']['controller_proj'] = { 'w': ivy.array( np.random.uniform(-proj_wlim, proj_wlim, size=[total_parameter_num, ctrl_output_size]).astype(np.float32)), 'b': ivy.zeros([total_parameter_num]) } out_wlim = (6 / (total_parameter_num + ctrl_input_size))**0.5 variables['ntm_cell']['output_proj'] = { 'w': ivy.array( np.random.uniform(-out_wlim, out_wlim, size=[ output_dim, ctrl_output_size + read_head_num * memory_vector_dim ]).astype(np.float32)), 'b': ivy.zeros([output_dim]) } # memory wlim = (6 / (2 * memory_vector_dim))**0.5 variables['ntm_cell']['read_weights'] = dict( zip(['w_' + str(i) for i in range(read_head_num)], [ ivy.variable( ivy.array( np.random.uniform(-wlim, wlim, [ memory_vector_dim, ]), 'float32')) for _ in range(read_head_num) ])) wlim = (6 / (2 * memory_size))**0.5 variables['ntm_cell']['write_weights'] = dict( zip(['w_' + str(i) for i in range(read_head_num + write_head_num)], [ ivy.variable( ivy.array(np.random.uniform(-wlim, wlim, [ memory_size, ]), 'float32')) for _ in range(read_head_num + write_head_num) ])) variables['ntm_cell']['memory'] = ivy.variable( ivy.ones([memory_size, memory_vector_dim]) * init_value) # memory object w vars ntm = ivy_mem.NTM(input_dim, output_dim, ctrl_output_size, ctrl_layers, memory_size, memory_vector_dim, read_head_num, write_head_num, Container(variables), usage, addressing_mode=addressing_mode, shift_range=shift_range, clip_value=clip_value, init_value=init_value, sequential_writing=True, retroactive_updates=False, with_erase=False) # test assert np.allclose(call(ntm, x), td.ntm_return[addressing_mode], atol=1e-6) # compilation test if call is helpers.torch_call: # pytest scripting does not support try-catch statements return helpers.assert_compilable(ntm)
def reset(self): self.x = ivy.random_uniform(-0.9, -0.2, [1]) self.x_vel = ivy.zeros([1]) return self.get_observation()
def __init__(self, a_s, d_s, alpha_s, dh_joint_scales, dh_joint_offsets, base_inv_ext_mat=None): """ Initialize robot manipulator instance :param a_s: Denavit–Hartenberg "a" parameters *[num_joints]* :type a_s: array :param d_s: Denavit–Hartenberg "d" parameters *[num_joints]* :type d_s: array :param alpha_s: Denavit–Hartenberg "alpha" parameters *[num_joints]* :type alpha_s: array :param dh_joint_scales: Scalars to apply to input joints *[num_joints]* :type dh_joint_scales: array :param dh_joint_offsets: Scalar offsets to apply to input joints *[num_joints]* :type dh_joint_offsets: array :param base_inv_ext_mat: Inverse extrinsic matrix of the robot base *[4,4]* :type base_inv_ext_mat: array, optional """ self._num_joints = a_s.shape[-1] # ToDo: incorporate the base_inv_ext_mat more elegantly, instead of the hack as in the sample_links method if base_inv_ext_mat is None: self._base_inv_ext_mat = _ivy.identity(4) else: self._base_inv_ext_mat = base_inv_ext_mat # NJ self._dis = d_s self._dh_joint_scales = dh_joint_scales self._dh_joint_offsets = dh_joint_offsets # Forward Kinematics Constant Matrices # Based on Denavit-Hartenberg Convention # Using same nomenclature as in: # Modelling, Planning and Control. Bruno Siciliano, Lorenzo Sciavicco, Luigi Villani, Giuseppe Oriolo # page 61 - 65 AidashtoAis_list = [_ivy.identity(4, batch_shape=[1])] # repeated blocks # 1 x 1 x 3 start_of_top_row = _ivy.array([[[1., 0., 0.]]]) # 1 x 1 x 1 zeros = _ivy.zeros((1, 1, 1)) # 1 x 1 x 4 bottom_row = _ivy.array([[[0., 0., 0., 1.]]]) for i in range(self._num_joints): # 1 x 1 x 1 a_i = _ivy.reshape(a_s[i], [1] * 3) cos_alpha = _ivy.reshape(_ivy.cos(alpha_s[i]), [1] * 3) sin_alpha = _ivy.reshape(_ivy.sin(alpha_s[i]), [1] * 3) # 1 x 1 x 4 top_row = _ivy.concatenate((start_of_top_row, a_i), -1) top_middle_row = _ivy.concatenate((zeros, cos_alpha, -sin_alpha, zeros), -1) bottom_middle_row = _ivy.concatenate((zeros, sin_alpha, cos_alpha, zeros), -1) # 1 x 4 x 4 AidashtoAi = _ivy.concatenate((top_row, top_middle_row, bottom_middle_row, bottom_row), 1) # list AidashtoAis_list.append(AidashtoAi) # NJ x 4 x 4 self._AidashtoAis = _ivy.concatenate(AidashtoAis_list, 0) # Constant Jacobian Params # Using same nomenclature as in: # Modelling, Planning and Control. Bruno Siciliano, Lorenzo Sciavicco, Luigi Villani, Giuseppe Oriolo # page 111 - 113 # 1 x 3 self._z0 = _ivy.array([[0], [0], [1]]) # 1 x 4 self._p0hat = _ivy.array([[0], [0], [0], [1]]) # link lengths # NJ self._link_lengths = (a_s ** 2 + d_s ** 2) ** 0.5
def depth_from_flow_and_cam_mats(flow, full_mats, inv_full_mats=None, camera_centers=None, uniform_pixel_coords=None, triangulation_method='cmp', batch_shape=None, image_dims=None, dev_str=None): """ Compute depth map :math:`\mathbf{X}\in\mathbb{R}^{h×w×1}` in frame 1 using optical flow :math:`\mathbf{U}_{1→2}\in\mathbb{R}^{h×w×2}` from frame 1 to 2, and the camera geometry.\n :param flow: Optical flow from frame 1 to 2 *[batch_shape,h,w,2]* :type flow: array :param full_mats: Full projection matrices *[batch_shape,2,3,4]* :type full_mats: array :param inv_full_mats: Inverse full projection matrices, inferred from full_mats if None and 'cmp' triangulation method *[batch_shape,2,3,4]* :type inv_full_mats: array, optional :param camera_centers: Camera centers, inferred from inv_full_mats if None and 'cmp' triangulation method *[batch_shape,2,3,1]* :type camera_centers: array, optional :param uniform_pixel_coords: Homogeneous uniform (integer) pixel co-ordinate images, inferred from image_dims if None *[batch_shape,h,w,3]* :type uniform_pixel_coords: array, optional :param triangulation_method: Triangulation method, one of [cmp|dlt], for closest mutual points or homogeneous dlt approach, closest_mutual_points by default :type triangulation_method: str, optional :param batch_shape: Shape of batch. Inferred from inputs if None. :type batch_shape: sequence of ints, optional :param image_dims: Image dimensions. Inferred from inputs in None. :type image_dims: sequence of ints, optional :param dev_str: device on which to create the array 'cuda:0', 'cuda:1', 'cpu' etc. Same as x if None. :type dev_str: str, optional :return: Depth map in frame 1 *[batch_shape,h,w,1]* """ if batch_shape is None: batch_shape = flow.shape[:-3] if image_dims is None: image_dims = flow.shape[-3:-1] # shapes as list batch_shape = list(batch_shape) image_dims = list(image_dims) if dev_str is None: dev_str = _ivy.dev_str(flow) if inv_full_mats is None: inv_full_mats = _ivy.inv( _ivy_mech.make_transformation_homogeneous(full_mats, batch_shape + [2], dev_str))[..., 0:3, :] if camera_centers is None: camera_centers = _ivy_svg.inv_ext_mat_to_camera_center(inv_full_mats) if uniform_pixel_coords is None: uniform_pixel_coords = _ivy_svg.create_uniform_pixel_coords_image( image_dims, batch_shape, dev_str=dev_str) # BS x H x W x 3 flow_homo = _ivy.concatenate( (flow, _ivy.zeros(batch_shape + image_dims + [1], dev_str=dev_str)), -1) # BS x H x W x 3 transformed_pixel_coords = uniform_pixel_coords + flow_homo # BS x 2 x H x W x 3 pixel_coords = _ivy.concatenate( (_ivy.expand_dims(uniform_pixel_coords, -4), _ivy.expand_dims(transformed_pixel_coords, -4)), -4) # BS x H x W x 1 return _ivy_tvg.triangulate_depth(pixel_coords, full_mats, inv_full_mats, camera_centers, triangulation_method, batch_shape, image_dims)[..., -1:]
def compute_link_matrices(self, joint_angles, link_num, batch_shape=None): """ Compute homogeneous transformation matrices relative to base frame, up to link_num of links. :param joint_angles: Joint angles of the robot *[batch_shape,num_joints]* :type joint_angles: array :param link_num: Link number for which to compute matrices up to :type link_num: int :param batch_shape: Shape of batch. Inferred from inputs if None. :type batch_shape: sequence of ints, optional :return: The link_num matrices, up the link_num *[batch_shape,link_num,4,4]* """ if batch_shape is None: batch_shape = joint_angles.shape[:-1] batch_shape = list(batch_shape) num_batch_dims = len(batch_shape) # BS x 1 x NJ try: dh_joint_angles = _ivy.expand_dims(joint_angles * self._dh_joint_scales - self._dh_joint_offsets, -2) except: d = 0 # BS x 1 x 4 x 4 A00 = _ivy.identity(4, batch_shape=batch_shape + [1]) Aitoip1dashs = list() Aiip1s = list() A0is = [A00] # repeated blocks # BS x 1 x NJ dis = _ivy.tile(_ivy.reshape(self._dis, [1] * num_batch_dims + [1, self._num_joints]), batch_shape + [1, 1]) # BS x 1 x 4 bottom_row = _ivy.tile( _ivy.reshape(_ivy.array([0., 0., 0., 1.]), [1] * num_batch_dims + [1, 4]), batch_shape + [1, 1]) # BS x 1 x 3 start_of_bottom_middle = _ivy.tile( _ivy.reshape(_ivy.array([0., 0., 1.]), [1] * num_batch_dims + [1, 3]), batch_shape + [1, 1]) # BS x 1 x 2 zeros = _ivy.zeros(batch_shape + [1, 2]) for i in range(self._num_joints): # BS x 1 x 4 top_row = _ivy.concatenate((_ivy.cos(dh_joint_angles[..., i:i + 1]), -_ivy.sin(dh_joint_angles[..., i:i + 1]), zeros), -1) top_middle_row = _ivy.concatenate((_ivy.sin(dh_joint_angles[..., i:i + 1]), _ivy.cos(dh_joint_angles[..., i:i + 1]), zeros), -1) bottom_middle_row = _ivy.concatenate((start_of_bottom_middle, dis[..., i:i + 1]), -1) # BS x 4 x 4 Aitoip1dash = _ivy.concatenate((top_row, top_middle_row, bottom_middle_row, bottom_row), -2) # (BSx4) x 4 Aitoip1dash_flat = _ivy.reshape(Aitoip1dash, (-1, 4)) # (BSx4) x 4 Aiip1_flat = _ivy.matmul(Aitoip1dash_flat, self._AidashtoAis[i + 1]) # BS x 4 x 4 Aiip1 = _ivy.reshape(Aiip1_flat, batch_shape + [4, 4]) # BS x 4 x 4 A0ip1 = _ivy.matmul(A0is[-1][..., 0, :, :], Aiip1) # append term to lists Aitoip1dashs.append(Aitoip1dash) Aiip1s.append(Aiip1) A0is.append(_ivy.expand_dims(A0ip1, -3)) if i + 1 == link_num: # BS x LN x 4 x 4 return _ivy.concatenate(A0is, -3) raise Exception('wrong parameter entered for link_num, please enter integer from 1-' + str(self._num_joints))
def main(batch_size=32, num_train_steps=31250, compile_flag=True, num_bits=8, seq_len=28, ctrl_output_size=100, memory_size=128, memory_vector_dim=28, overfit_flag=False, interactive=True, f=None): f = choose_random_framework() if f is None else f set_framework(f) # train config lr = 1e-3 if not overfit_flag else 1e-2 batch_size = batch_size if not overfit_flag else 1 num_train_steps = num_train_steps if not overfit_flag else 150 max_grad_norm = 50 # logging config vis_freq = 250 if not overfit_flag else 1 # optimizer optimizer = ivy.Adam(lr=lr) # ntm ntm = NTM(input_dim=num_bits + 1, output_dim=num_bits, ctrl_output_size=ctrl_output_size, ctrl_layers=1, memory_size=memory_size, memory_vector_dim=memory_vector_dim, read_head_num=1, write_head_num=1) # compile loss fn total_seq_example = ivy.random_uniform(shape=(batch_size, 2 * seq_len + 1, num_bits + 1)) target_seq_example = total_seq_example[:, 0:seq_len, :-1] if compile_flag: loss_fn_maybe_compiled = ivy.compile_fn( lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn(ntm, v, ttl_sq, trgt_sq, sq_ln), dynamic=False, example_inputs=[ ntm.v, total_seq_example, target_seq_example, seq_len ]) else: loss_fn_maybe_compiled = lambda v, ttl_sq, trgt_sq, sq_ln: loss_fn( ntm, v, ttl_sq, trgt_sq, sq_ln) # init input_seq_m1 = ivy.cast( ivy.random_uniform(0., 1., (batch_size, seq_len, num_bits)) > 0.5, 'float32') mw = None vw = None for i in range(num_train_steps): # sequence to copy if not overfit_flag: input_seq_m1 = ivy.cast( ivy.random_uniform(0., 1., (batch_size, seq_len, num_bits)) > 0.5, 'float32') target_seq = input_seq_m1 input_seq = ivy.concatenate( (input_seq_m1, ivy.zeros((batch_size, seq_len, 1))), -1) eos = ivy.ones((batch_size, 1, num_bits + 1)) output_seq = ivy.zeros_like(input_seq) total_seq = ivy.concatenate((input_seq, eos, output_seq), -2) # train step loss, pred_vals = train_step(loss_fn_maybe_compiled, optimizer, ntm, total_seq, target_seq, seq_len, mw, vw, ivy.array(i + 1, 'float32'), max_grad_norm) # log print('step: {}, loss: {}'.format(i, ivy.to_numpy(loss).item())) # visualize if i % vis_freq == 0: target_to_vis = (ivy.to_numpy(target_seq[0] * 255)).astype( np.uint8) target_to_vis = np.transpose( cv2.resize(target_to_vis, (560, 160), interpolation=cv2.INTER_NEAREST), (1, 0)) pred_to_vis = (ivy.to_numpy(pred_vals[0] * 255)).astype(np.uint8) pred_to_vis = np.transpose( cv2.resize(pred_to_vis, (560, 160), interpolation=cv2.INTER_NEAREST), (1, 0)) img_to_vis = np.concatenate((pred_to_vis, target_to_vis), 0) img_to_vis = cv2.resize(img_to_vis, (1120, 640), interpolation=cv2.INTER_NEAREST) img_to_vis[0:60, -200:] = 0 img_to_vis[5:55, -195:-5] = 255 cv2.putText(img_to_vis, 'step {}'.format(i), (935, 42), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) img_to_vis[0:60, 0:200] = 0 img_to_vis[5:55, 5:195] = 255 cv2.putText(img_to_vis, 'prediction', (7, 42), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) img_to_vis[320:380, 0:130] = 0 img_to_vis[325:375, 5:125] = 255 cv2.putText(img_to_vis, 'target', (7, 362), cv2.FONT_HERSHEY_SIMPLEX, 1.2, tuple([0] * 3), 2) if interactive: cv2.imshow('prediction_and_target', img_to_vis) if overfit_flag: cv2.waitKey(1) else: cv2.waitKey(100) cv2.destroyAllWindows()