def test_with_owner(): a = L.Attend(Owner()) a.build([(4, 10, 16), (), (4, 18, 16), ()]) src = torch.constant([0.0] * (4 * 10 * 16), shape=(4, 10, 16)) bias = torch.constant([0.0] * (4 * 10), shape=(4, 10)) bias = torch.expand_dims(torch.expand_dims(bias, axis=1), axis=3) mem = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16)) ctx = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16)) a.call([src, bias, mem, ctx])
def test_owner_none(): a = L.Attend(Owner()) a.build([(4, 10, 16)]) src = torch.constant([0.0] * (4 * 10 * 16), shape=(4, 10, 16)) a.call([src]) bias = torch.constant([0.0] * (4 * 10), shape=(4, 10)) bias = torch.expand_dims(torch.expand_dims(bias, axis=1), axis=3) a.call([src, bias]) ctx = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16)) a.call([src, bias, None, ctx])
def sensemap_model(x, sensemap, name="sensemap_model", do_transpose=False): """Apply sensitivity maps.""" if do_transpose: x_shape = x.get_shape().as_list() x = torch.expand_dims(x, axis=-2) x = torch.multiply(torch.conj(sensemap), x) x = torch.sum(x, axis=-1) else: x = torch.expand_dims(x, axis=-1) x = torch.multiply(x, sensemap) x = torch.sum(x, axis=3) return x
def min_distance_better_than_threshold(pred_control_points, gt_control_points, confidence, confidence_threshold, device="cpu"): error = torch.expand_dims(pred_control_points, 1) - torch.expand_dims( gt_control_points, 0) error = torch.sum(torch.abs(error), -1) # L1 distance of error (N_pred, N_gt, M) error = torch.mean( error, -1) # average L1 for all the control points. (N_pred, N_gt) error = torch.min(error, -1) # (B, N_pred) mask = torch.greater_equal(confidence, confidence_threshold) mask = torch.squeeze(mask, dim=-1) return torch.mean(error[mask]), torch.mean(mask)
def call(self, q_head, k_head_h, v_head_h, k_head_r, seg_embed, seg_mat, r_w_bias, r_r_bias, r_s_bias, attn_mask): # content based attention score ac = torch.einsum('ibnd,jbnd->ijbn', q_head + r_w_bias, k_head_h) # position based attention score bd = torch.einsum('ibnd,jbnd->ijbn', q_head + r_r_bias, k_head_r) bd = rel_shift(bd, klen=tf.shape(ac)[1]) # segment-based attention score if seg_mat is None: ef = 0 else: ef = torch.einsum('ibnd,snd->isbn', q_head + r_s_bias, seg_embed) tgt_shape = torch.shape(bd) ef = torch.where( torch.Tensor( np.broadcast_to(torch.expand_dims(seg_mat, 3), tgt_shape)), torch.Tensor(np.broadcast_to(ef[:, 1:, :, :], tgt_shape)), torch.Tensor(np.broadcast_to(ef[:, :1, :, :], tgt_shape))) # merges attention scores and performs masking attn_score = (ac + bd + ef) * self.scale if attn_mask is not None: attn_score = attn_score - 1e30 * attn_mask # attention probability attn_prob = functional.softmax(attn_score, 1) attn_prob = self.attention_probs_dropout(attn_prob) # attention output attn_vec = torch.einsum('ijbn,jbnd->ibnd', attn_prob, v_head_h)
def Fusion(closeness_output, period_output, trend_output, scope, shape): ''' Combining the output from the module into one tec map ''' closeness_output = torch.squeeze(closeness_output) period_output = torch.squeeze(period_output) trend_output = torch.squeeze(trend_output) # apply a linear transformation to each of the outputs: closeness, period, trend and then combine Wc = torch.get_variable("closeness_matrix", dtype=torch.float32, shape=shape, initializer=torch.contrib.layers.xavier_initializer(), trainable=True) Wp = torch.get_variable("period_matrix", dtype=torch.float32, shape=shape, initializer=torch.contrib.layers.xavier_initializer(), trainable=True) Wt = torch.get_variable("trend_matrix", dtype=torch.float32, shape=shape, initializer=torch.contrib.layers.xavier_initializer(), trainable=True) output = torch.reshape(closeness_output, [closeness_output.shape[0]*closeness_output.shape[1], closeness_output.shape[2]]) output = torch.matmul(output, Wc) closeness_output = torch.reshape(output, [closeness_output.shape[0], closeness_output.shape[1], closeness_output.shape[2]]) output = torch.reshape(period_output, [period_output.shape[0]*period_output.shape[1], period_output.shape[2]]) output = torch.matmul(output, Wp) period_output = torch.reshape(output, [period_output.shape[0], period_output.shape[1], period_output.shape[2]]) output = torch.reshape(trend_output, [trend_output.shape[0]*trend_output.shape[1], trend_output.shape[2]]) output = torch.matmul(output, Wt) trend_output = torch.reshape(output, [trend_output.shape[0], trend_output.shape[1], trend_output.shape[2]]) # fusion outputs = torch.add(torch.add(closeness_output, period_output), trend_output) # adding non-linearity outputs = torch.F.tanh(outputs) # converting the dimension from (B, H, W) -> (B, H, W, 1) to match ground truth labels outputs = torch.expand_dims(outputs, axis=3) return outputs
def get_dists(X): """Keras code to compute the pairwise distance matrix for a set of vectors specifie by the matrix X. """ x2 = torch.expand_dims(torch.sum(torch.power(X, 2), axis=1), 1) dists = x2 + torch.transpose(x2) - 2 * torch.dot(X, torch.transpose(X)) return dists
def gpi(self, observation, cumulant_weights): q_values = self.__call__(th.expand_dims(observation, axis=0))[0] q_w = th.tensordot(q_values, cumulant_weights, axes=[1, 0]) # [P,a] q_w_actions = th.reduce_max(q_w, axis=0) action = th.cast(th.argmax(q_w_actions), th.int32) return action
def convert_to_tensorboard_image_shape(image): """Ensures an image has shape [B, H, W, C], used for visualizing latents.""" image_shape = image.get_shape().as_list() if image_shape[-1] not in [1, 3]: image = torch.expand_dims(image, -1) image_shape = image.get_shape().as_list() output_image_shape = (4 - len(image_shape)) * [1] + image_shape if output_image_shape[0] is None: output_image_shape[0] = -1 image = torch.reshape(image, output_image_shape) return image
def axis_angle_to_rotation_matrix(axis, angle): B = angle.size(0) z = angle[:, 2] zeros = z.detach() * 0 ones = zeros.detach() + 1 Mat1 = torch.cat([ zeros, -torch.expand_dims(torch.expand_dims(axis[:, 2], -1), -1), torch.expand_dims(torch.expand_dims(axis[:, 1], -1), -1) ], axis=2) Mat2 = torch.cat([ zeros, zeros, -torch.expand_dims(torch.expand_dims(axis[:, 0], -1), -1) ], axis=2) Mat3 = torch.cat([zeros, zeros, zeros], axis=2) Mat = torch.cat([Mat1, Mat2, Mat3], axis=1) cp_axis = Mat - torch.transpose(Mat, perm=[0, 2, 1]) RotMat = torch.eye(3, batch_shape=[B]) + torch.sin(angle) * cp_axis + ( ones - torch.cose(angle)) * torch.matmaul(cp_axis, cp_axis) return RotMat
def append_tok(self, idx, i, **kw): cfg = self.cfg k = 2 * cfg.beam_size b = torch.range(cfg.batch_size * k) // k b = torch.reshape(b, (cfg.batch_size, k)) beam = idx // cfg.num_toks sel = torch.stack([b, beam], axis=2) y = torch.gather_nd(self.tgt, sel) ii = torch.constant([i] * cfg.batch_size * k) ii = torch.reshape(ii, (cfg.batch_size, k)) sel = torch.stack([b, beam, ii], axis=2) u = torch.expand_dims(idx % cfg.num_toks, axis=2) tgt = torch.tensor_scatter_nd_update(y, sel, u) return tgt
def top_logp(self, ctx, bias, i): cfg = self.cfg y = torch.zeros(( cfg.batch_size, cfg.beam_size, cfg.num_toks, )) y += torch.expand_dims(self.logp, axis=2) b = torch.range(cfg.batch_size) ii = torch.constant([i] * cfg.batch_size) for j in range(cfg.beam_size): jj = torch.constant([j] * cfg.batch_size) sel = torch.stack([b, jj, ii]) yj = self.to_logp(self.tgt[:, j, :], ctx, bias, i)[1] y = torch.tensor_scatter_nd_add(y, sel, yj) y = torch.reshape(y, (-1, cfg.beam_size * cfg.num_toks)) logp, idx = torch.top_k(y, k=2 * cfg.beam_size) return logp, idx
def propagate(self, X, full_cov=False, S=1, zs=None): sX = bf.tile(th.expand_dims(X, 0), [S, 1, 1]) Fs, Fmeans, Fvars = [], [], [] F = sX zs = zs or [ None, ] * len(self.layers) for layer, z in zip(self.layers, zs): F, Fmean, Fvar = layer.sample_from_conditional(F, z=z, full_cov=full_cov) Fs.append(F) Fmeans.append(Fmean) Fvars.append(Fvar) return Fs, Fmeans, Fvars
def relative_logits_1d(q, rel_k, H, W, Nh, transpose_mask): """Compute relative logits along one dimension.""" """Need to document inputs to make sure we test right""" """ q is B * H * W * Nh rel_k is from tf.get_variable and is of shape 2*H-1, dk/Nh H / W / Nh are ints and we transpose positions """ rel_logits = torch.einsum('bhxyd,md->bhxym', q, rel_k) # Collapse height and heads rel_logits = torch.reshape(rel_logits, [-1, Nh * H, W, 2 * W - 1]) rel_logits = rel_to_abs(rel_logits) # Shape it and tile height times rel_logits = torch.reshape(rel_logits, [-1, Nh, H, W, W]) rel_logits = torch.expand_dims(rel_logits, axis=3) rel_logits = torch.tile(rel_logits, [1, 1, 1, H, 1, 1]) # Reshape for adding to the logits. rel_logits = torch.transpose(rel_logits, transpose_mask) rel_logits = torch.reshape(rel_logits, [-1, Nh, H * W, H * W]) return rel_logits
def merge_pc_and_gripper_pc(pc, gripper_pc, instance_mode=0, pc_latent=None, gripper_pc_latent=None): """ Merges the object point cloud and gripper point cloud and adds a binary auxilary feature that indicates whether each point belongs to the object or to the gripper. """ pc_shape = pc.shape gripper_shape = gripper_pc.shape assert (len(pc_shape) == 3) assert (len(gripper_shape) == 3) assert (pc_shape[0] == gripper_shape[0]) npoints = pc.shape[1] batch_size = pc.shape[0] if instance_mode == 1: assert pc_shape[-1] == 3 latent_dist = [pc_latent, gripper_pc_latent] latent_dist = torch.cat(latent_dist, 1) l0_xyz = torch.cat((pc, gripper_pc), 1) labels = [ torch.ones((pc.shape[1], 1), dtype=torch.float32), torch.zeros((gripper_pc.shape[1], 1), dtype=torch.float32) ] labels = torch.cat(labels, 0) labels = torch.expand_dims(labels, 0) labels = torch.tile(labels, [batch_size, 1, 1]) if instance_mode == 1: l0_points = torch.cat([l0_xyz, latent_dist, labels], -1) else: l0_points = torch.cat([l0_xyz, labels], -1) return l0_xyz, l0_points
def search(self, tgt, ctx, i=None): cfg = self.cfg unk = torch.equal(tgt, cfg.UNK) prior = torch.one_hot(tgt, cfg.num_toks, 0.0, utils.big_neg) if i is not None: unk = unk[:, i] prior = prior[:, i, :] if torch.reduce_all(unk) is True: logi = prior else: y = self.decode(tgt, ctx) if i is not None: y = y[:, i, :] sh = y.shape # torch.int_shape(y) y = torch.reshape(y, (-1, sh[-1])) y = self.logits(y) y = torch.reshape(y, sh[:-1] + y.shape[-1:]) u = torch.expand_dims(unk, axis=2) u = torch.broadcast_to(u, y.shape) logi = torch.where(u, y, prior) logp = y - torch.reduce_logsumexp(y, axis=-1, keepdims=True) return logp, logi, unk
def quaternion_rotate(pc, q, inverse=False): """rotates a set of 3D points by a rotation, represented as a quaternion Args: pc: [B,N,3] point cloud q: [B,4] rotation quaternion Returns: q * pc * q' """ q_norm = q.norm(p=2, dim=-1).reshape(q.shape[0], 1) # TODO: Detach or not, the denominatior q = torch.div(q, q_norm) q = q.reshape(q.shape[0], 1, q.shape[1]) # [B,1,4] q_ = quaternion_conjugate(q) qmul = quaternion_multiply if not inverse: wxyz = qmul(qmul(q, pc), q_) # [B,N,4] else: wxyz = qmul(qmul(q_, pc), q) # [B,N,4] if len(wxyz.shape) == 2: # bug with batch size of 1 wxyz = torch.expand_dims(wxyz, axis=0) xyz = wxyz[:, :, 1:4] # [B,N,3] return xyz
def images_to_grid(images, grid_height=4, grid_width=4, image_border_value=0.5): """Combine images and arrange them in a grid. Args: images: Tensor of shape [B, H], [B, H, W], or [B, H, W, C]. grid_height: Height of the grid of images to output, or None. Either `grid_width` or `grid_height` must be set to an integer value. If None, `grid_height` is set to ceil(B/`grid_width`), and capped at `max_grid_height` when provided. grid_width: Width of the grid of images to output, or None. Either `grid_width` or `grid_height` must be set to an integer value. If None, `grid_width` is set to ceil(B/`grid_height`), and capped at `max_grid_width` when provided. max_grid_height: Maximum allowable height of the grid of images to output, or None. Only used when `grid_height` is None. max_grid_width: Maximum allowable width of the grid of images to output, or None. Only used when `grid_width` is None. image_border_value: None or scalar value of greyscale borderfor images. If None, then no border is rendered. Raises: ValueError: if neither of grid_width or grid_height are set to a positive integer. Returns: images: Tensor of shape [height*H, width*W, C]. C will be set to 1 if the input was provided with no channels. Contains all input images in a grid. """ # If only one dimension is set, infer how big the other one should be. images = images[:grid_height * grid_width, ...] # Pad with extra blank frames if grid_height x grid_width is less than the # number of frames provided. pre_images_shape = images.get_shape().as_list() if pre_images_shape[0] < grid_height * grid_width: pre_images_shape[0] = grid_height * grid_width - pre_images_shape[0] if image_border_value is not None: dummy_frames = image_border_value * torch.ones( shape=pre_images_shape, dtype=images.dtype) else: dummy_frames = torch.zeros(shape=pre_images_shape, dtype=images.dtype) images = torch.concat([images, dummy_frames], axis=0) if image_border_value is not None: images = _pad_images(images, image_border_value=image_border_value) images_shape = images.get_shape().as_list() images = torch.reshape(images, [grid_height, grid_width] + images_shape[1:]) if len(images_shape) == 2: images = torch.expand_dims(images, -1) if len(images_shape) <= 3: images = torch.expand_dims(images, -1) image_height, image_width, channels = images.get_shape().as_list()[2:] images = torch.transpose(images, perm=[0, 2, 1, 3, 4]) images = torch.reshape( images, [grid_height * image_height, grid_width * image_width, channels]) return images
def forward(self, h, x, presence=None): batch_size, n_input_points = int(x.shape[0]), int(x.shape[0]) self.vote_shape = [batch_size, self._n_caps, self._n_votes, 6] res = self.capsule(h) # The paper literally just grabs these coordinates from our OV * OP # composition. Constructing a two-dimensional vote. res.vote = res.vote[..., :-1, -1] # Reshape dimensions for constellation specific use case. def pool_dim(x, dim_begin, dim_end): combined_shape = list(x.shape[:dim_begin]) + [-1] + list( x.shape[dim_end:]) return x.view(combined_shape) for k, v in res.items(): print("raw out : ", v.shape) if k == "vote" or k == "scale": res[k] = pool_dim(v, 1, 3) if k == "vote_presence": print("ROSA PARKS: ", v.shape) res[k] = pool_dim(v, 1, 3) print("ROSA PARKS: ", res[k].shape) likelihood = _capsule.OrderInvariantCapsuleLikelihood( self._n_votes, res.vote, res.scale, res.vote_presence) ll_res = likelihood(x, presence) # # Mixing KL div. # soft_layer = torch.nn.Softmax(dim=1) mixing_probs = soft_layer(ll_res.mixing_logits) prior_mixing_log_prob = math.scalar_log(1. / n_input_points) mixing_kl = mixing_probs * \ (ll_res.mixing_log_prob - prior_mixing_log_prob) mixing_kl = torch.mean(torch.sum(mixing_kl, -1)) # # Sparsity loss. # from_capsule = ll_res.is_from_capsule # torch implementation of tf.one_hot idx = torch.eye(self._n_caps) wins_per_caps = torch.stack([ idx[from_capsule[b].type(torch.LongTensor)] for b in range(from_capsule.shape[0]) ]) if presence is not None: wins_per_caps *= torch.expand_dims(presence, -1) wins_per_caps = torch.sum(wins_per_caps, 1) has_any_wins = torch.gt(wins_per_caps, 0).float() should_be_active = torch.gt(wins_per_caps, 1).float() # https://pytorch.org/docs/stable/generated/torch.nn.MultiLabelSoftMarginLoss.html # From math, looks to be same as `tf.nn.sigmoid_cross_entropy_with_logits`. # TODO: not rigorous cross-implementation softmargin_loss = torch.nn.MultiLabelSoftMarginLoss() sparsity_loss = softmargin_loss(should_be_active, res.pres_logit_per_caps) # sparsity_loss = tf.reduce_sum(sparsity_loss * has_any_wins, -1) # sparsity_loss = tf.reduce_mean(sparsity_loss) caps_presence_prob = torch.max( torch.reshape(res.vote_presence, [batch_size, self._n_caps, self._n_votes]), 2)[0] # # Constructing loss ensemble. # print(torch.mean(res.scale)) return EasyDict(mixing_kl=mixing_kl, sparsity_loss=sparsity_loss, caps_presence_prob=caps_presence_prob, mean_scale=torch.mean(res.scale))
def forward(self, inputs): #if not (self.built): # self.build(inputs.shape) # inputs = torch.ops.convert_to_tensor(inputs, dtype=self.dtype) print(inputs.dtype, self.kernel.dtype, self.dendriticW.dtype) # if not (inputs.dtype == self.dendriticW.dtype): # print("casting") # inputs = torch.cast(inputs, dtype=self.dendriticW.dtype) print('input shape', inputs.shape) if self.weight_twice: # each dendrit COULD have unique weight for each input, meaning Wshape=[input,dendrite,units] output = inputs.unsqueeze(-1) print(output.dtype) print(output.shape, self.kernel.shape) if self.uniqueW: output = torch.multiply(output, self.kernel) else: output = torch.tensordot(output, self.kernel, dims=([ -1, ], [ 0, ])) print(output.shape, 'first weighting') print(self.bias.shape) if self.use_bias: output += self.bias # torch.transpose(output + self.bias) print(output.shape, 'bias1') else: output = torch.transpose(inputs) # units,x,batch # loopv1 # condition= lambda inn,hold: torch.less(inn,self.input_shapes[-1]) # looper=lambda inn,hold : [torch.add(inn,incr), torch.unsorted_segment_sum(output[inn],self.dendrites[inn],self.seql)] # output=torch.while_loop(condition,looper,[ix,hold]) # output=torch.stack(output[1]) if self.version == 3: if self.weight_twice: output = torch.unstack(output) output = torch.stack([ torch.unsorted_segment_sum(data, self.dendrites[i], self.seql) for i, data in enumerate(output) ]) else: output = torch.stack([ torch.unsorted_segment_sum(output, seq, self.seql) for seq in self.dendrites ]) if self.version == 2: if self.weight_twice: print(self.debuildshape) output = torch.reshape(output, self.debuildshape) output = torch.unsorted_segment_sum( output, torch.reshape(self.dendrites, self.deseqshape), self.num_id) output = torch.reshape(output, self.rebuildshape) print(torch.transpose(output).shape, self.rebuildshape) else: print(self.debuildshape) output = torch.matmul( torch.expand_dims(output, 0), torch.ones( (self.units, *[1 for _ in range(self.len_input)]), dtype=output.dtype)) output = torch.reshape(torch.transpose(output), self.debuildshape) output = torch.unsorted_segment_sum( output, torch.reshape(self.dendrites, self.deseqshape), self.num_id) output = torch.reshape(output, self.rebuildshape) # print(torch.transpose(output).shape, self.rebuildshape) if self.version == 1: output = self.dendritic_op(output, ) # too much squashing print(output.shape, 'unsorted shape', self.dendriticW.shape, output.dtype, self.dendriticW.dtype) output = torch.mul(output, self.dendriticW) # matmul # output = torch.tensordot(output, self.dendriticW,)# dims=([[-1,],[0,]])) else: print(output.shape, self.dendriticW.shape) output = torch.mul( output, self.dendriticW ) # perfect since it's elementwise and not dot product print(output.shape, '2w shape', self.dendriticB.shape) if self.use_bias: output += self.dendriticB print(output.shape, '2b shape') output = torch.sum(output, -2) # sum the dendrites if self.activation is not None: return self.activation(output) print('outshap is {}'.format(output.shape)) # print('GOOD OUTPUT SHAPE') if output.shape==(*self.input_shapes[:-1],self.units) else print("BAD OUTPUT SHAPE") return (output)
def test_expand_dims_variable(self): var = Variable(torch.range(0, 9).view(-1, 2)) ret = torch.expand_dims(var, 0) expected = var.view(1, -1, 2) assert ret.size() == expected.size()
def test_expand_dims_tensor(self): var = torch.range(0, 9).view(-1, 2) ret = torch.expand_dims(var, 0) expected = var.view(1, -1, 2) assert ret.size() == expected.size()