def generate(self, x: nd.NDArray = None, include_intermediate: bool = False, return_attn_params: bool = False) -> \ Union[nd.NDArray, Tuple[nd.NDArray, nd.NDArray]]: """ Generate a batch of samples from model. See Section 2.3 in paper. If x is None, this method generates unconditional samples from the model (as explained in Section 2.3 in the paper). If x is provided, this method reconstructs the input to generate the sample. This is not really a true sample from the model because the model looks at the image it is trying to generate. However, this is useful for seeing how the model generates a particular image. (I believe this is how the figures in the paper are generated.) :param x: Input to generate images from. :param include_intermediate: If True, samples from all timesteps (not only the last timestep) are returned. :param return_attn_params: If True, returns attention params along with generated samples. :return: n x input dim array of generated samples. If include_intermediate is True, then steps x n x input dim. """ canvases = [] attn_params = [] canvas = nd.zeros((self._batch_size, self._input_dim), ctx=self._ctx) h_dec = nd.broadcast_to(self._dec_rnn_h_init.data(), (self._batch_size, 0)) c_dec = nd.broadcast_to(self._dec_rnn_c_init.data(), (self._batch_size, 0)) if x is not None: h_enc = nd.broadcast_to(self._enc_rnn_h_init.data(), (self._batch_size, 0)) c_enc = nd.broadcast_to(self._enc_rnn_c_init.data(), (self._batch_size, 0)) for i in range(self._num_steps): canvases.append(nd.sigmoid(canvas)) if x is not None: err = x - nd.sigmoid(canvas) r, _ = self._read_layer(x, err, h_dec, c_dec) _, (h_enc, c_enc) = self._enc_rnn(nd.concat(r, h_dec, c_dec, dim=1), [h_enc, c_enc]) q = self._enc_dense(h_enc) z = self._latent_layer(q) else: z = nd.random.normal(shape=(self._batch_size, self._latent_dim), ctx=self._ctx) _, (h_dec, c_dec) = self._dec_rnn(z, [h_dec, c_dec]) w, attn_param = self._write_layer(h_dec, c_dec) attn_params.append(attn_param) canvas = canvas + w canvases.append(nd.sigmoid(canvas)) if include_intermediate: samples = nd.stack(*canvases, axis=0) else: samples = canvases[-1] if return_attn_params: return samples, nd.stack(*attn_params, axis=0) else: return samples
def compute_scale(self, F, data: Tensor, observed_indicator: Tensor) -> Tensor: """ Parameters ---------- F A module that can either refer to the Symbol API or the NDArray API in MXNet. data tensor containing the data to be scaled. observed_indicator observed_indicator: binary tensor with the same shape as ``data``, that has 1 in correspondence of observed data points, and 0 in correspondence of missing data points. Returns ------- Tensor shape (N, T, C) or (N, C, T) scaled along the specified axis. """ axis_zero = nd.prod( data == data.zeros_like(), self.axis, keepdims=True ) # Along the specified axis, which array are always at zero axis_zero = nd.broadcast_to( axis_zero, shape=data.shape) # Broadcast it to the shape of data min_val = nd.where( 1 - observed_indicator, nd.broadcast_to(data.max(keepdims=True), shape=data.shape), data, ).min( axis=self.axis, keepdims=True ) # return the min value along specified axis while ignoring value according to observed_indicator max_val = nd.where( 1 - observed_indicator, nd.broadcast_to(data.min(keepdims=True), shape=data.shape), data, ).max( axis=self.axis, keepdims=True ) # return the max value along specified axis while ignoring value according to observed_indicator scaled_data = (data - min_val) / (max_val - min_val) # Rescale scaled_data = nd.where( axis_zero, scaled_data.zeros_like(), scaled_data ) # Clip Nan values to zero if the data was equal to zero along specified axis scaled_data = nd.where( scaled_data != scaled_data, scaled_data.ones_like(), scaled_data ) # Clip the Nan values to one. scaled_date!=scaled_data tells us where the Nan values are in scaled_data return nd.where( 1 - observed_indicator, scaled_data.zeros_like(), scaled_data ) # Replace data with zero where observed_indicator tells us to.
def hybrid_forward(self, F, fts, ys, ftt, yt): """ Semantic Alignment Loss :param F: Function :param yt: label for the target domain [N] :param ftt: features for the target domain [N, K] :param ys: label for the source domain [M] :param fts: features for the source domain [M, K] :return: """ if self._fn: # Normalize ft fts = F.L2Normalization(fts, mode='instance') ftt = F.L2Normalization(ftt, mode='instance') fts_rpt = F.broadcast_to(fts.expand_dims(axis=0), shape=(self._bs_tgt, self._bs_src, self._embed_size)) ftt_rpt = F.broadcast_to(ftt.expand_dims(axis=1), shape=(self._bs_tgt, self._bs_src, self._embed_size)) dists = F.sum(F.square(ftt_rpt - fts_rpt), axis=2) yt_rpt = F.broadcast_to(yt.expand_dims(axis=1), shape=(self._bs_tgt, self._bs_src)).astype('int32') ys_rpt = F.broadcast_to(ys.expand_dims(axis=0), shape=(self._bs_tgt, self._bs_src)).astype('int32') y_same = F.equal(yt_rpt, ys_rpt).astype('float32') y_diff = F.not_equal(yt_rpt, ys_rpt).astype('float32') intra_cls_dists = dists * y_same inter_cls_dists = dists * y_diff max_dists = F.max(dists, axis=1, keepdims=True) max_dists = F.broadcast_to(max_dists, shape=(self._bs_tgt, self._bs_src)) revised_inter_cls_dists = F.where(y_same, max_dists, inter_cls_dists) max_intra_cls_dist = F.max(intra_cls_dists, axis=1) min_inter_cls_dist = F.min(revised_inter_cls_dists, axis=1) loss = F.relu(max_intra_cls_dist - min_inter_cls_dist + self._margin) return loss
def forward(self, x): # input X is a 3D feature map self.P = F.batch_dot( F.broadcast_to(self.weight.data(), shape=(self.gram.shape)), self.gram.data()) return F.batch_dot( F.SwapAxis(self.P, 1, 2).broadcast_to( (x.shape[0], self.C, self.C)), x.reshape((0, 0, x.shape[2] * x.shape[3]))).reshape(x.shape)
def generate(self, v_q: nd.NDArray, x_context: nd.NDArray, v_context: nd.NDArray, include_intermediate: bool = False, **kwargs) -> Union[nd.NDArray, Tuple[nd.NDArray, nd.NDArray]]: """ Generate a batch of samples from model. See Algorithm S3 in paper. :param v_q: Query view camera info. :param x_context: Context frames. :param v_context: Context camera info. :param include_intermediate: If True, samples from all timesteps (not only the last timestep) are returned. :return: n x *image_shape array of generated samples. If include_intermediate is True, then steps x n x *image_shape. """ u = nd.zeros((self._batch_size, *self._upsample_output_shape), ctx=self._ctx) # canvas (reconstruction) h_dec = nd.zeros((self._batch_size, *self._rnn_hidden_shape), ctx=self._ctx) c_dec = nd.zeros((self._batch_size, *self._rnn_hidden_shape), ctx=self._ctx) # reshape camera information so we can concat it to image data v_q = nd.broadcast_to( nd.expand_dims(nd.expand_dims(v_q, axis=-1), axis=-1), (0, 0, *self._downsample_output_shape[1:])) outs = [] # sample(s) over time r = self._representation_nn(x_context, v_context) for i in range(self._num_steps): outs.append(self._out_layer(u)) # Eq. S11 p = self._p_layer(h_dec) p = nd.reshape(p, (self._batch_size, -1)) z = self._latent_layer(p) gen_z = nd.reshape(z, (self._batch_size, self._num_latent_maps, *self._downsample_output_shape[1:])) _, (h_dec, c_dec) = self._gen_rnn(nd.concat(gen_z, v_q, r, dim=1), [h_dec, c_dec]) u = u + self._upsample_nn(h_dec) outs.append(self._out_layer(u)) if include_intermediate: samples = nd.stack(*outs, axis=0) else: samples = outs[-1] return nd.clip(samples, a_min=0.0, a_max=1.0)
def hybrid_forward(self, F, X, gram, weight): self.P = F.batch_dot(F.broadcast_to(weight, shape=(self.gram.shape)), gram) if not isinstance(X, mx.symbol.Symbol): return F.batch_dot( F.SwapAxis(self.P, 1, 2).broadcast_to( (X.shape[0], self.C, self.C)), X.reshape((0, 0, X.shape[2] * X.shape[3]))).reshape(X.shape) else: width = X.slice_axis(axis=2, begin=0, end=0) width = width.ones_like() width = width.sum() height = X.slice_axis(axis=3, begin=0, end=0) height = width.ones_like() height = width.sum() print "width", width print "height", height arg_shapes, out_shapes, aux_shapes = X.infer_shape_partial( data=(1, 3, self.width, self.height)) #1 , RGB, Width, Height return F.batch_dot( F.SwapAxis(self.P, 1, 2).broadcast_to( (out_shapes[0][0], self.C, self.C)), X.reshape((0, 0, out_shapes[0][2] * out_shapes[0][3]))).reshape(out_shapes[0])
def forward(self, X): # input X is a 3D feature map self.P = F.batch_dot(F.broadcast_to(self.weight.data(), shape=(self.gram.shape)), self.gram.data()) return F.batch_dot(F.SwapAxis(self.P,1,2).broadcast_to((X.shape[0], self.C, self.C)), X.reshape((0,0,X.shape[2]*X.shape[3]))).reshape(X.shape)