def generate(self, x: nd.NDArray = None, include_intermediate: bool = False, return_attn_params: bool = False) -> \
Union[nd.NDArray, Tuple[nd.NDArray, nd.NDArray]]:
"""
Generate a batch of samples from model. See Section 2.3 in paper.
If x is None, this method generates unconditional samples from the model (as explained in Section 2.3 in the
paper).
If x is provided, this method reconstructs the input to generate the sample. This is not really a true sample
from the model because the model looks at the image it is trying to generate. However, this is useful for seeing
how the model generates a particular image. (I believe this is how the figures in the paper are generated.)
:param x: Input to generate images from.
:param include_intermediate: If True, samples from all timesteps (not only the last timestep) are returned.
:param return_attn_params: If True, returns attention params along with generated samples.
:return: n x input dim array of generated samples. If include_intermediate is True, then steps x n x input dim.
"""
canvases = []
attn_params = []
canvas = nd.zeros((self._batch_size, self._input_dim), ctx=self._ctx)
h_dec = nd.broadcast_to(self._dec_rnn_h_init.data(), (self._batch_size, 0))
c_dec = nd.broadcast_to(self._dec_rnn_c_init.data(), (self._batch_size, 0))
if x is not None:
h_enc = nd.broadcast_to(self._enc_rnn_h_init.data(), (self._batch_size, 0))
c_enc = nd.broadcast_to(self._enc_rnn_c_init.data(), (self._batch_size, 0))
for i in range(self._num_steps):
canvases.append(nd.sigmoid(canvas))
if x is not None:
err = x - nd.sigmoid(canvas)
r, _ = self._read_layer(x, err, h_dec, c_dec)
_, (h_enc, c_enc) = self._enc_rnn(nd.concat(r, h_dec, c_dec, dim=1), [h_enc, c_enc])
q = self._enc_dense(h_enc)
z = self._latent_layer(q)
else:
z = nd.random.normal(shape=(self._batch_size, self._latent_dim), ctx=self._ctx)
_, (h_dec, c_dec) = self._dec_rnn(z, [h_dec, c_dec])
w, attn_param = self._write_layer(h_dec, c_dec)
attn_params.append(attn_param)
canvas = canvas + w
canvases.append(nd.sigmoid(canvas))
if include_intermediate:
samples = nd.stack(*canvases, axis=0)
else:
samples = canvases[-1]
if return_attn_params:
return samples, nd.stack(*attn_params, axis=0)
else:
return samples
def compute_scale(self, F, data: Tensor,
observed_indicator: Tensor) -> Tensor:
"""
Parameters
----------
F
A module that can either refer to the Symbol API or the NDArray
API in MXNet.
data
tensor containing the data to be scaled.
observed_indicator
observed_indicator: binary tensor with the same shape as
``data``, that has 1 in correspondence of observed data points,
and 0 in correspondence of missing data points.
Returns
-------
Tensor
shape (N, T, C) or (N, C, T) scaled along the specified axis.
"""
axis_zero = nd.prod(
data == data.zeros_like(), self.axis, keepdims=True
) # Along the specified axis, which array are always at zero
axis_zero = nd.broadcast_to(
axis_zero, shape=data.shape) # Broadcast it to the shape of data
min_val = nd.where(
1 - observed_indicator,
nd.broadcast_to(data.max(keepdims=True), shape=data.shape),
data,
).min(
axis=self.axis, keepdims=True
) # return the min value along specified axis while ignoring value according to observed_indicator
max_val = nd.where(
1 - observed_indicator,
nd.broadcast_to(data.min(keepdims=True), shape=data.shape),
data,
).max(
axis=self.axis, keepdims=True
) # return the max value along specified axis while ignoring value according to observed_indicator
scaled_data = (data - min_val) / (max_val - min_val) # Rescale
scaled_data = nd.where(
axis_zero, scaled_data.zeros_like(), scaled_data
) # Clip Nan values to zero if the data was equal to zero along specified axis
scaled_data = nd.where(
scaled_data != scaled_data, scaled_data.ones_like(), scaled_data
) # Clip the Nan values to one. scaled_date!=scaled_data tells us where the Nan values are in scaled_data
return nd.where(
1 - observed_indicator, scaled_data.zeros_like(), scaled_data
) # Replace data with zero where observed_indicator tells us to.
def hybrid_forward(self, F, fts, ys, ftt, yt):
"""
Semantic Alignment Loss
:param F: Function
:param yt: label for the target domain [N]
:param ftt: features for the target domain [N, K]
:param ys: label for the source domain [M]
:param fts: features for the source domain [M, K]
:return:
"""
if self._fn:
# Normalize ft
fts = F.L2Normalization(fts, mode='instance')
ftt = F.L2Normalization(ftt, mode='instance')
fts_rpt = F.broadcast_to(fts.expand_dims(axis=0),
shape=(self._bs_tgt, self._bs_src,
self._embed_size))
ftt_rpt = F.broadcast_to(ftt.expand_dims(axis=1),
shape=(self._bs_tgt, self._bs_src,
self._embed_size))
dists = F.sum(F.square(ftt_rpt - fts_rpt), axis=2)
yt_rpt = F.broadcast_to(yt.expand_dims(axis=1),
shape=(self._bs_tgt,
self._bs_src)).astype('int32')
ys_rpt = F.broadcast_to(ys.expand_dims(axis=0),
shape=(self._bs_tgt,
self._bs_src)).astype('int32')
y_same = F.equal(yt_rpt, ys_rpt).astype('float32')
y_diff = F.not_equal(yt_rpt, ys_rpt).astype('float32')
intra_cls_dists = dists * y_same
inter_cls_dists = dists * y_diff
max_dists = F.max(dists, axis=1, keepdims=True)
max_dists = F.broadcast_to(max_dists,
shape=(self._bs_tgt, self._bs_src))
revised_inter_cls_dists = F.where(y_same, max_dists, inter_cls_dists)
max_intra_cls_dist = F.max(intra_cls_dists, axis=1)
min_inter_cls_dist = F.min(revised_inter_cls_dists, axis=1)
loss = F.relu(max_intra_cls_dist - min_inter_cls_dist + self._margin)
return loss
def forward(self, x):
# input X is a 3D feature map
self.P = F.batch_dot(
F.broadcast_to(self.weight.data(), shape=(self.gram.shape)),
self.gram.data())
return F.batch_dot(
F.SwapAxis(self.P, 1, 2).broadcast_to(
(x.shape[0], self.C, self.C)),
x.reshape((0, 0, x.shape[2] * x.shape[3]))).reshape(x.shape)
def generate(self,
v_q: nd.NDArray,
x_context: nd.NDArray,
v_context: nd.NDArray,
include_intermediate: bool = False,
**kwargs) -> Union[nd.NDArray, Tuple[nd.NDArray, nd.NDArray]]:
"""
Generate a batch of samples from model. See Algorithm S3 in paper.
:param v_q: Query view camera info.
:param x_context: Context frames.
:param v_context: Context camera info.
:param include_intermediate: If True, samples from all timesteps (not only the last timestep) are returned.
:return: n x *image_shape array of generated samples. If include_intermediate is True,
then steps x n x *image_shape.
"""
u = nd.zeros((self._batch_size, *self._upsample_output_shape),
ctx=self._ctx) # canvas (reconstruction)
h_dec = nd.zeros((self._batch_size, *self._rnn_hidden_shape),
ctx=self._ctx)
c_dec = nd.zeros((self._batch_size, *self._rnn_hidden_shape),
ctx=self._ctx)
# reshape camera information so we can concat it to image data
v_q = nd.broadcast_to(
nd.expand_dims(nd.expand_dims(v_q, axis=-1), axis=-1),
(0, 0, *self._downsample_output_shape[1:]))
outs = [] # sample(s) over time
r = self._representation_nn(x_context, v_context)
for i in range(self._num_steps):
outs.append(self._out_layer(u))
# Eq. S11
p = self._p_layer(h_dec)
p = nd.reshape(p, (self._batch_size, -1))
z = self._latent_layer(p)
gen_z = nd.reshape(z, (self._batch_size, self._num_latent_maps,
*self._downsample_output_shape[1:]))
_, (h_dec, c_dec) = self._gen_rnn(nd.concat(gen_z, v_q, r, dim=1),
[h_dec, c_dec])
u = u + self._upsample_nn(h_dec)
outs.append(self._out_layer(u))
if include_intermediate:
samples = nd.stack(*outs, axis=0)
else:
samples = outs[-1]
return nd.clip(samples, a_min=0.0, a_max=1.0)
def hybrid_forward(self, F, X, gram, weight):
self.P = F.batch_dot(F.broadcast_to(weight, shape=(self.gram.shape)),
gram)
if not isinstance(X, mx.symbol.Symbol):
return F.batch_dot(
F.SwapAxis(self.P, 1, 2).broadcast_to(
(X.shape[0], self.C, self.C)),
X.reshape((0, 0, X.shape[2] * X.shape[3]))).reshape(X.shape)
else:
width = X.slice_axis(axis=2, begin=0, end=0)
width = width.ones_like()
width = width.sum()
height = X.slice_axis(axis=3, begin=0, end=0)
height = width.ones_like()
height = width.sum()
print "width", width
print "height", height
arg_shapes, out_shapes, aux_shapes = X.infer_shape_partial(
data=(1, 3, self.width, self.height)) #1 , RGB, Width, Height
return F.batch_dot(
F.SwapAxis(self.P, 1, 2).broadcast_to(
(out_shapes[0][0], self.C, self.C)),
X.reshape((0, 0, out_shapes[0][2] *
out_shapes[0][3]))).reshape(out_shapes[0])
def forward(self, X):
# input X is a 3D feature map
self.P = F.batch_dot(F.broadcast_to(self.weight.data(), shape=(self.gram.shape)), self.gram.data())
return F.batch_dot(F.SwapAxis(self.P,1,2).broadcast_to((X.shape[0], self.C, self.C)), X.reshape((0,0,X.shape[2]*X.shape[3]))).reshape(X.shape)
