def cycle_back_classification_fast(batch1, batch2):
u, u_indices = batch1
v, v_indices = batch2
len_u = len(u)
len_v = len(v)
t = cupy.arange(len(u))
u_repeat = F.repeat(F.expand_dims(u, 0), len_v, 0).transpose(1, 0, 2)
length1 = -F.sum((u_repeat - v)**2, axis=2)
sim_12 = length1 / u.shape[1]
sim_12 /= CONFIG.softmax_tmp
alpha = F.softmax(sim_12)
v_tilde = F.matmul(alpha, v)
v_tilde_repeat = F.repeat(F.expand_dims(v_tilde, 0), len_u,
0).transpose(1, 0, 2)
length2 = -F.sum((v_tilde_repeat - u)**2, axis=2)
sim_21 = length2 / u.shape[1]
sim_21 /= CONFIG.softmax_tmp
loss = (F.softmax_cross_entropy(sim_21, t))
return loss * len_u
def __call__(self, x, t1, t2):
h1 = F.relu(self.l1(x))
h2 = F.relu(self.l2(x))
b1 = self.b1(self.one1)
b2 = self.b2(self.one2)
b1 = F.expand_dims(F.repeat(b1, h1.shape[0]), axis=1)
b2 = F.expand_dims(F.repeat(b2, h2.shape[0]), axis=1)
loss = F.sum(F.exp(-b1) * (h1 - t1)**2. + b1) + \
F.sum(F.exp(-b2) * (h2 - t2)**2. + b2)
loss = F.mean(loss)
if self.iteration % 100 == 0:
print('loss:', loss.data, ', sigma1:',
F.exp(b1[0]).data[0]**0.5, ', sigma2:',
F.exp(b2[0]).data[0]**0.5)
chainer.reporter.report(
{
'loss': loss,
'b1': float(F.exp(b1[0]).data**0.5),
'b2': float(F.exp(b2[0]).data**0.5),
'Lb1': float(self.l1.b.data),
'Lb2': float(self.l2.b.data),
}, self)
self.iteration += 1
return loss
def outer(a, b):
assert a.shape[0] == b.shape[0]
assert a.ndim == 2
assert b.ndim == 2
M = a.shape[1]
N = b.shape[1]
a = F.repeat(a[:, :, None], N, axis=2)
b = F.repeat(b[:, None, :], M, axis=1)
return a * b
def blend_image(*image_pairs):
result_image = []
for fore, back in zip(image_pairs[::2], image_pairs[1::2]):
fore = utils.unwrapped(fore)
back = utils.unwrapped(back)
if back.shape[1] > fore.shape[1]:
fore = F.repeat(fore, back.shape[1] // fore.shape[1], axis=1)
elif back.shape[1] < fore.shape[1]:
back = F.repeat(back, fore.shape[1] // back.shape[1], axis=1)
result_image.append(normalize(fore) * 0.5 + normalize(back) * 0.5)
return result_image
def forward(self, stories):
"""Compute the forward inference pass for given stories."""
self.log = dict()
# ---------------------------
vctx, vq, va, supps = stories # (B, R, P, C), (B, Q), (B,), (B, I)
# Embed stories
# ectx = F.embed_id(vctx, wordeye, ignore_label=0) # (B, R, P, C, V)
# eq = F.embed_id(vq, wordeye, ignore_label=0) # (B, Q, V)
ectx = self.embed(vctx) # (B, R, P, C, V)
eq = self.embed(vq) # (B, Q, V)
# ---------------------------
# Embed predicates
embedded_preds = seq_rnn_embed(vctx, ectx, self.pred_rnn,
reverse=True) # (B, R, P, E)
vector_preds = vctx[
..., 0] # (B, R, P) first character to check if pred is empty
embedded_query = seq_rnn_embed(vq, eq, self.pred_rnn,
reverse=True) # (B, E)
embedded_rules = embedded_preds[:, :, 0] # (B, R, E) head of rule
# ---------------------------
# Perform iterative updates
state = embedded_query # (B, E)
repeated_query = F.repeat(embedded_query[:, None], vctx.shape[1],
1) # (B, R, E)
rule_mask = np.all(vctx == 0, (2, 3)) # (B, R)
for _ in range(supps.shape[-1]):
# Compute attention over memory
repeated_state = F.repeat(state[:, None], vctx.shape[1],
1) # (B, R, E)
combined = F.concat([
repeated_state, embedded_rules, repeated_query,
F.squared_difference(repeated_state, embedded_rules),
embedded_rules * repeated_state
], -1) # (B, R, 5*E)
att = F.tanh(self.att_dense1(combined,
n_batch_axes=2)) # (B, R, E//2)
att = self.att_dense2(att, n_batch_axes=2) # (B, R, 1)
att = F.squeeze(att, -1) # (B, R)
att += rule_mask * MINUS_INF # (B, R)
self.tolog('raw_att', att)
att = F.softmax(att) # (B, R)
self.tolog('att', att)
# Iterate state
new_states = seq_rnn_embed(
vector_preds,
embedded_preds,
self.unifier,
initial_state=repeated_state) # (B, R, E)
# Update state
# (B, R) x (B, R, E) -> (B, E)
state = F.einsum('br,bre->be', att, new_states) # (B, E)
return self.out_linear(state)[:, 0] # (B,)
def get_all_prob_or_log_prob(self, is_log=False):
segments = self.parametric_segments
num_of_action_types = len(segments)
action_types_logits = self.beta * self.logits[:, :num_of_action_types]
if is_log:
action_types_probs = F.log_softmax(action_types_logits)
else:
action_types_probs = F.softmax(action_types_logits)
# action_types_probs = F.softmax(action_types_logits)
# if is_log:
# print("LOG")
# print(action_types_probs)
# action_types_probs = action_types_probs.data * np.power(self.segments_sizes, 1/4)
# action_types_probs = action_types_probs / np.expand_dims(np.sum(action_types_probs, axis=1), axis=1)
# action_types_probs = chainer.Variable(action_types_probs.astype(np.float32))
# if is_log:
# action_types_probs = F.log(action_types_probs)
# print(action_types_probs)
result = []
logits_offset = num_of_action_types
for i in range(num_of_action_types):
action_type_prob = action_types_probs[:, i:i + 1]
if not segments[i]: # if no parameters for this action type
result.append(action_type_prob)
else:
segments_factor = 1
for sub_seg_size in segments[i]:
segments_factor *= sub_seg_size
if is_log:
sub_seg_probs = F.log_softmax(
self.beta *
self.logits[:, logits_offset:logits_offset +
sub_seg_size])
else:
sub_seg_probs = F.softmax(
self.beta *
self.logits[:, logits_offset:logits_offset +
sub_seg_size])
if is_log:
action_type_prob = F.repeat(
action_type_prob, sub_seg_size, axis=1) + F.tile(
sub_seg_probs, segments_factor // sub_seg_size)
else:
action_type_prob = F.repeat(
action_type_prob, sub_seg_size, axis=1) * F.tile(
sub_seg_probs, segments_factor // sub_seg_size)
logits_offset += sub_seg_size
result.append(action_type_prob)
res = F.concat(tuple(result))
return res
def __call__(self, imgs, questions):
feat = self.feat_extractor(imgs)
# Append relative coordinates to each location in the feature maps.
n, c, h, w = feat.shape
spatial_area = h * w
xp = self.xp
coords_h = xp.linspace(-1, 1, h, dtype=feat.dtype)
coords_w = xp.linspace(-1, 1, w, dtype=feat.dtype)
coords_hh, coords_ww = xp.meshgrid(coords_h, coords_w)
coords_hh = coords_hh[None]
coords_ww = coords_ww[None]
coords = xp.concatenate((coords_hh, coords_ww), axis=0)
coords = coords.reshape(2, -1)
coords = coords[None] # (1, 2, spatial_area * spatial_area)
coords = xp.repeat(coords, n, axis=0)
# Coordinates may be cached here but the performance gain is not
# significant so it is skipped in favor of readability.
feat = feat.reshape(n, c, spatial_area)
h = F.concat((feat, coords), axis=1) # (n, c + 2, spatial_area)
# Create coordinate pairs (differentiable meshgrid).
h_hh = F.expand_dims(h, 2)
h_ww = F.expand_dims(h, 3)
h_hh = F.repeat(h_hh, spatial_area, axis=2)
h_ww = F.repeat(h_ww, spatial_area, axis=3)
h = F.concat((h_hh, h_ww), axis=1)
# Append questions to each coordinate pair.
questions = questions.astype(imgs.dtype)
questions = questions[:, :, None, None]
questions = F.tile(questions, (1, 1, spatial_area, spatial_area))
h = F.concat((h, questions), axis=1)
# (n, (c + 2) * 2 + questions_length, spatial_area, spatial_area)
# g.
h = F.transpose(h, (0, 2, 3, 1))
h = F.reshape(h, (n * spatial_area * spatial_area, -1))
h = self.g(h)
h = F.reshape(h, (n, spatial_area * spatial_area, -1))
h = F.sum(h, axis=1)
h = self.f(h)
# Logits.
h = self.fc(h)
return h
def __call__(self, imgs, questions):
feat = self.feat_extractor(imgs)
# Append relative coordinates to each location in the feature maps.
n, c, h, w = feat.shape
spatial_area = h * w
xp = self.xp
coords_h = xp.linspace(-1, 1, h, dtype=feat.dtype)
coords_w = xp.linspace(-1, 1, w, dtype=feat.dtype)
coords_hh, coords_ww = xp.meshgrid(coords_h, coords_w)
coords_hh = coords_hh[None]
coords_ww = coords_ww[None]
coords = xp.concatenate((coords_hh, coords_ww), axis=0)
coords = coords.reshape(2, -1)
coords = coords[None] # (1, 2, spatial_area * spatial_area)
coords = xp.repeat(coords, n, axis=0)
# Coordinates may be cached here but the performance gain is not
# significant so it is skipped in favor of readability.
feat = feat.reshape(n, c, spatial_area)
h = F.concat((feat, coords), axis=1) # (n, c + 2, spatial_area)
# Create coordinate pairs (differentiable meshgrid).
h_hh = F.expand_dims(h, 2)
h_ww = F.expand_dims(h, 3)
h_hh = F.repeat(h_hh, spatial_area, axis=2)
h_ww = F.repeat(h_ww, spatial_area, axis=3)
h = F.concat((h_hh, h_ww), axis=1)
# Append questions to each coordinate pair.
questions = questions.astype(imgs.dtype)
questions = questions[:, :, None, None]
questions = F.tile(questions, (1, 1, spatial_area, spatial_area))
h = F.concat((h, questions), axis=1)
# (n, (c + 2) * 2 + questions_length, spatial_area, spatial_area)
# g.
h = F.transpose(h, (0, 2, 3, 1))
h = F.reshape(h, (n * spatial_area * spatial_area, -1))
h = self.g(h)
h = F.reshape(h, (n, spatial_area * spatial_area, -1))
h = F.sum(h, axis=1)
h = self.f(h)
# Logits.
h = self.fc(h)
return h
def __call__(self, x, x_=None, **kwargs):
if x_ is None:
x_ = x
if self.k > 1:
x_ = F.repeat(x_, self.k, axis=0)
q_z = self.encode(x, **kwargs)
z = self.sample(q_z)
p_x = self.decode(z, **kwargs)
p_z = self.prior()
# 追加誤差関数
# mse_vel = F.mean_squared_error(x_, p_x.mean)
# mse_vor = F.mean_squared_error(*map(vorticity, (x_, p_x.mean)))
# y = F.sigmoid(p_x.mean)
# mse_vel = self.batch_mean(F.squared_error(*map(logit, (x_, y))))
# mse_vor = self.batch_mean(F.squared_error(*map(vorticity_logit15,
# (x_, y))))
# reporter.report({'mse_vel': mse_vel}, self)
# reporter.report({'mse_vor': mse_vor}, self)
# 誤差関数
reconstr = self.batch_mean(p_x.log_prob(x_))
# reconstr = -self.batch_mean((p_x.mean - x_) ** 2)
kl_penalty = self.batch_mean(chainer.kl_divergence(q_z, p_z))
loss = self.beta * kl_penalty - reconstr
reporter.report({'loss': loss}, self)
reporter.report({'reconstr': reconstr}, self)
reporter.report({'kl_penalty': kl_penalty}, self)
return loss
def return_injected(self, h, z, n_layer, mult_until_exec=None):
""" Performs the Hadamard products with z. """
# # check whether to skip the hadamard.
skip_injection = False
if self.thresh_skip is not None and self.thresh_skip[n_layer - 1] > 0:
# # skip the hadamard, iff the random number is smaller than the threshold.
skip_injection = np.random.uniform() < self.thresh_skip[n_layer -
1]
if not skip_injection and mult_until_exec is not None:
skip_injection = mult_until_exec <= n_layer
if self.mult_lat and not skip_injection:
if self.use_localz:
# # apply local transformation.
z1 = getattr(self, 'locz{}'.format(n_layer))(z)
else:
z1 = z
# # appropriately reshape z for the elementwise multiplication.
sh = h.shape
z1 = F.reshape(z1, (sh[0], sh[1], 1))
if self.normalize_preinject:
z1 /= F.sqrt(F.mean(z1 * z1, axis=1, keepdims=True) + 1e-8)
z2 = F.repeat(z1, sh[3] * sh[2], axis=2)
z2 = F.reshape(z2, sh)
ret = h * z2 + h if self.add_h_injection else h * z2
return ret
return h
def forward(self, equery, vmemory, ememory, mask, iteration=0):
"""Compute an attention over memory given the query."""
# equery.shape == (..., E)
# vmemory.shape == (..., Ms, M)
# ememory.shape == (..., Ms, E)
# mask.shape == (..., Ms)
# Setup memory embedding
eq = F.repeat(equery[..., None, :], vmemory.shape[-2],
-2) # (..., Ms, E)
# Compute content based attention
merged = F.concat(
[eq, ememory, eq * ememory,
F.squared_difference(eq, ememory)], -1) # (..., Ms, 4*E)
inter = self.att_linear(merged, n_batch_axes=len(vmemory.shape) -
1) # (..., Ms, E)
inter = F.tanh(inter) # (..., Ms, E)
inter = F.dropout(inter, DROPOUT) # (..., Ms, E)
# Split into sentences
lengths = np.sum(np.any((vmemory != 0), -1), -1) # (...,)
mems = [s[..., :l, :] for s, l in zip(F.separate(inter, 0), lengths)
] # B x [(M1, E), (M2, E), ...]
_, bimems = self.att_birnn(None,
mems) # B x [(M1, 2*E), (M2, 2*E), ...]
bimems = F.pad_sequence(bimems) # (..., Ms, 2*E)
att = self.att_score(bimems, n_batch_axes=len(vmemory.shape) -
1) # (..., Ms, 1)
att = F.squeeze(att, -1) # (..., Ms)
if mask is not None:
att += mask * MINUS_INF # (..., Ms)
return att
def _decode_multiple(self, s, z=None, decode_num=10):
if z is None:
xp = chainer.backend.get_array_module(s)
z = chainer.Variable(
xp.random.normal(0,
1,
size=(s.shape[0], decode_num,
self._latent_dim)))
z = F.cast(z, typ=xp.float32)
z = F.clip(z, -0.5, 0.5)
s = F.expand_dims(s, axis=0)
s = F.repeat(s, repeats=decode_num, axis=0)
s = F.transpose(s, axes=(1, 0, 2))
x = F.concat((s, z), axis=2)
x = F.reshape(x, shape=(-1, x.shape[-1]))
h = self._linear3(x)
h = F.relu(h)
h = self._linear4(h)
h = F.relu(h)
h = self._linear5(h)
h = F.reshape(h, shape=(-1, decode_num, h.shape[-1]))
return F.tanh(h), h
def _compute_target_q_value(self, batch):
with chainer.using_config('train', False), \
chainer.using_config('enable_backprop', False):
(_, _, r, s_next, non_terminal) = batch
r = F.reshape(r, shape=(*r.shape, 1))
non_terminal = F.reshape(non_terminal,
shape=(*non_terminal.shape, 1))
s_next_rep = F.repeat(x=s_next,
repeats=self._num_action_samples,
axis=0)
a_next_rep = self._vae._decode(s_next_rep)
perturbed_action = self._target_perturbator(s_next_rep, a_next_rep)
q_values = F.stack([
q_target(s_next_rep, perturbed_action)
for q_target in self._target_q_ensembles
])
assert q_values.shape == (self._num_q_ensembles, self._batch_size *
self._num_action_samples, 1)
weighted_q_minmax = self._lambda * F.min(q_values, axis=0) \
+ (1 - self._lambda) * F.max(q_values, axis=0)
assert weighted_q_minmax.shape == (self._batch_size *
self._num_action_samples, 1)
next_q_value = F.max(F.reshape(weighted_q_minmax,
shape=(self._batch_size, -1)),
axis=1,
keepdims=True)
assert next_q_value.shape == (self._batch_size, 1)
target_q_value = r + self._gamma * next_q_value * non_terminal
target_q_value.unchain()
assert target_q_value.shape == (self._batch_size, 1)
return target_q_value
def expand_time_batch(m, time, n_batch):
""" add [time, n_batch] dimension
:param m: input chainer variable
:param time:
:param n_batch:
:return: time times batch times m
"""
# expand two dimensions
m = F.expand_dims(F.expand_dims(m, 0), 0)
# repeat along batch dimension
m = F.repeat(m, n_batch, axis=1)
# repeat along time dimension
m = F.repeat(m, time, axis=0)
assert list(m.shape)[0] == time
assert list(m.shape)[1] == n_batch
return m
def create_2d_window(window_size, channel, xp):
_1D_window = F.reshape(gaussian(window_size, 1.5, xp), (-1, 1))
_2D_window = F.reshape(
F.tensordot(_1D_window, _1D_window.transpose(), axes=1),
(1, 1, -1, window_size))
window = F.repeat(_2D_window, channel, axis=0)
return window
def attention_layer(masked_weights):
h1 = attention_layer1(masked_weights)
attention_layer_output = attention_layer2(h1)
attention_layer_output = F.softmax(attention_layer_output)
print(attention_layer_output)
attention_layer_output = F.repeat(attention_layer_output,
(1, 1, 1, 1, 1, 1),
axis=1)
return masked_weights * attention_layer_output
def attention_layer(masked_weights):
h1 = attention_layer1(masked_weights)
h2 = attention_layer2(h1)
attention_layer_output = attention_layer3(h2)
attention_layer_output = F.softmax(attention_layer_output)
rep_attention_layer_output = F.repeat(attention_layer_output,
(1, 1, 1, 1, 1, 1),
axis=1)
return masked_weights * rep_attention_layer_output, attention_layer_output
def cycle_back_regression_fast(batch1, batch2):
var_lambda = CONFIG.variance_lambda
u, u_indices = batch1
v, v_indices = batch2
len_u = len(u)
len_v = len(v)
t = u_indices
import ipdb
ipdb.set_trace()
if CONFIG.regression.normalize_indices:
t = t / len(u)
u_repeat = F.repeat(F.expand_dims(u, 0), len_v, 0).transpose(1, 0, 2)
length1 = -F.sum((u_repeat - v)**2, axis=2)
sim_12 = length1 / u.shape[1]
sim_12 /= CONFIG.softmax_tmp
alpha = F.softmax(sim_12)
v_tilde = F.matmul(alpha, v)
v_tilde_repeat = F.repeat(F.expand_dims(v_tilde, 0), len_u,
0).transpose(1, 0, 2)
length2 = -F.sum((v_tilde_repeat - u)**2, axis=2)
sim_21 = length2 / u.shape[1]
sim_21 /= CONFIG.softmax_tmp
beta = F.softmax(sim_21)
myu = F.sum(beta * t, axis=1)
var = F.sum(((beta - t)**2 * beta), axis=1)
log_var = F.log(var)
loss = (myu - t)**2 * F.exp(-log_var) + var_lambda * log_var
return F.sum(loss)
def loss(
self,
class_id,
quaternion_true,
translation_true,
quaternion_pred,
translation_pred,
confidence_pred,
):
xp = self.xp
B = class_id.shape[0]
# prepare
quaternion_true = quaternion_true.astype(np.float32)
translation_true = translation_true.astype(np.float32)
loss = 0
for i in range(B):
n_point = quaternion_pred[i].shape[0]
T_cad2cam_pred = morefusion.functions.transformation_matrix(
quaternion_pred[i], translation_pred[i]
) # (M, 4, 4)
T_cad2cam_true = morefusion.functions.transformation_matrix(
quaternion_true[i], translation_true[i]
) # (4, 4)
T_cad2cam_true = F.repeat(T_cad2cam_true[None], n_point, axis=0)
class_id_i = int(class_id[i])
is_symmetric = (
class_id_i in morefusion.datasets.ycb_video.class_ids_symmetric
)
cad_pcd = self._models.get_pcd(class_id=class_id_i)
cad_pcd = xp.asarray(cad_pcd, dtype=np.float32)
add = morefusion.functions.average_distance_l1(
cad_pcd,
T_cad2cam_true,
T_cad2cam_pred,
symmetric=is_symmetric,
)
loss_i = F.mean(
add * confidence_pred[i]
- self._lambda_confidence * F.log(confidence_pred[i])
)
loss += loss_i
loss /= B
values = {"loss": loss}
chainer.report(values, observer=self)
return loss
def expand_batch(m, n_batch):
""" add [n_batch] dimension
:param m: input chainer variable
:param n_batch:
:return: batch times m
"""
# expand two dimensions
m = F.expand_dims(m, 0)
# repeat along batch dimension
m = F.repeat(m, n_batch, axis=0)
assert list(m.shape)[0] == n_batch
return m
def batch_skew(vec, batch_size=None):
"""
vec is N x 3, batch_size is int
returns N x 3 x 3. Skew_sym version of each matrix.
"""
xp = vec.xp
if batch_size is None:
batch_size = vec.shape[0]
col_inds = xp.array([1, 2, 3, 5, 6, 7])
indices = F.reshape(
F.repeat(col_inds.reshape(1, -1), batch_size, axis=0) +
F.repeat(F.reshape(xp.arange(0, batch_size) * 9, [-1, 1]), 6, axis=1),
[-1, 1])
updates = F.reshape(
F.stack(
[-vec[:, 2], vec[:, 1], vec[:, 2], -vec[:, 0], -vec[:, 1],
vec[:, 0]], axis=1), [-1])
res = Variable(xp.zeros((batch_size * 3 * 3), 'f'))
res.data[indices.reshape(-1).data] = updates.data
res = F.reshape(res, [batch_size, 3, 3])
return res
def __call__(self, src_seq, dst_seq):
"""
[[9, 20, 18, ..., ],
[0, 2, 101, ..., ],
...
]
などIDのリストがbatch分くると想定
ただしnp.arrayで、np.int32.
:param src_seq:
:param dst_seq: 先頭に必ずBOSを付与
:return:
出力は(ch, height, width)
"""
batch_input_image = None
for src, dst in zip(src_seq, dst_seq):
# train用
embedded_src_tokens = self.emb_src(src)
embedded_dst_tokens = self.emb_dst(dst)
input_src_array = F.reshape(embedded_src_tokens, (-1, len(src), 1))
input_dst_array = F.reshape(embedded_dst_tokens, (-1, 1, len(dst)))
input_src_array = F.repeat(input_src_array, len(dst), axis=2)
input_dst_array = F.repeat(input_dst_array, len(src), axis=1)
concat = F.reshape(
F.concat([input_src_array, input_dst_array], axis=0),
(1, -1, len(src), len(dst)))
if not batch_input_image:
batch_input_image = concat
else:
batch_input_image = F.concat([batch_input_image, concat],
axis=0)
return batch_input_image
def batch_skew(vec, batch_size=None):
"""
vec is N x 3, batch_size is int
returns N x 3 x 3. Skew_sym version of each matrix.
"""
xp = vec.xp
if batch_size is None:
batch_size = vec.shape[0]
col_inds = xp.array([1, 2, 3, 5, 6, 7])
indices = F.reshape(
F.repeat(col_inds.reshape(1, -1), batch_size, axis=0) +
F.repeat(F.reshape(xp.arange(0, batch_size) * 9, [-1, 1]), 6, axis=1),
[-1, 1])
updates = F.reshape(
F.stack(
[-vec[:, 2], vec[:, 1], vec[:, 2], -vec[:, 0], -vec[:, 1],
vec[:, 0]], axis=1), [-1])
res = Variable(xp.zeros((batch_size * 3 * 3), 'f'))
res.data[indices.reshape(-1).data] = updates.data
res = F.reshape(res, [batch_size, 3, 3])
return res
def __call__(self, x, mask):
xp = cuda.get_array_module(x)
# point-wise feauture
pwf = self.fcn(x)
# locally aggregated feature
laf = F.repeat(F.expand_dims(F.max(pwf, axis=2), 2), self.T, axis=2)
# point-wise concat feature
pwcf = F.concat((pwf, laf), axis=-1)
# apply mask
mask = xp.expand_dims(mask, -1).repeat(self.units * 2, axis=-1)
pwcf *= mask
return pwcf
def forward(self, x):
b_ = cf.tanh(self.b)
# Not sure if implementation is wrong
m_ = cf.softplus(self.m)
# m = cf.repeat(m, 8, axis=2)
# m = cf.repeat(m, 8, axis=1)
m_ = cf.repeat(m_, 16, axis=2)
m_ = cf.repeat(m_, 16, axis=1)
b_ = b_ * m_
x_ = cf.add(x, b_)
x_ = cf.clip(x_, -0.5, 0.5)
z = []
zs, logdet = self.encoder.forward_step(x_)
for (zi, mean, ln_var) in zs:
z.append(zi)
z = merge_factorized_z(z)
return z, zs, logdet, xp.sum(xp.abs(b_.data)), xp.tanh(
self.b.data * 1), m_, x_
def gradient_correlation(x, t, normalize=True, absolute=False):
assert x.ndim == t.ndim
xp = chainer.cuda.get_array_module(x)
n_grad_dim = x.ndim - 2
gc = []
for i in range(n_grad_dim):
kernel_shape = tuple(np.roll((3, ) + (1, ) * (n_grad_dim - 1),
shift=i))
w = xp.array([-1.0, 0.0, 1.0]).reshape((
1,
1,
) + kernel_shape)
x_grad = F.convolution_nd(x, w)
t_grad = F.convolution_nd(t, w)
x_grad_mean = F.mean(x_grad, axis=tuple(range(1, x.ndim)))
t_grad_mean = F.mean(t_grad, axis=tuple(range(1, x.ndim)))
repeat_shape = (1, ) + x_grad.shape[1:]
x_grad_mean = F.reshape(
F.repeat(x_grad_mean,
functools.reduce(lambda x, y: x * y, repeat_shape)),
x_grad.shape)
t_grad_mean = F.reshape(
F.repeat(t_grad_mean,
functools.reduce(lambda x, y: x * y, repeat_shape)),
x_grad.shape)
x_norm_grad = x_grad - x_grad_mean
t_norm_grad = t_grad - t_grad_mean
gc.append(1.0 - F.sum(x_norm_grad * t_norm_grad) /
(F.sqrt(F.sum(x_norm_grad**2)) *
F.sqrt(F.sum(t_norm_grad**2)) + 1e-8))
return F.absolute(sum(gc) / len(gc))
def __call__(self,
task_encoding=None,
image_encoding=None,
data_out=None,
seed=None,
return_sample=False,
train=True):
with chainer.using_config('train', train), chainer.using_config(
'enable_backprop', train):
xp = cuda.cupy
sequence_len = len(image_encoding)
x = image_encoding
y = data_out
if self.AUTO_REGRESSIVE:
y = F.swapaxes(y, 0, 1)
y = F.reshape(y, (y.shape[0], y.shape[1] * y.shape[2], 1))
y = F.swapaxes(y, 0, 1)
x = F.swapaxes(x, 0, 1)
x = F.repeat(x, (1, self.OUT_DIM, 1))
x = F.swapaxes(x, 0, 1)
sequence_len = x.shape[0]
cost = 0
# if type(seed) == Variable:
# last_joint = seed
# else:
last_joint = Variable(xp.zeros(y.shape[1:], dtype=np.float32))
sample_toRet = np.zeros(self.OUT_DIM, dtype=np.float32)
for j in range(sequence_len):
batch_cost, sample = self.process_batch(
x[j],
y[j],
task_encoding,
last_joint,
return_sample=return_sample)
last_joint = y[j]
if self.AUTO_REGRESSIVE:
sample_toRet[j % self.OUT_DIM] = cuda.to_cpu(
sample.data)[0, 0]
cost += batch_cost
cost /= x.shape[0]
if self.AUTO_REGRESSIVE:
sample = sample_toRet
return cost, sample
def read(self, k, b):
# k: (1, M_DIM*kr), b: (1, kr)
kr = b.shape[1]
K = k.reshape(kr, M_DIM)
C = F.matmul(F.normalize(K + EPS),
F.transpose(F.normalize(self.M + EPS)))
B = F.repeat(b, N_mem).reshape(kr, N_mem) # beta
if kr == Kr:
self.W_predictor = F.softmax(B *
C) # B*C: elementwise multiplication
M = F.matmul(self.W_predictor, self.M)
elif kr == Krp:
self.W_policy = F.softmax(B * C)
M = F.matmul(self.W_policy, self.M)
else:
raise (ValueError)
return M.reshape((1, -1))
def average_distance(points, transform_true, transforms_pred, symmetric=False):
"""Translation introduced pose_loss proposed in DenseFusion paper.
Original pose_loss looks like below:
.. math::
m = |M|
PLOSS(\\tilde{q}, q) =
1 / m \\sum_{x \\in M} | R(\\tilde{q})x - R(q)x |
where M is set of point_xyz, q~ and q are quaternion,
R(q~) and R(q) are rotation matrix of ground truth and predicted,
and m is size of the set M.
If we introduce translation here, it will be:
.. math::
PLOSS2(\\tilde{T}, T) =
1 / m \\sum_{x \\in M} | \\tilde{T}x - Tx |
"""
n_points = points.shape[0]
n_pred = transforms_pred.shape[0]
assert points.shape == (n_points, 3)
assert transform_true.shape == (4, 4)
assert transforms_pred.shape == (n_pred, 4, 4)
points_true = transform_points(points, transform_true)
points_pred = transform_points(points, transforms_pred)
assert points_true.shape == (n_points, 3)
assert points_pred.shape == (n_pred, n_points, 3)
if symmetric:
ref = points_true.array
query = points_pred.array.reshape(n_pred * n_points, 3)
indices = geometry_module.nn(ref, query)
points_true = points_true[indices]
points_true = points_true.reshape(n_pred, n_points, 3)
else:
points_true = F.repeat(points_true[None], n_pred, axis=0)
return F.mean(
F.sqrt(F.sum((points_true - points_pred) ** 2, axis=2)), axis=1
)
def prepare(self, images, downscale=1):
if self.use_hist_eq:
# Histogram equalization
images = images.astype(int, copy=False)
values, counts = self.xp.unique(images, return_counts=True)
p = self.xp.zeros(256, dtype=int)
p[values] = counts
d1 = 255 / images.size * p.cumsum().astype(np.float32, copy=False)
images = d1[images]
images = F.repeat(images, 3, axis=1)
if downscale > 1:
output_shape = [x // downscale for x in images.shape[2:]]
images = F.resize_images(images, tuple(output_shape))
images -= self.xp.array([103.063, 115.903, 123.152],
dtype=np.float32).reshape(1, 3, 1, 1)
return images
def __call__(self, h_rgb, pcd):
B, _, n_point = h_rgb.shape
# conv1
h_rgb = F.relu(self.conv1_rgb(h_rgb))
h_pcd = F.relu(self.conv1_pcd(pcd))
feat1 = F.concat((h_rgb, h_pcd), axis=1)
# conv2
h_rgb = F.relu(self.conv2_rgb(h_rgb))
h_pcd = F.relu(self.conv2_pcd(h_pcd))
feat2 = F.concat((h_rgb, h_pcd), axis=1)
# conv3, conv4
h = F.relu(self.conv3(feat2))
h = F.relu(self.conv4(h))
h = F.average_pooling_1d(h, n_point)
h = h.reshape((B, 1024, 1))
feat3 = F.repeat(h, n_point, axis=2)
feat = F.concat((feat1, feat2, feat3), axis=1)
return feat
def __call__(self, h_img, pcd):
B = h_img.shape[0]
# conv1
h_img = F.relu(self.conv1_img(h_img))
h_pcd = F.relu(self.conv1_pcd(pcd))
feat1 = F.concat((h_pcd, h_img), axis=1)
# conv2
h_img = F.relu(self.conv2_img(h_img))
h_pcd = F.relu(self.conv2_pcd(h_pcd))
feat2 = F.concat((h_pcd, h_img), axis=1)
# conv3, conv4
h = F.relu(self.conv3(feat2))
h = F.relu(self.conv4(h))
h = F.average_pooling_1d(h, self.n_point)
h = h.reshape((B, 1024, 1))
feat3 = F.repeat(h, self.n_point, axis=2)
feat = F.concat((feat1, feat2, feat3), axis=1)
return feat
def __call__(self, beta, theta, get_skin=False, with_a=False):
batch_size = beta.shape[0]
# 1. Add shape blend shapes
# (N x 10) x (10 x 6890*3) = N x 6890 x 3
self.beta_shapedirs = F.matmul(beta, self.shapedirs)
v_shaped = F.reshape(
F.matmul(beta, self.shapedirs),
[-1, self.size[0], self.size[1]]) + \
F.repeat(self.v_template[None, ], batch_size, axis=0)
self.v_shaped = v_shaped
# 2. Infer shape-dependent joint locations.
Jx = F.matmul(v_shaped[:, :, 0], self.J_regressor)
Jy = F.matmul(v_shaped[:, :, 1], self.J_regressor)
Jz = F.matmul(v_shaped[:, :, 2], self.J_regressor)
J = F.stack([Jx, Jy, Jz], axis=2)
self.J = J
# 3. Add pose blend shapes
# N x 24 x 3 x 3
Rs = F.reshape(
batch_rodrigues(F.reshape(theta, [-1, 3])), [-1, 24, 3, 3])
self.Rs = Rs
# Ignore global rotation.
pose_feature = F.reshape(Rs[:, 1:, :, :] -
F.repeat(F.repeat(Variable(self.xp.array(self.xp.eye(3), 'f'))[
None, ], 23, axis=0)[None, ], batch_size, axis=0),
[-1, 207])
self.pose_feature = pose_feature
# (N x 207) x (207, 20670) -> N x 6890 x 3
v_posed = F.reshape(
F.matmul(pose_feature, self.posedirs),
[-1, self.size[0], self.size[1]]) + v_shaped
# 4. Get the global joint location
self.J_transformed, A = batch_global_rigid_transformation(
Rs, J, self.parents)
# 5. Do skinning:
# W is N x 6890 x 24
W = F.reshape(
F.tile(self.weights, (batch_size, 1)), [batch_size, -1, 24])
# (N x 6890 x 24) x (N x 24 x 16)
T = F.reshape(
F.matmul(W, F.reshape(A, [batch_size, 24, 16])),
[batch_size, -1, 4, 4])
v_posed_homo = F.concat(
[v_posed, self.xp.ones([batch_size, v_posed.shape[1], 1], 'f')], 2)
v_homo = F.matmul(T, F.expand_dims(v_posed_homo, -1))
verts = v_homo[:, :, :3, 0]
# Get cocoplus or lsp joints:
joint_x = F.matmul(verts[:, :, 0], self.joint_regressor)
joint_y = F.matmul(verts[:, :, 1], self.joint_regressor)
joint_z = F.matmul(verts[:, :, 2], self.joint_regressor)
joints = F.stack([joint_x, joint_y, joint_z], axis=2)
return verts, joints, Rs, A
def test_value_error(self):
x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32)
with self.assertRaises(ValueError):
functions.repeat(x, self.repeats, self.axis)
def test_type_error_axis_bool(self):
x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32)
with self.assertRaises(TypeError):
functions.repeat(x, 1, True)
def f(x):
return functions.repeat(x, self.repeats, self.axis)
def test_type_error_repeats_str(self):
x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32)
with self.assertRaises(TypeError):
functions.repeat(x, 'a')
def f(x):
y = functions.repeat(x, self.repeats, self.axis)
return y * y
def forward(self, inputs, devices):
x, = inputs
y = functions.repeat(x, self.repeats, self.axis)
return y,
def check_forward(self, x_data):
y = functions.repeat(x_data, self.repeats, self.axis)
y_expected = self._repeat(self.x, self.repeats, self.axis)
self.assertEqual(y.dtype, y_expected.dtype)
testing.assert_allclose(
y.data, y_expected, **self.check_forward_options)
