def calc_op(a, x, pos, mode):
onehotter = ["BI", "GD", "PA", "A", "P", "R", "A2"]
h = 0.
for cand in range(ncand):
if mode in onehotter and float(a[pos, cand].data) == 0.:
skip = True
else:
c = x.data.copy()
if BINOMIAL:
c[:, cand + 1:] = 0.
c = xp.sum(c, axis=1)
#c /= (cand+1.)
else:
c = c[:, cand]
#if cand == 0:
# c[:,0] *= 0.
c = c.reshape((batch_size, 1, 4, 4))
h += F.scale(c, a[pos, cand])
if mode == "GD" and float(a[pos, cand].data) == 0.:
z = 1. + (0. * x.data[:, 0].reshape((batch_size, 1, 4, 4)))
h += F.scale(z, a[pos, cand])
return h
def __call__(self, x):
# Initialize peephole weights
if self.peephole and self.peep_c_i.array is None:
self.initialize_params(x.shape)
# Initialize state
if self.c is None:
self.initialize_state(x.shape)
xifoc = self.w_xifoc(x)
xi, xf, xo, xc = F.split_axis(xifoc, 4, axis=1)
hifoc = self.w_hifoc(self.h)
hi, hf, ho, hc = F.split_axis(hifoc, 4, axis=1)
ci = F.sigmoid(xi + hi + (
F.scale(self.c, self.peep_c_i, 1) if self.peephole else 0))
cf = F.sigmoid(xf + hf + (
F.scale(self.c, self.peep_c_f, 1) if self.peephole else 0))
cc = cf * self.c + ci * F.tanh(xc + hc)
co = F.sigmoid(xo + ho +
(F.scale(cc, self.peep_c_o, 1) if self.peephole else 0))
ch = co * F.tanh(cc)
self.c = cc
self.h = ch
return ch
def test_scale_invalid_shape(self):
x1 = chainer.Variable(numpy.zeros((3, 2, 3), numpy.float32))
x2 = chainer.Variable(numpy.zeros((2), numpy.float32))
axis = 0
with chainer.using_config('debug', True):
with self.assertRaises(AssertionError):
functions.scale(x1, x2, axis)
def test_scale_invalid_shape(self):
x1 = chainer.Variable(numpy.zeros((3, 2, 3), numpy.float32))
x2 = chainer.Variable(numpy.zeros((2), numpy.float32))
axis = 0
with chainer.using_config('debug', True):
with self.assertRaises(AssertionError):
functions.scale(x1, x2, axis)
def __call__(self, x):
f = self.extract(x)
x_recon = self.reconstruct(f)
loss = F.mean_absolute_error(F.scale(
x, self.mask), F.scale(x_recon, self.mask)) * self.loss_const
report({'loss': loss}, self)
return loss
def write_weighting(allocation_gate, write_gate, allocation_weighting,
write_content_weighting):
""" (batch_size,) -> (batch_size,) -> (batch_size,n_locations) -> (batch_size,n_locations) -> (batch_size,n_locations) """
ga, gw, a, c = allocation_gate, write_gate, allocation_weighting, write_content_weighting
r = F.scale((F.scale(a, ga, axis=0) + F.scale(c, 1 - ga, axis=0)),
gw,
axis=0)
return r
def read_weightings(read_modes, backward_weightings, read_content_weightings,
forward_weightings):
""" (batch_size,n_read_heads,3) -> (batch_size,n_read_heads,n_locations) -> (batch_size,n_read_heads,n_locations) -> (batch_size,n_read_heads,n_locations) -> (batch_size,n_read_heads,n_locations) """
pi, b, c, f = read_modes, backward_weightings, read_content_weightings, forward_weightings
x = F.scale(b, pi[:, :, 0], axis=0)
y = F.scale(c, pi[:, :, 1], axis=0)
z = F.scale(f, pi[:, :, 2], axis=0)
return x + y + z
def test_forward(self):
x = np.ones(shape=(1, 32, 50, 38), dtype=np.float32)
out_channels = 16
shape = x.shape
target_shape = (shape[0], out_channels, shape[2], shape[3])
initial_c = chainer.Variable(np.ones(target_shape, dtype=np.float32))
initial_h = chainer.Variable(np.ones(target_shape, dtype=np.float32))
conv_lstm = links.ConvLSTM(in_channels=32,
out_channels=out_channels,
ksize=5)
conv_lstm.reset_state(initial_c, initial_h)
test_Wci = chainer.Parameter(
np.random.rand(out_channels, shape[2],
shape[3]).astype(np.float32))
test_Wcf = chainer.Parameter(
np.random.rand(out_channels, shape[2],
shape[3]).astype(np.float32))
test_Wco = chainer.Parameter(
np.random.rand(out_channels, shape[2],
shape[3]).astype(np.float32))
conv_lstm.Wci = chainer.Parameter(
np.reshape(test_Wci.data, (1, out_channels, shape[2], shape[3])))
conv_lstm.Wcf = chainer.Parameter(
np.reshape(test_Wcf.data, (1, out_channels, shape[2], shape[3])))
conv_lstm.Wco = chainer.Parameter(
np.reshape(test_Wco.data, (1, out_channels, shape[2], shape[3])))
Wxi = conv_lstm.Wxi
Wxf = conv_lstm.Wxf
Wxo = conv_lstm.Wxo
Wxc = conv_lstm.Wxc
Whi = conv_lstm.Whi
Whf = conv_lstm.Whf
Who = conv_lstm.Who
Whc = conv_lstm.Whc
Wci = test_Wci
Wcf = test_Wcf
Wco = test_Wco
it = F.sigmoid(
Wxi(x) + Whi(initial_h) + F.scale(initial_c, Wci, axis=1))
ft = F.sigmoid(
Wxf(x) + Whf(initial_h) + F.scale(initial_c, Wcf, axis=1))
ct = ft * initial_c + it * F.tanh(Wxc(x) + Whc(initial_h))
ot = F.sigmoid(Wxo(x) + Who(initial_h) + F.scale(ct, Wco, axis=1))
expected = ot * F.tanh(ct)
actual = conv_lstm(x)
assert actual.data == pytest.approx(expected.data)
def C(M, k, beta):
# "_" means length of mini-batch.
# M : _ * N * W -- this type of comments indicate a shape of variable.
# k : _ * W
# beta : _ * 1
# -> _ * N
denominator = F.sqrt(
F.scale(F.sum(M * M, 2), F.batch_l2_norm_squared(k), 0) + 1e-5)
D = F.scale(F.reshape(F.batch_matmul(M, k), (M.shape[0], M.shape[1])),
1 / denominator, 0)
return F.softmax(D)
def get_context(self, attention: Variable):
context = F.sum(F.scale(F.broadcast_to(
self.encoded, attention.shape + (self.encoder_output_size, )),
attention,
axis=0),
axis=1)
return context
def __call__(self, x):
if self.Wci.data is None:
self.initialize_params(x.data.shape)
if self.pc is None:
self.initialize_state(x.data.shape)
ci = F.sigmoid(self.wxi(x) + self.whi(self.ph) + F.scale(self.pc, self.Wci, 1))
cf = F.sigmoid(self.wxf(x) + self.whf(self.ph) + F.scale(self.pc, self.Wcf, 1))
cc = cf * self.pc + ci * F.tanh(self.wxc(x) + self.whc(self.ph))
co = F.sigmoid(self.wxo(x) + self.who(self.ph) + F.scale(cc, self.Wco, 1))
ch = co * F.tanh(cc)
self.pc = cc
self.ph = ch
return ch
def __call__(self, x):
# chainer requires explicit broadcast for avoiding latent bugs
u = F.mean(x, -1, keepdims=True)
u = F.broadcast_to(u, x.shape)
s = F.mean((x - u) ** 2, -1, keepdims=True)
s = F.broadcast_to(s, x.shape)
x = (x - u) / F.sqrt(s + self.e)
return F.bias(F.scale(x, self.g, axis=2), self.b, axis=2)
def encode_decode_train(self,
in_word_list,
out_word_list,
train=True,
sample=False):
xp = cuda.cupy if self.gpuid >= 0 else np
self.reset_state()
# Add GO_ID, EOS_ID to decoder input
decoder_word_list = [GO_ID] + out_word_list + [EOS_ID]
# encode list of words/tokens
enc_states = self.encode_list(in_word_list, train=train)
# initialize decoder LSTM to final encoder state
self.set_decoder_state()
# decode and compute loss
# convert list of tokens into chainer variable list
var_dec = (Variable(xp.asarray(decoder_word_list,
dtype=np.int32).reshape((-1, 1)),
volatile=not train))
# Initialise first decoded word to GOID
pred_word = Variable(xp.asarray([GO_ID], dtype=np.int32),
volatile=not train)
# compute loss
self.loss = 0
# decode tokens
for next_word_var in var_dec[1:]:
self.decode(pred_word, train=train)
if self.attn == NO_ATTN:
predicted_out = self.out(self[self.lstm_dec[-1]].h)
else:
''' __QUESTION Add attention '''
prevh = self[self.lstm_dec[-1]].h
alpha = F.softmax(matmul(prevh, enc_states, transb=True))
ctxt = F.reshape(
M.sum(F.scale(enc_states, F.transpose(alpha), axis=0),
axis=0), (1, 200))
predicted_out = self.out(self.attn_out(F.concat(
(ctxt, prevh))))
# compute loss
prob = F.softmax(predicted_out)
pred_word = self.select_word(prob, train=train, sample=False)
# pred_word = Variable(xp.asarray([pred_word.data], dtype=np.int32), volatile=not train)
'''
___QUESTION-1-DESCRIBE-E-START___
Explain what loss is computed with an example. What does this value mean?
The cross-entropy is a soft measure of how close the network got to the
correct answer. Here it is used to find how close the predicted word
(predicted_out) was to the expected word (next_word_var).
'''
self.loss += F.softmax_cross_entropy(predicted_out, next_word_var)
'''___QUESTION-1-DESCRIBE-E-END___'''
report({"loss": self.loss}, self)
return self.loss
def __call__(self, batch):
"""
Input
-----
batch: Input Variable of shape [N, hidden_dim]
"""
# calc mini-batch squared mean
nu = F.mean(F.square(batch), axis=0)
# Normalize
sig_hat = F.rsqrt(F.bias(nu, self.eps))
activated = F.scale(batch, sig_hat)
# shift
shift = F.bias(F.scale(activated, self.gamma), self.beta)
# TRU
return F.maximum(shift, self.tau)
def memory_retention_vector(free_gates, read_weightings_prev):
""" (batch_size,n_read_heads) -> (batch_size,n_read_heads,n_locations) -> (batch_size,n_locations) """
batch_size, n_read_heads = free_gates.shape
n_locations = read_weightings_prev.shape[2]
t = 1 - F.scale(read_weightings_prev, free_gates, axis=0)
r = var_fones((batch_size, n_locations))
for i in range(n_read_heads):
r *= t[:, i, :]
return r
def forward(self, X):
if self.WCi.data is None:
self.init_params(X.data.shape)
if self.pc is None:
self.init_state(X.data.shape)
i_t = F.sigmoid(
self.WXi(X) + self.WHi(self.ph) + F.scale(self.pc, self.WCi, 1))
f_t = F.sigmoid(
self.WXf(X) + self.WHf(self.ph) + F.scale(self.pc, self.WCf, 1))
C_t = f_t * self.pc + i_t * F.tanh(self.WXc(X) + self.WHc(self.ph))
o_t = F.sigmoid(
self.WXo(X) + self.WHo(self.ph) + F.scale(C_t, self.WCo, 1))
H_t = o_t * F.tanh(C_t)
self.pc = C_t
self.ph = H_t
return H_t
def __call__(self, x, attentions):
am, an, ay, ax = attentions.shape
ret_pred = []
for i in range(self.att_num):
item_attention = attentions[:, i].reshape(am, 1, ay, ax)
sh = F.scale(x, item_attention, axis=0)
sh = self.res5(sh)
sh = F.average_pooling_2d(sh, 7, stride=1)
sh = self['att{}'.format(i)](sh)
ret_pred.append(sh)
return ret_pred
def content_based_addressing(memory, keys, strengths):
""" (M,n_locations,width) -> (M,N,width) -> (M,N) -> (M,N,n_locations) """
M, n_locations, width = memory.shape
N = keys.shape[1]
m, k, s = memory, keys, strengths
m = F.reshape(F.normalize(F.reshape(m, (-1, width))),
(M, n_locations, width))
k = F.reshape(F.normalize(F.reshape(k, (-1, width))), (M, N, width))
t = F.scale(F.batch_matmul(k, m, transb=True), s, axis=0)
r = F.reshape(F.softmax(F.reshape(t, (-1, n_locations))),
(M, N, n_locations))
return r
def __init__(self, q_dist, z_values, q_values_formatter=lambda x: x):
assert isinstance(q_dist, chainer.Variable)
assert not isinstance(z_values, chainer.Variable)
assert q_dist.ndim == 3
assert z_values.ndim == 1
assert q_dist.shape[2] == z_values.shape[0]
self.xp = cuda.get_array_module(q_dist.array)
self.z_values = z_values
self.q_values = F.sum(F.scale(q_dist, self.z_values, axis=2), axis=2)
self.q_dist = q_dist
self.n_actions = q_dist.array.shape[1]
self.q_values_formatter = q_values_formatter
def decode(self):
arr_sum = None
a = None
for i, model in enumerate(self.models):
output = model.chainer_model.decode()
if i == 0:
arr_sum = output.y
if hasattr(output, "a"): a = output.a
else:
arr_sum += output.y
prob = F.scale(
arr_sum, nmtrain.environment.Variable(self.normalization_constant))
return nmtrain.models.decoders.Output(y=prob, a=a)
def Cr(M, k, beta):
# M : _ * N * W
# k : _ * R * W
# beta : _ * R
# -> _ * R * N
[batch_size, R, N] = k.shape
M_R = F.broadcast_to(F.reshape(M, (M.shape[0], 1, M.shape[1], M.shape[2])),
(batch_size, R, M.shape[1], M.shape[2]))
# calculate denominator of D
M_R_sum = F.sum(M_R * M_R, 3)
k_sum = F.sum(k * k, 2)
k_sum = F.reshape(k_sum, (batch_size, R, 1))
denominator = F.sqrt(F.scale(M_R_sum, k_sum, 0) + 1e-5)
cross = F.sum(
M_R * F.broadcast_to(F.reshape(k, (M.shape[0], R, 1, N)), M_R.shape),
3)
D = cross / denominator
# calculate softmax
D_exp = F.exp(D)
D_denominator = F.sum(D_exp, 2)
return F.scale(D_exp, 1 / F.reshape(D_denominator, (batch_size, R, 1)), 0)
def __call__(self, hx, cx, lemmas, poses, deps, dirs, counts):
ls_f = sequence_embed(self.lstm.lemma_embed, lemmas)
ps_f = sequence_embed(self.lstm.pos_embed, poses)
ds_f = sequence_embed(self.lstm.dep_embed, deps)
dirs_f = sequence_embed(self.lstm.dir_embed, dirs)
paths_f = [F.concat((ls_f[i], ps_f[i], ds_f[i], dirs_f[i]), axis=1) for i in range(len(lemmas))]
hy, cy, ys = self.lstm(hx, cx, paths_f)
hy = F.scale(hy[self.n_layers - 1], counts, axis=0)
hy = F.sum(hy, axis=0) / F.sum(counts).data
last_hidden = hy
return last_hidden
def decoder_predict(self,
start_word,
enc_states,
max_predict_len=MAX_PREDICT_LEN,
sample=False):
xp = cuda.cupy if self.gpuid >= 0 else np
# __QUESTION -- Following code is to assist with ATTENTION
# alpha_arr should store the alphas for every predicted word
alpha_arr = xp.empty((0, enc_states.shape[0]), dtype=xp.float32)
# return list of predicted words
predicted_sent = []
# load start symbol
with chainer.no_backprop_mode():
pred_word = Variable(xp.asarray([start_word], dtype=np.int32))
pred_count = 0
# start prediction loop
while pred_count < max_predict_len and (int(pred_word.data) !=
(EOS_ID)):
self.decode(pred_word, train=False)
if self.attn == NO_ATTN:
prob = F.softmax(self.out(self[self.lstm_dec[-1]].h))
else:
# __QUESTION Add attention
score = F.matmul(self[self.lstm_dec[-1]].h, enc_states.T)
at = F.softmax(score)
ct = F.reshape(
F.sum(F.scale(enc_states, at.T, axis=0), axis=0),
(1, enc_states.shape[1]))
av = F.tanh(
self.avout(
F.concat((ct, self[self.lstm_dec[-1]].h), axis=1)))
prob = F.softmax(self.out(av))
alpha_arr = xp.append(alpha_arr, at.array, axis=0)
pred_word = self.select_word(prob, train=False, sample=sample)
# add integer id of predicted word to output list
predicted_sent.append(int(pred_word.data))
pred_count += 1
# __QUESTION Add attention
# When implementing attention, make sure to use alpha_array to store
# your attention vectors.
# The visualisation function in nmt_translate.py assumes such an array as input.
return predicted_sent, alpha_arr
def decoder_predict(self,
start_word,
enc_states,
max_predict_len=MAX_PREDICT_LEN,
sample=False):
xp = cuda.cupy if self.gpuid >= 0 else np
# __QUESTION -- Following code is to assist with ATTENTION
# alpha_arr should store the alphas for every predicted word
alpha_arr = xp.empty((0, enc_states.shape[0]), dtype=xp.float32)
# return list of predicted words
predicted_sent = []
# load start symbol
pred_word = Variable(xp.asarray([start_word], dtype=np.int32),
volatile=True)
pred_count = 0
# start prediction loop
while pred_count < max_predict_len and (int(pred_word.data) !=
(EOS_ID)):
self.decode(pred_word, train=False)
if self.attn == NO_ATTN:
predicted_out = self.out(self[self.lstm_dec[-1]].h)
else:
''' __QUESTION Add attention '''
prevh = self[self.lstm_dec[-1]].h
alpha = F.softmax(matmul(prevh, enc_states, transb=True))
ctxt = F.reshape(
M.sum(F.scale(enc_states, F.transpose(alpha), axis=0),
axis=0), (1, 200))
alpha_arr = xp.concatenate((alpha_arr, alpha.data))
predicted_out = self.out(self.attn_out(F.concat(
(ctxt, prevh))))
prob = F.softmax(predicted_out)
pred_word = self.select_word(prob, train=False, sample=sample)
# add integer id of predicted word to output list
predicted_sent.append(int(pred_word.data))
pred_count += 1
# __QUESTION Add attention
# When implementing attention, make sure to use alpha_arr to store
# your attention vectors.
# The visualisation function in nmt_translate.py assumes such an array as input.
return predicted_sent, alpha_arr
def __call__(self, r):
# Define initial input
rbn = self.bn(r) #Check shape with print(r.shape)
r0 = F.relu(self.ffconv(rbn))
# Set update rate
one = Variable(xp.array([1], dtype=xp.float32))
update_rate = F.absolute(self.update_rate(one))
# Recurrent loop
for t in range(self.LoopTimes):
if t == 0:
rt = r0
pt = self.fbconv(rt) #Check shape with print(pt.shape)
e = F.relu(r - pt) # r & pt shape is the same.
rt = rt + F.scale(self.ffconv(e), update_rate)
return rt + self.bpconv(rbn)
def extract(self, x):
assert self.reconstruction_loss_attached or self.pca_loss_attached
assert self.pca_loss_attached or not self.pca_loss_attached
original_shape = list(x.shape)
new_shape = copy(original_shape)
new_shape.insert(1, 1)
x_masked = F.reshape(F.scale(x, self.mask), new_shape)
padding_history = []
shape_history = []
h = x_masked
for downsample_degree in range(self.tmp_n_blocks):
for conv_idx in range(self.n_conv_per_block):
conv = self.__getattribute__("conv_{}_{}".format(
downsample_degree, conv_idx))
if self.debug:
print("conv_{}_{}".format(downsample_degree, conv_idx),
conv.W.shape)
h = conv(h)
if self.debug:
print("\t{}".format(h.shape))
if not (downsample_degree == self.tmp_n_blocks - 1
and conv_idx == self.n_conv_per_block - 1):
if self.debug: # 特徴抽出層でReLUせず，
print("relu")
h = F.relu(h)
if downsample_degree != self.tmp_n_blocks - 1:
shape = h.shape[2:]
shape_history.append(shape)
padding = tuple([x % 2 for x in shape])
padding_history.append(padding)
if self.debug:
print("average_pooling_nd")
h = F.average_pooling_nd(h, 2, 2,
padding) # dimensionの0側にpaddingがかかる
if self.debug:
print("\t{}".format(h.shape))
return h
def callback(msg_1):
global i
global j
global timagev
global Nline
global count
global Lpast
global inputgpu
global prefv
global opt_biasv
global swopt
global Lmin
global vwkeep
global Lth
global mask_brrc
global imgbt, imgrt, imggt
j = j + 1
#print j
if j == 1:
cur_img = preprocess_image(msg_1) #current image
#standard deviation and mean for current image
imgbc = (np.reshape(cur_img[0][0], (1, 128, 256)) + 1.0) * 0.5
imggc = (np.reshape(cur_img[0][1], (1, 128, 256)) + 1.0) * 0.5
imgrc = (np.reshape(cur_img[0][2], (1, 128, 256)) + 1.0) * 0.5
mean_cbgr = np.zeros((3, 1))
std_cbgr = np.zeros((3, 1))
mean_ct = np.zeros((3, 1))
std_ct = np.zeros((3, 1))
mean_cbgr[0] = np.sum(imgbc) / countm
mean_cbgr[1] = np.sum(imggc) / countm
mean_cbgr[2] = np.sum(imgrc) / countm
std_cbgr[0] = np.sqrt(
np.sum(np.square(imgbc - mask_brr1 * mean_cbgr[0])) / countm)
std_cbgr[1] = np.sqrt(
np.sum(np.square(imggc - mask_brr1 * mean_cbgr[1])) / countm)
std_cbgr[2] = np.sqrt(
np.sum(np.square(imgrc - mask_brr1 * mean_cbgr[2])) / countm)
#standard deviation and mean for subgoal image
imgrt = (np.reshape(goal_img[0][0], (1, 128, 256)) + 1) * 0.5
imggt = (np.reshape(goal_img[0][1], (1, 128, 256)) + 1) * 0.5
imgbt = (np.reshape(goal_img[0][2], (1, 128, 256)) + 1) * 0.5
mean_tbgr = np.zeros((3, 1))
std_tbgr = np.zeros((3, 1))
mean_tbgr[0] = np.sum(imgbt) / countm
mean_tbgr[1] = np.sum(imggt) / countm
mean_tbgr[2] = np.sum(imgrt) / countm
std_tbgr[0] = np.sqrt(
np.sum(np.square(imgbt - mask_brr1 * mean_tbgr[0])) / countm)
std_tbgr[1] = np.sqrt(
np.sum(np.square(imggt - mask_brr1 * mean_tbgr[1])) / countm)
std_tbgr[2] = np.sqrt(
np.sum(np.square(imgrt - mask_brr1 * mean_tbgr[2])) / countm)
#mean_ct[0] = (mean_cbgr[0] + mean_tbgr[0])*0.5
#mean_ct[1] = (mean_cbgr[1] + mean_tbgr[1])*0.5
#mean_ct[2] = (mean_cbgr[2] + mean_tbgr[2])*0.5
#std_ct[0] = (std_cbgr[0] + std_tbgr[0])*0.5
#std_ct[1] = (std_cbgr[1] + std_tbgr[1])*0.5
#std_ct[2] = (std_cbgr[2] + std_tbgr[2])*0.5
mean_ct[0] = mean_cbgr[0]
mean_ct[1] = mean_cbgr[1]
mean_ct[2] = mean_cbgr[2]
std_ct[0] = std_cbgr[0]
std_ct[1] = std_cbgr[1]
std_ct[2] = std_cbgr[2]
xcg = F.clip(Variable(cuda.to_gpu(cur_img)), -1.0, 1.0)
imgbtt = (imgbt - mean_tbgr[0]) / std_tbgr[0] * std_ct[0] + mean_ct[0]
imggtt = (imggt - mean_tbgr[1]) / std_tbgr[1] * std_ct[1] + mean_ct[1]
imgrtt = (imgrt - mean_tbgr[2]) / std_tbgr[2] * std_ct[2] + mean_ct[2]
goalt_img = np.array(
(np.reshape(np.concatenate((imgbtt, imggtt, imgrtt), axis=0),
(1, 3, 128, 256)) * mask_c - 0.5) * 2.0,
dtype=np.float32)
timage = F.clip(Variable(cuda.to_gpu(goalt_img)), -1.0, 1.0)
#current image
xcgf = F.get_item(xcg, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(0, 128, 1)))
xcgb = F.flip(F.get_item(xcg, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(128, 256, 1))),
axis=3)
xcgx = F.concat((xcgf, xcgb), axis=1)
#subgoal image
xpgf = F.get_item(timage, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(0, 128, 1)))
xpgb_wof = F.get_item(timage, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(128, 256, 1)))
xpgb = F.flip(xpgb_wof, axis=3)
xcpg = F.concat((xcgf, xcgb, xpgf, xpgb), axis=1)
#GONet
with chainer.using_config('train', False):
img_gen = gen(invg(xcgf))
dis_real = dis(xcgf)
dis_gen = dis(img_gen)
outputc = fl(xcgf - img_gen, dis_real - dis_gen, dis_real)
#DVMPC
with chainer.using_config('train', False), chainer.no_backprop_mode():
vwres = dvmpc(xcpg)
vwsp = F.separate(vwres, axis=1)
v1 = F.reshape(vwsp[0], (batchsize, 1, 1, 1))
v2 = F.reshape(vwsp[1], (batchsize, 1, 1, 1))
v3 = F.reshape(vwsp[2], (batchsize, 1, 1, 1))
v4 = F.reshape(vwsp[3], (batchsize, 1, 1, 1))
v5 = F.reshape(vwsp[4], (batchsize, 1, 1, 1))
v6 = F.reshape(vwsp[5], (batchsize, 1, 1, 1))
v7 = F.reshape(vwsp[6], (batchsize, 1, 1, 1))
v8 = F.reshape(vwsp[7], (batchsize, 1, 1, 1))
w1 = F.reshape(vwsp[8], (batchsize, 1, 1, 1))
w2 = F.reshape(vwsp[9], (batchsize, 1, 1, 1))
w3 = F.reshape(vwsp[10], (batchsize, 1, 1, 1))
w4 = F.reshape(vwsp[11], (batchsize, 1, 1, 1))
w5 = F.reshape(vwsp[12], (batchsize, 1, 1, 1))
w6 = F.reshape(vwsp[13], (batchsize, 1, 1, 1))
w7 = F.reshape(vwsp[14], (batchsize, 1, 1, 1))
w8 = F.reshape(vwsp[15], (batchsize, 1, 1, 1))
vresg = F.tanh(F.concat(
(v1, v2, v3, v4, v5, v6, v7, v8), axis=1)) * 0.5
wresg = F.tanh(F.concat(
(w1, w2, w3, w4, w5, w6, w7, w8), axis=1)) * 1.0
#print "linear", v1, v2, v3, v4, v5, v6, v7, v8
#print "angular", w1, w2, w3, w4, w5, w6, w7, w8
#VUNet-360
with chainer.using_config('train', False):
z = enc(xcgx)
x = dec(z, vresg, wresg)
xsp = F.separate(x, axis=1)
apf0f_f = F.reshape(xsp[0], (batchsize, 1, 128, 128))
apf1f_f = F.reshape(xsp[1], (batchsize, 1, 128, 128))
apf2f_f = F.reshape(xsp[2], (batchsize, 1, 128, 128))
apf3f_f = F.reshape(xsp[3], (batchsize, 1, 128, 128))
apf4f_f = F.reshape(xsp[4], (batchsize, 1, 128, 128))
apf5f_f = F.reshape(xsp[5], (batchsize, 1, 128, 128))
apf6f_f = F.reshape(xsp[6], (batchsize, 1, 128, 128))
apf7f_f = F.reshape(xsp[7], (batchsize, 1, 128, 128))
apf8f_f = F.reshape(xsp[8], (batchsize, 1, 128, 128))
apf9f_f = F.reshape(xsp[9], (batchsize, 1, 128, 128))
apf10f_f = F.reshape(xsp[10], (batchsize, 1, 128, 128))
apf11f_f = F.reshape(xsp[11], (batchsize, 1, 128, 128))
apf12f_f = F.reshape(xsp[12], (batchsize, 1, 128, 128))
apf13f_f = F.reshape(xsp[13], (batchsize, 1, 128, 128))
apf14f_f = F.reshape(xsp[14], (batchsize, 1, 128, 128))
apf15f_f = F.reshape(xsp[15], (batchsize, 1, 128, 128))
apf0b_f = F.reshape(xsp[16], (batchsize, 1, 128, 128))
apf1b_f = F.reshape(xsp[17], (batchsize, 1, 128, 128))
apf2b_f = F.reshape(xsp[18], (batchsize, 1, 128, 128))
apf3b_f = F.reshape(xsp[19], (batchsize, 1, 128, 128))
apf4b_f = F.reshape(xsp[20], (batchsize, 1, 128, 128))
apf5b_f = F.reshape(xsp[21], (batchsize, 1, 128, 128))
apf6b_f = F.reshape(xsp[22], (batchsize, 1, 128, 128))
apf7b_f = F.reshape(xsp[23], (batchsize, 1, 128, 128))
apf8b_f = F.reshape(xsp[24], (batchsize, 1, 128, 128))
apf9b_f = F.reshape(xsp[25], (batchsize, 1, 128, 128))
apf10b_f = F.reshape(xsp[26], (batchsize, 1, 128, 128))
apf11b_f = F.reshape(xsp[27], (batchsize, 1, 128, 128))
apf12b_f = F.reshape(xsp[28], (batchsize, 1, 128, 128))
apf13b_f = F.reshape(xsp[29], (batchsize, 1, 128, 128))
apf14b_f = F.reshape(xsp[30], (batchsize, 1, 128, 128))
apf15b_f = F.reshape(xsp[31], (batchsize, 1, 128, 128))
w1f_f = F.reshape(xsp[32], (batchsize, 1, 128, 128))
w2f_f = F.reshape(xsp[33], (batchsize, 1, 128, 128))
w3f_f = F.reshape(xsp[34], (batchsize, 1, 128, 128))
w4f_f = F.reshape(xsp[35], (batchsize, 1, 128, 128))
w5f_f = F.reshape(xsp[36], (batchsize, 1, 128, 128))
w6f_f = F.reshape(xsp[37], (batchsize, 1, 128, 128))
w7f_f = F.reshape(xsp[38], (batchsize, 1, 128, 128))
w8f_f = F.reshape(xsp[39], (batchsize, 1, 128, 128))
w1b_f = F.reshape(xsp[40], (batchsize, 1, 128, 128))
w2b_f = F.reshape(xsp[41], (batchsize, 1, 128, 128))
w3b_f = F.reshape(xsp[42], (batchsize, 1, 128, 128))
w4b_f = F.reshape(xsp[43], (batchsize, 1, 128, 128))
w5b_f = F.reshape(xsp[44], (batchsize, 1, 128, 128))
w6b_f = F.reshape(xsp[45], (batchsize, 1, 128, 128))
w7b_f = F.reshape(xsp[46], (batchsize, 1, 128, 128))
w8b_f = F.reshape(xsp[47], (batchsize, 1, 128, 128))
apf_f1f_f = F.concat((apf0f_f, apf1f_f), axis=1)
apf_f2f_f = F.concat((apf2f_f, apf3f_f), axis=1)
apf_f3f_f = F.concat((apf4f_f, apf5f_f), axis=1)
apf_f4f_f = F.concat((apf6f_f, apf7f_f), axis=1)
apf_f5f_f = F.concat((apf8f_f, apf9f_f), axis=1)
apf_f6f_f = F.concat((apf10f_f, apf11f_f), axis=1)
apf_f7f_f = F.concat((apf12f_f, apf13f_f), axis=1)
apf_f8f_f = F.concat((apf14f_f, apf15f_f), axis=1)
apf_f1b_f = F.concat((apf0b_f, apf1b_f), axis=1)
apf_f2b_f = F.concat((apf2b_f, apf3b_f), axis=1)
apf_f3b_f = F.concat((apf4b_f, apf5b_f), axis=1)
apf_f4b_f = F.concat((apf6b_f, apf7b_f), axis=1)
apf_f5b_f = F.concat((apf8b_f, apf9b_f), axis=1)
apf_f6b_f = F.concat((apf10b_f, apf11b_f), axis=1)
apf_f7b_f = F.concat((apf12b_f, apf13b_f), axis=1)
apf_f8b_f = F.concat((apf14b_f, apf15b_f), axis=1)
genL1f_f = F.spatial_transformer_sampler(xcgf, apf_f1f_f)
genL2f_f = F.spatial_transformer_sampler(xcgf, apf_f2f_f)
genL3f_f = F.spatial_transformer_sampler(xcgf, apf_f3f_f)
genL4f_f = F.spatial_transformer_sampler(xcgf, apf_f4f_f)
genL5f_f = F.spatial_transformer_sampler(xcgf, apf_f5f_f)
genL6f_f = F.spatial_transformer_sampler(xcgf, apf_f6f_f)
genL7f_f = F.spatial_transformer_sampler(xcgf, apf_f7f_f)
genL8f_f = F.spatial_transformer_sampler(xcgf, apf_f8f_f)
genL1b_f = F.spatial_transformer_sampler(xcgb, apf_f1b_f)
genL2b_f = F.spatial_transformer_sampler(xcgb, apf_f2b_f)
genL3b_f = F.spatial_transformer_sampler(xcgb, apf_f3b_f)
genL4b_f = F.spatial_transformer_sampler(xcgb, apf_f4b_f)
genL5b_f = F.spatial_transformer_sampler(xcgb, apf_f5b_f)
genL6b_f = F.spatial_transformer_sampler(xcgb, apf_f6b_f)
genL7b_f = F.spatial_transformer_sampler(xcgb, apf_f7b_f)
genL8b_f = F.spatial_transformer_sampler(xcgb, apf_f8b_f)
mask1_f = F.concat((w1f_f, w1b_f), axis=1)
mask2_f = F.concat((w2f_f, w2b_f), axis=1)
mask3_f = F.concat((w3f_f, w3b_f), axis=1)
mask4_f = F.concat((w4f_f, w4b_f), axis=1)
mask5_f = F.concat((w5f_f, w5b_f), axis=1)
mask6_f = F.concat((w6f_f, w6b_f), axis=1)
mask7_f = F.concat((w7f_f, w7b_f), axis=1)
mask8_f = F.concat((w8f_f, w8b_f), axis=1)
mask_soft1_f = F.softmax(mask1_f, axis=1)
mask_soft2_f = F.softmax(mask2_f, axis=1)
mask_soft3_f = F.softmax(mask3_f, axis=1)
mask_soft4_f = F.softmax(mask4_f, axis=1)
mask_soft5_f = F.softmax(mask5_f, axis=1)
mask_soft6_f = F.softmax(mask6_f, axis=1)
mask_soft7_f = F.softmax(mask7_f, axis=1)
mask_soft8_f = F.softmax(mask8_f, axis=1)
mask_sep1_f = F.separate(mask_soft1_f, axis=1)
mask_sep2_f = F.separate(mask_soft2_f, axis=1)
mask_sep3_f = F.separate(mask_soft3_f, axis=1)
mask_sep4_f = F.separate(mask_soft4_f, axis=1)
mask_sep5_f = F.separate(mask_soft5_f, axis=1)
mask_sep6_f = F.separate(mask_soft6_f, axis=1)
mask_sep7_f = F.separate(mask_soft7_f, axis=1)
mask_sep8_f = F.separate(mask_soft8_f, axis=1)
mask_1f_f = F.reshape(mask_sep1_f[0], (batchsize, 1, 128, 128))
mask_1b_f = F.reshape(mask_sep1_f[1], (batchsize, 1, 128, 128))
mask_2f_f = F.reshape(mask_sep2_f[0], (batchsize, 1, 128, 128))
mask_2b_f = F.reshape(mask_sep2_f[1], (batchsize, 1, 128, 128))
mask_3f_f = F.reshape(mask_sep3_f[0], (batchsize, 1, 128, 128))
mask_3b_f = F.reshape(mask_sep3_f[1], (batchsize, 1, 128, 128))
mask_4f_f = F.reshape(mask_sep4_f[0], (batchsize, 1, 128, 128))
mask_4b_f = F.reshape(mask_sep4_f[1], (batchsize, 1, 128, 128))
mask_5f_f = F.reshape(mask_sep5_f[0], (batchsize, 1, 128, 128))
mask_5b_f = F.reshape(mask_sep5_f[1], (batchsize, 1, 128, 128))
mask_6f_f = F.reshape(mask_sep6_f[0], (batchsize, 1, 128, 128))
mask_6b_f = F.reshape(mask_sep6_f[1], (batchsize, 1, 128, 128))
mask_7f_f = F.reshape(mask_sep7_f[0], (batchsize, 1, 128, 128))
mask_7b_f = F.reshape(mask_sep7_f[1], (batchsize, 1, 128, 128))
mask_8f_f = F.reshape(mask_sep8_f[0], (batchsize, 1, 128, 128))
mask_8b_f = F.reshape(mask_sep8_f[1], (batchsize, 1, 128, 128))
genL1x_f = F.scale(genL1f_f, mask_1f_f, axis=0) + F.scale(
genL1b_f, mask_1b_f, axis=0)
genL2x_f = F.scale(genL2f_f, mask_2f_f, axis=0) + F.scale(
genL2b_f, mask_2b_f, axis=0)
genL3x_f = F.scale(genL3f_f, mask_3f_f, axis=0) + F.scale(
genL3b_f, mask_3b_f, axis=0)
genL4x_f = F.scale(genL4f_f, mask_4f_f, axis=0) + F.scale(
genL4b_f, mask_4b_f, axis=0)
genL5x_f = F.scale(genL5f_f, mask_5f_f, axis=0) + F.scale(
genL5b_f, mask_5b_f, axis=0)
genL6x_f = F.scale(genL6f_f, mask_6f_f, axis=0) + F.scale(
genL6b_f, mask_6b_f, axis=0)
genL7x_f = F.scale(genL7f_f, mask_7f_f, axis=0) + F.scale(
genL7b_f, mask_7b_f, axis=0)
genL8x_f = F.scale(genL8f_f, mask_8f_f, axis=0) + F.scale(
genL8b_f, mask_8b_f, axis=0)
xap_f = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, 5.0 * genL8x_f),
axis=1)
apf0f_b = F.reshape(xsp[48], (batchsize, 1, 128, 128))
apf1f_b = F.reshape(xsp[49], (batchsize, 1, 128, 128))
apf2f_b = F.reshape(xsp[50], (batchsize, 1, 128, 128))
apf3f_b = F.reshape(xsp[51], (batchsize, 1, 128, 128))
apf4f_b = F.reshape(xsp[52], (batchsize, 1, 128, 128))
apf5f_b = F.reshape(xsp[53], (batchsize, 1, 128, 128))
apf6f_b = F.reshape(xsp[54], (batchsize, 1, 128, 128))
apf7f_b = F.reshape(xsp[55], (batchsize, 1, 128, 128))
apf8f_b = F.reshape(xsp[56], (batchsize, 1, 128, 128))
apf9f_b = F.reshape(xsp[57], (batchsize, 1, 128, 128))
apf10f_b = F.reshape(xsp[58], (batchsize, 1, 128, 128))
apf11f_b = F.reshape(xsp[59], (batchsize, 1, 128, 128))
apf12f_b = F.reshape(xsp[60], (batchsize, 1, 128, 128))
apf13f_b = F.reshape(xsp[61], (batchsize, 1, 128, 128))
apf14f_b = F.reshape(xsp[62], (batchsize, 1, 128, 128))
apf15f_b = F.reshape(xsp[63], (batchsize, 1, 128, 128))
apf0b_b = F.reshape(xsp[64], (batchsize, 1, 128, 128))
apf1b_b = F.reshape(xsp[65], (batchsize, 1, 128, 128))
apf2b_b = F.reshape(xsp[66], (batchsize, 1, 128, 128))
apf3b_b = F.reshape(xsp[67], (batchsize, 1, 128, 128))
apf4b_b = F.reshape(xsp[68], (batchsize, 1, 128, 128))
apf5b_b = F.reshape(xsp[69], (batchsize, 1, 128, 128))
apf6b_b = F.reshape(xsp[70], (batchsize, 1, 128, 128))
apf7b_b = F.reshape(xsp[71], (batchsize, 1, 128, 128))
apf8b_b = F.reshape(xsp[72], (batchsize, 1, 128, 128))
apf9b_b = F.reshape(xsp[73], (batchsize, 1, 128, 128))
apf10b_b = F.reshape(xsp[74], (batchsize, 1, 128, 128))
apf11b_b = F.reshape(xsp[75], (batchsize, 1, 128, 128))
apf12b_b = F.reshape(xsp[76], (batchsize, 1, 128, 128))
apf13b_b = F.reshape(xsp[77], (batchsize, 1, 128, 128))
apf14b_b = F.reshape(xsp[78], (batchsize, 1, 128, 128))
apf15b_b = F.reshape(xsp[79], (batchsize, 1, 128, 128))
w1f_b = F.reshape(xsp[80], (batchsize, 1, 128, 128))
w2f_b = F.reshape(xsp[81], (batchsize, 1, 128, 128))
w3f_b = F.reshape(xsp[82], (batchsize, 1, 128, 128))
w4f_b = F.reshape(xsp[83], (batchsize, 1, 128, 128))
w5f_b = F.reshape(xsp[84], (batchsize, 1, 128, 128))
w6f_b = F.reshape(xsp[85], (batchsize, 1, 128, 128))
w7f_b = F.reshape(xsp[86], (batchsize, 1, 128, 128))
w8f_b = F.reshape(xsp[87], (batchsize, 1, 128, 128))
w1b_b = F.reshape(xsp[88], (batchsize, 1, 128, 128))
w2b_b = F.reshape(xsp[89], (batchsize, 1, 128, 128))
w3b_b = F.reshape(xsp[90], (batchsize, 1, 128, 128))
w4b_b = F.reshape(xsp[91], (batchsize, 1, 128, 128))
w5b_b = F.reshape(xsp[92], (batchsize, 1, 128, 128))
w6b_b = F.reshape(xsp[93], (batchsize, 1, 128, 128))
w7b_b = F.reshape(xsp[94], (batchsize, 1, 128, 128))
w8b_b = F.reshape(xsp[95], (batchsize, 1, 128, 128))
apf_b1f_b = F.concat((apf0f_b, apf1f_b), axis=1)
apf_b2f_b = F.concat((apf2f_b, apf3f_b), axis=1)
apf_b3f_b = F.concat((apf4f_b, apf5f_b), axis=1)
apf_b4f_b = F.concat((apf6f_b, apf7f_b), axis=1)
apf_b5f_b = F.concat((apf8f_b, apf9f_b), axis=1)
apf_b6f_b = F.concat((apf10f_b, apf11f_b), axis=1)
apf_b7f_b = F.concat((apf12f_b, apf13f_b), axis=1)
apf_b8f_b = F.concat((apf14f_b, apf15f_b), axis=1)
apf_b1b_b = F.concat((apf0b_b, apf1b_b), axis=1)
apf_b2b_b = F.concat((apf2b_b, apf3b_b), axis=1)
apf_b3b_b = F.concat((apf4b_b, apf5b_b), axis=1)
apf_b4b_b = F.concat((apf6b_b, apf7b_b), axis=1)
apf_b5b_b = F.concat((apf8b_b, apf9b_b), axis=1)
apf_b6b_b = F.concat((apf10b_b, apf11b_b), axis=1)
apf_b7b_b = F.concat((apf12b_b, apf13b_b), axis=1)
apf_b8b_b = F.concat((apf14b_b, apf15b_b), axis=1)
genL1f_b = F.spatial_transformer_sampler(xcgf, apf_b1f_b)
genL2f_b = F.spatial_transformer_sampler(xcgf, apf_b2f_b)
genL3f_b = F.spatial_transformer_sampler(xcgf, apf_b3f_b)
genL4f_b = F.spatial_transformer_sampler(xcgf, apf_b4f_b)
genL5f_b = F.spatial_transformer_sampler(xcgf, apf_b5f_b)
genL6f_b = F.spatial_transformer_sampler(xcgf, apf_b6f_b)
genL7f_b = F.spatial_transformer_sampler(xcgf, apf_b7f_b)
genL8f_b = F.spatial_transformer_sampler(xcgf, apf_b8f_b)
genL1b_b = F.spatial_transformer_sampler(xcgb, apf_b1b_b)
genL2b_b = F.spatial_transformer_sampler(xcgb, apf_b2b_b)
genL3b_b = F.spatial_transformer_sampler(xcgb, apf_b3b_b)
genL4b_b = F.spatial_transformer_sampler(xcgb, apf_b4b_b)
genL5b_b = F.spatial_transformer_sampler(xcgb, apf_b5b_b)
genL6b_b = F.spatial_transformer_sampler(xcgb, apf_b6b_b)
genL7b_b = F.spatial_transformer_sampler(xcgb, apf_b7b_b)
genL8b_b = F.spatial_transformer_sampler(xcgb, apf_b8b_b)
mask1_b = F.concat((w1f_b, w1b_b), axis=1)
mask2_b = F.concat((w2f_b, w2b_b), axis=1)
mask3_b = F.concat((w3f_b, w3b_b), axis=1)
mask4_b = F.concat((w4f_b, w4b_b), axis=1)
mask5_b = F.concat((w5f_b, w5b_b), axis=1)
mask6_b = F.concat((w6f_b, w6b_b), axis=1)
mask7_b = F.concat((w7f_b, w7b_b), axis=1)
mask8_b = F.concat((w8f_b, w8b_b), axis=1)
mask_soft1_b = F.softmax(mask1_b, axis=1)
mask_soft2_b = F.softmax(mask2_b, axis=1)
mask_soft3_b = F.softmax(mask3_b, axis=1)
mask_soft4_b = F.softmax(mask4_b, axis=1)
mask_soft5_b = F.softmax(mask5_b, axis=1)
mask_soft6_b = F.softmax(mask6_b, axis=1)
mask_soft7_b = F.softmax(mask7_b, axis=1)
mask_soft8_b = F.softmax(mask8_b, axis=1)
mask_sep1_b = F.separate(mask_soft1_b, axis=1)
mask_sep2_b = F.separate(mask_soft2_b, axis=1)
mask_sep3_b = F.separate(mask_soft3_b, axis=1)
mask_sep4_b = F.separate(mask_soft4_b, axis=1)
mask_sep5_b = F.separate(mask_soft5_b, axis=1)
mask_sep6_b = F.separate(mask_soft6_b, axis=1)
mask_sep7_b = F.separate(mask_soft7_b, axis=1)
mask_sep8_b = F.separate(mask_soft8_b, axis=1)
mask_1f_b = F.reshape(mask_sep1_b[0], (batchsize, 1, 128, 128))
mask_1b_b = F.reshape(mask_sep1_b[1], (batchsize, 1, 128, 128))
mask_2f_b = F.reshape(mask_sep2_b[0], (batchsize, 1, 128, 128))
mask_2b_b = F.reshape(mask_sep2_b[1], (batchsize, 1, 128, 128))
mask_3f_b = F.reshape(mask_sep3_b[0], (batchsize, 1, 128, 128))
mask_3b_b = F.reshape(mask_sep3_b[1], (batchsize, 1, 128, 128))
mask_4f_b = F.reshape(mask_sep4_b[0], (batchsize, 1, 128, 128))
mask_4b_b = F.reshape(mask_sep4_b[1], (batchsize, 1, 128, 128))
mask_5f_b = F.reshape(mask_sep5_b[0], (batchsize, 1, 128, 128))
mask_5b_b = F.reshape(mask_sep5_b[1], (batchsize, 1, 128, 128))
mask_6f_b = F.reshape(mask_sep6_b[0], (batchsize, 1, 128, 128))
mask_6b_b = F.reshape(mask_sep6_b[1], (batchsize, 1, 128, 128))
mask_7f_b = F.reshape(mask_sep7_b[0], (batchsize, 1, 128, 128))
mask_7b_b = F.reshape(mask_sep7_b[1], (batchsize, 1, 128, 128))
mask_8f_b = F.reshape(mask_sep8_b[0], (batchsize, 1, 128, 128))
mask_8b_b = F.reshape(mask_sep8_b[1], (batchsize, 1, 128, 128))
genL1x_b = F.scale(genL1f_b, mask_1f_b, axis=0) + F.scale(
genL1b_b, mask_1b_b, axis=0)
genL2x_b = F.scale(genL2f_b, mask_2f_b, axis=0) + F.scale(
genL2b_b, mask_2b_b, axis=0)
genL3x_b = F.scale(genL3f_b, mask_3f_b, axis=0) + F.scale(
genL3b_b, mask_3b_b, axis=0)
genL4x_b = F.scale(genL4f_b, mask_4f_b, axis=0) + F.scale(
genL4b_b, mask_4b_b, axis=0)
genL5x_b = F.scale(genL5f_b, mask_5f_b, axis=0) + F.scale(
genL5b_b, mask_5b_b, axis=0)
genL6x_b = F.scale(genL6f_b, mask_6f_b, axis=0) + F.scale(
genL6b_b, mask_6b_b, axis=0)
genL7x_b = F.scale(genL7f_b, mask_7f_b, axis=0) + F.scale(
genL7b_b, mask_7b_b, axis=0)
genL8x_b = F.scale(genL8f_b, mask_8f_b, axis=0) + F.scale(
genL8b_b, mask_8b_b, axis=0)
xap_b = F.concat((genL1x_b, genL2x_b, genL3x_b, genL4x_b, genL5x_b,
genL6x_b, genL7x_b, 5.0 * genL8x_b),
axis=1)
#VUNet-360 end
#GONet
xgonet = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, genL8x_f),
axis=0)
with chainer.using_config('train', False):
img_gen = gen(invg(xgonet))
dis_real = dis(xgonet)
dis_gen = dis(img_gen)
output = fl(xgonet - img_gen, dis_real - dis_gen, dis_real)
xrefy = F.concat((Variable(cur_img), Variable(goal_img)), axis=3)
xgonetx = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, genL8x_f),
axis=3)
xgonety = F.concat((genL1x_b, genL2x_b, genL3x_b, genL4x_b, genL5x_b,
genL6x_b, genL7x_b, genL8x_b),
axis=3)
msg_pub = Twist()
#velocity cap and publish
vt = cuda.to_cpu(vresg.data[0][0])
wt = cuda.to_cpu(wresg.data[0][0])
if np.absolute(vt) < 0.2:
msg_pub.linear.x = vt
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = wt
else:
if np.absolute(wt) < 0.001:
msg_pub.linear.x = 0.2 * np.sign(vt)
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = 0.0
else:
rd = vt / wt
msg_pub.linear.x = 0.2 * np.sign(vt)
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = 0.2 * np.sign(vt) / rd
#front predicted image
out_imgc = cuda.to_cpu(xgonetx.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 128 * 8))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_genf.publish(imgm)
#back predicted image
out_imgc = cuda.to_cpu(xgonety.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 128 * 8))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_genb.publish(imgm)
#current and goal image
out_imgc = cuda.to_cpu(xrefy.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 256 * 2))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_ref.publish(imgm)
#velocities
msg_out.publish(msg_pub)
j = 0
def precedence_weighting(write_weighting, precedence_weighting_prev):
""" (batch_size,n_locations) -> (batch_size,n_locations) -> (batch_size,n_locations) """
w, p = write_weighting, precedence_weighting_prev
r = F.scale(p, (1 - F.sum(w, axis=1)), axis=0) + w
return r
def attn_layer(self, ht, Hs):
score = F.matmul(ht, Hs, transb=True)
a_t = F.softmax(score)
c_t = F.reshape(M.sum(F.scale(Hs, F.transpose(a_t), axis=0), axis=0),
(1, ht.shape[1]))
return c_t, a_t
def __call__(self, x, t, dataset, train=True):
# Create variables
x = Variable(x)
x.to_gpu(self.gpu_id)
t = Variable(t)
t.to_gpu(self.gpu_id)
with chainer.using_config('train', False):
with chainer.using_config('enable_backprop', False):
xo = self.segmentation.predictor(x)
y = self.segmentation.classifiers[0](xo)
y = F.separate(F.softmax(y), axis=1)
# foreground, head, torso-hand, lower-body, shoes
segprob = F.stack(
(1.0 - y[0], y[1] + y[2] + y[4] + y[13],
y[5] + y[6] + y[7] + y[11] + y[10] + y[3] + y[14] + y[15],
y[9] + y[16] + y[17] + y[12], y[18] + y[19] + y[8]),
axis=1)
# Forward
with chainer.using_config('train', train):
with chainer.using_config('enable_backprop', train):
x = F.resize_images(x, self.args.scales_reid)
# InceptionV3 backbone
x = self.predictor(x)
x_a = F.average_pooling_2d(x, x.shape[-2:])
# Resize to segmentation map resolution
x = F.resize_images(x, segprob.shape[-2:])
# aggregate features at semantic parts
xl = F.scale(F.batch_matmul(
F.reshape(segprob,
(segprob.shape[0], segprob.shape[1], -1)),
F.reshape(x, (x.shape[0], x.shape[1], -1)),
transb=True),
1.0 / F.sum(segprob, axis=(2, 3)),
axis=0)
xfg, xl = F.split_axis(xl, [1], axis=1)
xl = F.max(xl, axis=1, keepdims=True)
x = F.concat((xfg, xl), axis=2)
# Classifiers
x_s = F.reshape(x, (-1, 2 * 2048, 1, 1))
x = F.concat((x_s, x_a), axis=1)
if train:
self.y_s = self.classifiers[0](x)
# Loss
self.loss = F.softmax_cross_entropy(
F.squeeze(self.y_s, axis=(2, 3)), t)
# Clear grads for uninitialized params
self.cleargrads()
# Backwards
self.loss.backward()
# Reporter
self.reporter.update(
{dataset: {
'loss': self.loss.data.tolist()
}})
else:
x = F.squeeze(x)
x.to_cpu()
self.reporter.update({dataset: {'features': x.data}})
def check_backward(self, x1_data, x2_data, axis, y_grad):
x = (x1_data, x2_data)
gradient_check.check_backward(
lambda x, y: functions.scale(x, y, axis),
x, y_grad,
**self.check_backward_options)
def check_forward(self, x1_data, x2_data, axis, y_expected):
x1 = chainer.Variable(x1_data)
x2 = chainer.Variable(x2_data)
y = functions.scale(x1, x2, axis)
testing.assert_allclose(y_expected, y.data)
def __call__(self, hs, ys):
"""Core function of Decoder layer.
Args:
hs (list of chainer.Variable | N-dimension array): Input variable from encoder.
ys (list of chainer.Variable | N-dimension array): Input variable of decoder.
Returns:
chainer.Variable: A variable holding a scalar array of the training loss.
chainer.Variable: A variable holding a scalar array of the accuracy.
"""
self.loss = None
# prepare input and output word sequences with sos/eos IDs
eos = self.xp.array([self.eos], 'i')
sos = self.xp.array([self.sos], 'i')
ys_in = [F.concat([sos, y], axis=0) for y in ys]
ys_out = [F.concat([y, eos], axis=0) for y in ys]
# padding for ys with -1
# pys: utt x olen
pad_ys_in = F.pad_sequence(ys_in, padding=self.eos)
pad_ys_out = F.pad_sequence(ys_out, padding=-1)
# get dim, length info
batch = pad_ys_out.shape[0]
olength = pad_ys_out.shape[1]
logging.info(self.__class__.__name__ + ' input lengths: ' +
str(self.xp.array([h.shape[0] for h in hs])))
logging.info(self.__class__.__name__ + ' output lengths: ' +
str(self.xp.array([y.shape[0] for y in ys_out])))
# initialization
c_list = [None] # list of cell state of each layer
z_list = [None] # list of hidden state of each layer
for _ in six.moves.range(1, self.dlayers):
c_list.append(None)
z_list.append(None)
att_w = None
z_all = []
self.att.reset() # reset pre-computation of h
# pre-computation of embedding
eys = self.embed(pad_ys_in) # utt x olen x zdim
eys = F.separate(eys, axis=1)
# loop for an output sequence
for i in six.moves.range(olength):
att_c, att_w = self.att(hs, z_list[0], att_w)
if i > 0 and random.random() < self.sampling_probability:
logging.info(' scheduled sampling ')
z_out = self.output(z_all[-1])
z_out = F.argmax(F.log_softmax(z_out), axis=1)
z_out = self.embed(z_out)
ey = F.hstack((z_out, att_c)) # utt x (zdim + hdim)
else:
ey = F.hstack((eys[i], att_c)) # utt x (zdim + hdim)
z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list,
c_list)
z_all.append(z_list[-1])
z_all = F.stack(z_all, axis=1).reshape(batch * olength, self.dunits)
# compute loss
y_all = self.output(z_all)
self.loss = F.softmax_cross_entropy(y_all, F.flatten(pad_ys_out))
# -1: eos, which is removed in the loss computation
self.loss *= (np.mean([len(x) for x in ys_in]) - 1)
acc = F.accuracy(y_all, F.flatten(pad_ys_out), ignore_label=-1)
logging.info('att loss:' + str(self.loss.data))
# show predicted character sequence for debug
if self.verbose > 0 and self.char_list is not None:
y_hat = y_all.reshape(batch, olength, -1)
y_true = pad_ys_out
for (i, y_hat_), y_true_ in zip(enumerate(y_hat.data),
y_true.data):
if i == MAX_DECODER_OUTPUT:
break
idx_hat = self.xp.argmax(y_hat_[y_true_ != -1], axis=1)
idx_true = y_true_[y_true_ != -1]
seq_hat = [self.char_list[int(idx)] for idx in idx_hat]
seq_true = [self.char_list[int(idx)] for idx in idx_true]
seq_hat = "".join(seq_hat).replace('<space>', ' ')
seq_true = "".join(seq_true).replace('<space>', ' ')
logging.info("groundtruth[%d]: " % i + seq_true)
logging.info("prediction [%d]: " % i + seq_hat)
if self.labeldist is not None:
if self.vlabeldist is None:
self.vlabeldist = chainer.Variable(
self.xp.asarray(self.labeldist))
loss_reg = -F.sum(
F.scale(F.log_softmax(y_all), self.vlabeldist,
axis=1)) / len(ys_in)
self.loss = (
1. - self.lsm_weight) * self.loss + self.lsm_weight * loss_reg
return self.loss, acc
