def listmle(x, t, nr_docs):
"""
The ListMLE loss as in Xia et al (2008), Listwise Approach to Learning to
Rank - Theory and Algorithm.
:param x: The activation of the previous layer
:type x: chainer.Variable
:param t: The target labels
:type t: chainer.Variable
:param nr_docs: The number of documents per query
:type nr_docs: chainer.Variable
:return: The loss
:rtype: chainer.Variable
"""
t, nr_docs = as_variable(t), as_variable(nr_docs)
# Get the ground truth by sorting activations by the relevance labels
indices = argsort(t, axis=1)
x_hat = select_items_per_row(x, cf.flip(indices, axis=1))
# Compute MLE loss
per_sample_loss = -cf.sum(x_hat - logcumsumexp(x_hat), axis=1)
return cf.mean(per_sample_loss)
def check_forward(self, x_data, axis):
x = chainer.Variable(x_data)
y = functions.flip(x, axis)
flip_func = getattr(numpy, 'flip', functions.array.flip._flip)
expected_y = flip_func(x_data, axis)
testing.assert_allclose(y.data, expected_y)
def seq_rnn_embed(vxs, exs, rnn_layer, initial_state=None, reverse=False):
"""Embed given sequences using rnn."""
# vxs.shape == (..., S)
# exs.shape == (..., S, E)
# initial_state == (..., E)
assert vxs.shape == exs.shape[:
-1], "Sequence embedding dimensions do not match."
lengths = np.sum(vxs != 0, -1).flatten() # (X,)
seqs = F.reshape(exs, (-1, ) + exs.shape[-2:]) # (X, S, E)
if reverse:
toembed = [
F.flip(s[..., :l, :], -2)
for s, l in zip(F.separate(seqs, 0), lengths) if l != 0
] # Y x [(S1, E), (S2, E), ...]
else:
toembed = [
s[..., :l, :] for s, l in zip(F.separate(seqs, 0), lengths)
if l != 0
] # Y x [(S1, E), (S2, E), ...]
if initial_state is not None:
initial_state = F.reshape(initial_state, (-1, EMBED)) # (X, E)
initial_state = initial_state[None,
np.flatnonzero(lengths)] # (1, Y, E)
hs, ys = rnn_layer(initial_state,
toembed) # (1, Y, E), Y x [(S1, 2*E), (S2, 2*E), ...]
hs = hs[0] # (Y, E)
if hs.shape[0] == lengths.size:
hs = F.reshape(hs, vxs.shape[:-1] + (EMBED, )) # (..., E)
return hs
# Add zero values back to match original shape
embeds = np.zeros((lengths.size, EMBED), dtype=np.float32) # (X, E)
idxs = np.nonzero(lengths) # (Y,)
embeds = F.scatter_add(embeds, idxs, hs) # (X, E)
embeds = F.reshape(embeds, vxs.shape[:-1] + (EMBED, )) # (..., E)
return embeds # (..., E)
def VGGprepare_am_input(var):
xp = get_array_module(var)
# var = F.resize_images(var, size)
var = F.transpose(var, (0, 2, 3, 1)) # [[W, H, C]]
var = F.flip(var, 3)
var -= xp.array([[103.939, 116.779, 123.68]], dtype=xp.float32)
var = F.transpose(var, (0, 3, 1, 2))
return var
def __call__(self, xs, labels):
xs = F.stack(xs) # B, T, D
labels = F.stack(labels)
forward_out = self.forward_ld_rnn(xs, labels)
reverse_xs = F.flip(xs, axis=1)
reverse_label = F.flip(labels, axis=1)
backward_out = self.backward_ld_rnn(reverse_xs, reverse_label)
forward_out = F.stack(forward_out) # B, T, D
backward_out = F.stack(backward_out) # B, reverse_T, D
backward_out = F.flip(backward_out, axis=1)
bi_out = F.concat((forward_out, backward_out), axis=2)
bi_out = F.reshape(bi_out,
(xs.shape[0] * xs.shape[1], 2 * self.mid_size))
final_out = self.fusion_fc(bi_out)
final_out = F.reshape(final_out,
(xs.shape[0], xs.shape[1], self.out_size))
return list(F.separate(final_out, axis=0))
def callback(msg_1):
global i
global j
global timagev
global Nline
global count
global Lpast
global inputgpu
global prefv
global opt_biasv
global swopt
global Lmin
global vwkeep
global Lth
global mask_brrc
global imgbt, imgrt, imggt
j = j + 1
#print j
if j == 1:
cur_img = preprocess_image(msg_1) #current image
#standard deviation and mean for current image
imgbc = (np.reshape(cur_img[0][0], (1, 128, 256)) + 1.0) * 0.5
imggc = (np.reshape(cur_img[0][1], (1, 128, 256)) + 1.0) * 0.5
imgrc = (np.reshape(cur_img[0][2], (1, 128, 256)) + 1.0) * 0.5
mean_cbgr = np.zeros((3, 1))
std_cbgr = np.zeros((3, 1))
mean_ct = np.zeros((3, 1))
std_ct = np.zeros((3, 1))
mean_cbgr[0] = np.sum(imgbc) / countm
mean_cbgr[1] = np.sum(imggc) / countm
mean_cbgr[2] = np.sum(imgrc) / countm
std_cbgr[0] = np.sqrt(
np.sum(np.square(imgbc - mask_brr1 * mean_cbgr[0])) / countm)
std_cbgr[1] = np.sqrt(
np.sum(np.square(imggc - mask_brr1 * mean_cbgr[1])) / countm)
std_cbgr[2] = np.sqrt(
np.sum(np.square(imgrc - mask_brr1 * mean_cbgr[2])) / countm)
#standard deviation and mean for subgoal image
imgrt = (np.reshape(goal_img[0][0], (1, 128, 256)) + 1) * 0.5
imggt = (np.reshape(goal_img[0][1], (1, 128, 256)) + 1) * 0.5
imgbt = (np.reshape(goal_img[0][2], (1, 128, 256)) + 1) * 0.5
mean_tbgr = np.zeros((3, 1))
std_tbgr = np.zeros((3, 1))
mean_tbgr[0] = np.sum(imgbt) / countm
mean_tbgr[1] = np.sum(imggt) / countm
mean_tbgr[2] = np.sum(imgrt) / countm
std_tbgr[0] = np.sqrt(
np.sum(np.square(imgbt - mask_brr1 * mean_tbgr[0])) / countm)
std_tbgr[1] = np.sqrt(
np.sum(np.square(imggt - mask_brr1 * mean_tbgr[1])) / countm)
std_tbgr[2] = np.sqrt(
np.sum(np.square(imgrt - mask_brr1 * mean_tbgr[2])) / countm)
#mean_ct[0] = (mean_cbgr[0] + mean_tbgr[0])*0.5
#mean_ct[1] = (mean_cbgr[1] + mean_tbgr[1])*0.5
#mean_ct[2] = (mean_cbgr[2] + mean_tbgr[2])*0.5
#std_ct[0] = (std_cbgr[0] + std_tbgr[0])*0.5
#std_ct[1] = (std_cbgr[1] + std_tbgr[1])*0.5
#std_ct[2] = (std_cbgr[2] + std_tbgr[2])*0.5
mean_ct[0] = mean_cbgr[0]
mean_ct[1] = mean_cbgr[1]
mean_ct[2] = mean_cbgr[2]
std_ct[0] = std_cbgr[0]
std_ct[1] = std_cbgr[1]
std_ct[2] = std_cbgr[2]
xcg = F.clip(Variable(cuda.to_gpu(cur_img)), -1.0, 1.0)
imgbtt = (imgbt - mean_tbgr[0]) / std_tbgr[0] * std_ct[0] + mean_ct[0]
imggtt = (imggt - mean_tbgr[1]) / std_tbgr[1] * std_ct[1] + mean_ct[1]
imgrtt = (imgrt - mean_tbgr[2]) / std_tbgr[2] * std_ct[2] + mean_ct[2]
goalt_img = np.array(
(np.reshape(np.concatenate((imgbtt, imggtt, imgrtt), axis=0),
(1, 3, 128, 256)) * mask_c - 0.5) * 2.0,
dtype=np.float32)
timage = F.clip(Variable(cuda.to_gpu(goalt_img)), -1.0, 1.0)
#current image
xcgf = F.get_item(xcg, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(0, 128, 1)))
xcgb = F.flip(F.get_item(xcg, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(128, 256, 1))),
axis=3)
xcgx = F.concat((xcgf, xcgb), axis=1)
#subgoal image
xpgf = F.get_item(timage, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(0, 128, 1)))
xpgb_wof = F.get_item(timage, (slice(0, batchsize, 1), slice(
0, 3, 1), slice(0, 128, 1), slice(128, 256, 1)))
xpgb = F.flip(xpgb_wof, axis=3)
xcpg = F.concat((xcgf, xcgb, xpgf, xpgb), axis=1)
#GONet
with chainer.using_config('train', False):
img_gen = gen(invg(xcgf))
dis_real = dis(xcgf)
dis_gen = dis(img_gen)
outputc = fl(xcgf - img_gen, dis_real - dis_gen, dis_real)
#DVMPC
with chainer.using_config('train', False), chainer.no_backprop_mode():
vwres = dvmpc(xcpg)
vwsp = F.separate(vwres, axis=1)
v1 = F.reshape(vwsp[0], (batchsize, 1, 1, 1))
v2 = F.reshape(vwsp[1], (batchsize, 1, 1, 1))
v3 = F.reshape(vwsp[2], (batchsize, 1, 1, 1))
v4 = F.reshape(vwsp[3], (batchsize, 1, 1, 1))
v5 = F.reshape(vwsp[4], (batchsize, 1, 1, 1))
v6 = F.reshape(vwsp[5], (batchsize, 1, 1, 1))
v7 = F.reshape(vwsp[6], (batchsize, 1, 1, 1))
v8 = F.reshape(vwsp[7], (batchsize, 1, 1, 1))
w1 = F.reshape(vwsp[8], (batchsize, 1, 1, 1))
w2 = F.reshape(vwsp[9], (batchsize, 1, 1, 1))
w3 = F.reshape(vwsp[10], (batchsize, 1, 1, 1))
w4 = F.reshape(vwsp[11], (batchsize, 1, 1, 1))
w5 = F.reshape(vwsp[12], (batchsize, 1, 1, 1))
w6 = F.reshape(vwsp[13], (batchsize, 1, 1, 1))
w7 = F.reshape(vwsp[14], (batchsize, 1, 1, 1))
w8 = F.reshape(vwsp[15], (batchsize, 1, 1, 1))
vresg = F.tanh(F.concat(
(v1, v2, v3, v4, v5, v6, v7, v8), axis=1)) * 0.5
wresg = F.tanh(F.concat(
(w1, w2, w3, w4, w5, w6, w7, w8), axis=1)) * 1.0
#print "linear", v1, v2, v3, v4, v5, v6, v7, v8
#print "angular", w1, w2, w3, w4, w5, w6, w7, w8
#VUNet-360
with chainer.using_config('train', False):
z = enc(xcgx)
x = dec(z, vresg, wresg)
xsp = F.separate(x, axis=1)
apf0f_f = F.reshape(xsp[0], (batchsize, 1, 128, 128))
apf1f_f = F.reshape(xsp[1], (batchsize, 1, 128, 128))
apf2f_f = F.reshape(xsp[2], (batchsize, 1, 128, 128))
apf3f_f = F.reshape(xsp[3], (batchsize, 1, 128, 128))
apf4f_f = F.reshape(xsp[4], (batchsize, 1, 128, 128))
apf5f_f = F.reshape(xsp[5], (batchsize, 1, 128, 128))
apf6f_f = F.reshape(xsp[6], (batchsize, 1, 128, 128))
apf7f_f = F.reshape(xsp[7], (batchsize, 1, 128, 128))
apf8f_f = F.reshape(xsp[8], (batchsize, 1, 128, 128))
apf9f_f = F.reshape(xsp[9], (batchsize, 1, 128, 128))
apf10f_f = F.reshape(xsp[10], (batchsize, 1, 128, 128))
apf11f_f = F.reshape(xsp[11], (batchsize, 1, 128, 128))
apf12f_f = F.reshape(xsp[12], (batchsize, 1, 128, 128))
apf13f_f = F.reshape(xsp[13], (batchsize, 1, 128, 128))
apf14f_f = F.reshape(xsp[14], (batchsize, 1, 128, 128))
apf15f_f = F.reshape(xsp[15], (batchsize, 1, 128, 128))
apf0b_f = F.reshape(xsp[16], (batchsize, 1, 128, 128))
apf1b_f = F.reshape(xsp[17], (batchsize, 1, 128, 128))
apf2b_f = F.reshape(xsp[18], (batchsize, 1, 128, 128))
apf3b_f = F.reshape(xsp[19], (batchsize, 1, 128, 128))
apf4b_f = F.reshape(xsp[20], (batchsize, 1, 128, 128))
apf5b_f = F.reshape(xsp[21], (batchsize, 1, 128, 128))
apf6b_f = F.reshape(xsp[22], (batchsize, 1, 128, 128))
apf7b_f = F.reshape(xsp[23], (batchsize, 1, 128, 128))
apf8b_f = F.reshape(xsp[24], (batchsize, 1, 128, 128))
apf9b_f = F.reshape(xsp[25], (batchsize, 1, 128, 128))
apf10b_f = F.reshape(xsp[26], (batchsize, 1, 128, 128))
apf11b_f = F.reshape(xsp[27], (batchsize, 1, 128, 128))
apf12b_f = F.reshape(xsp[28], (batchsize, 1, 128, 128))
apf13b_f = F.reshape(xsp[29], (batchsize, 1, 128, 128))
apf14b_f = F.reshape(xsp[30], (batchsize, 1, 128, 128))
apf15b_f = F.reshape(xsp[31], (batchsize, 1, 128, 128))
w1f_f = F.reshape(xsp[32], (batchsize, 1, 128, 128))
w2f_f = F.reshape(xsp[33], (batchsize, 1, 128, 128))
w3f_f = F.reshape(xsp[34], (batchsize, 1, 128, 128))
w4f_f = F.reshape(xsp[35], (batchsize, 1, 128, 128))
w5f_f = F.reshape(xsp[36], (batchsize, 1, 128, 128))
w6f_f = F.reshape(xsp[37], (batchsize, 1, 128, 128))
w7f_f = F.reshape(xsp[38], (batchsize, 1, 128, 128))
w8f_f = F.reshape(xsp[39], (batchsize, 1, 128, 128))
w1b_f = F.reshape(xsp[40], (batchsize, 1, 128, 128))
w2b_f = F.reshape(xsp[41], (batchsize, 1, 128, 128))
w3b_f = F.reshape(xsp[42], (batchsize, 1, 128, 128))
w4b_f = F.reshape(xsp[43], (batchsize, 1, 128, 128))
w5b_f = F.reshape(xsp[44], (batchsize, 1, 128, 128))
w6b_f = F.reshape(xsp[45], (batchsize, 1, 128, 128))
w7b_f = F.reshape(xsp[46], (batchsize, 1, 128, 128))
w8b_f = F.reshape(xsp[47], (batchsize, 1, 128, 128))
apf_f1f_f = F.concat((apf0f_f, apf1f_f), axis=1)
apf_f2f_f = F.concat((apf2f_f, apf3f_f), axis=1)
apf_f3f_f = F.concat((apf4f_f, apf5f_f), axis=1)
apf_f4f_f = F.concat((apf6f_f, apf7f_f), axis=1)
apf_f5f_f = F.concat((apf8f_f, apf9f_f), axis=1)
apf_f6f_f = F.concat((apf10f_f, apf11f_f), axis=1)
apf_f7f_f = F.concat((apf12f_f, apf13f_f), axis=1)
apf_f8f_f = F.concat((apf14f_f, apf15f_f), axis=1)
apf_f1b_f = F.concat((apf0b_f, apf1b_f), axis=1)
apf_f2b_f = F.concat((apf2b_f, apf3b_f), axis=1)
apf_f3b_f = F.concat((apf4b_f, apf5b_f), axis=1)
apf_f4b_f = F.concat((apf6b_f, apf7b_f), axis=1)
apf_f5b_f = F.concat((apf8b_f, apf9b_f), axis=1)
apf_f6b_f = F.concat((apf10b_f, apf11b_f), axis=1)
apf_f7b_f = F.concat((apf12b_f, apf13b_f), axis=1)
apf_f8b_f = F.concat((apf14b_f, apf15b_f), axis=1)
genL1f_f = F.spatial_transformer_sampler(xcgf, apf_f1f_f)
genL2f_f = F.spatial_transformer_sampler(xcgf, apf_f2f_f)
genL3f_f = F.spatial_transformer_sampler(xcgf, apf_f3f_f)
genL4f_f = F.spatial_transformer_sampler(xcgf, apf_f4f_f)
genL5f_f = F.spatial_transformer_sampler(xcgf, apf_f5f_f)
genL6f_f = F.spatial_transformer_sampler(xcgf, apf_f6f_f)
genL7f_f = F.spatial_transformer_sampler(xcgf, apf_f7f_f)
genL8f_f = F.spatial_transformer_sampler(xcgf, apf_f8f_f)
genL1b_f = F.spatial_transformer_sampler(xcgb, apf_f1b_f)
genL2b_f = F.spatial_transformer_sampler(xcgb, apf_f2b_f)
genL3b_f = F.spatial_transformer_sampler(xcgb, apf_f3b_f)
genL4b_f = F.spatial_transformer_sampler(xcgb, apf_f4b_f)
genL5b_f = F.spatial_transformer_sampler(xcgb, apf_f5b_f)
genL6b_f = F.spatial_transformer_sampler(xcgb, apf_f6b_f)
genL7b_f = F.spatial_transformer_sampler(xcgb, apf_f7b_f)
genL8b_f = F.spatial_transformer_sampler(xcgb, apf_f8b_f)
mask1_f = F.concat((w1f_f, w1b_f), axis=1)
mask2_f = F.concat((w2f_f, w2b_f), axis=1)
mask3_f = F.concat((w3f_f, w3b_f), axis=1)
mask4_f = F.concat((w4f_f, w4b_f), axis=1)
mask5_f = F.concat((w5f_f, w5b_f), axis=1)
mask6_f = F.concat((w6f_f, w6b_f), axis=1)
mask7_f = F.concat((w7f_f, w7b_f), axis=1)
mask8_f = F.concat((w8f_f, w8b_f), axis=1)
mask_soft1_f = F.softmax(mask1_f, axis=1)
mask_soft2_f = F.softmax(mask2_f, axis=1)
mask_soft3_f = F.softmax(mask3_f, axis=1)
mask_soft4_f = F.softmax(mask4_f, axis=1)
mask_soft5_f = F.softmax(mask5_f, axis=1)
mask_soft6_f = F.softmax(mask6_f, axis=1)
mask_soft7_f = F.softmax(mask7_f, axis=1)
mask_soft8_f = F.softmax(mask8_f, axis=1)
mask_sep1_f = F.separate(mask_soft1_f, axis=1)
mask_sep2_f = F.separate(mask_soft2_f, axis=1)
mask_sep3_f = F.separate(mask_soft3_f, axis=1)
mask_sep4_f = F.separate(mask_soft4_f, axis=1)
mask_sep5_f = F.separate(mask_soft5_f, axis=1)
mask_sep6_f = F.separate(mask_soft6_f, axis=1)
mask_sep7_f = F.separate(mask_soft7_f, axis=1)
mask_sep8_f = F.separate(mask_soft8_f, axis=1)
mask_1f_f = F.reshape(mask_sep1_f[0], (batchsize, 1, 128, 128))
mask_1b_f = F.reshape(mask_sep1_f[1], (batchsize, 1, 128, 128))
mask_2f_f = F.reshape(mask_sep2_f[0], (batchsize, 1, 128, 128))
mask_2b_f = F.reshape(mask_sep2_f[1], (batchsize, 1, 128, 128))
mask_3f_f = F.reshape(mask_sep3_f[0], (batchsize, 1, 128, 128))
mask_3b_f = F.reshape(mask_sep3_f[1], (batchsize, 1, 128, 128))
mask_4f_f = F.reshape(mask_sep4_f[0], (batchsize, 1, 128, 128))
mask_4b_f = F.reshape(mask_sep4_f[1], (batchsize, 1, 128, 128))
mask_5f_f = F.reshape(mask_sep5_f[0], (batchsize, 1, 128, 128))
mask_5b_f = F.reshape(mask_sep5_f[1], (batchsize, 1, 128, 128))
mask_6f_f = F.reshape(mask_sep6_f[0], (batchsize, 1, 128, 128))
mask_6b_f = F.reshape(mask_sep6_f[1], (batchsize, 1, 128, 128))
mask_7f_f = F.reshape(mask_sep7_f[0], (batchsize, 1, 128, 128))
mask_7b_f = F.reshape(mask_sep7_f[1], (batchsize, 1, 128, 128))
mask_8f_f = F.reshape(mask_sep8_f[0], (batchsize, 1, 128, 128))
mask_8b_f = F.reshape(mask_sep8_f[1], (batchsize, 1, 128, 128))
genL1x_f = F.scale(genL1f_f, mask_1f_f, axis=0) + F.scale(
genL1b_f, mask_1b_f, axis=0)
genL2x_f = F.scale(genL2f_f, mask_2f_f, axis=0) + F.scale(
genL2b_f, mask_2b_f, axis=0)
genL3x_f = F.scale(genL3f_f, mask_3f_f, axis=0) + F.scale(
genL3b_f, mask_3b_f, axis=0)
genL4x_f = F.scale(genL4f_f, mask_4f_f, axis=0) + F.scale(
genL4b_f, mask_4b_f, axis=0)
genL5x_f = F.scale(genL5f_f, mask_5f_f, axis=0) + F.scale(
genL5b_f, mask_5b_f, axis=0)
genL6x_f = F.scale(genL6f_f, mask_6f_f, axis=0) + F.scale(
genL6b_f, mask_6b_f, axis=0)
genL7x_f = F.scale(genL7f_f, mask_7f_f, axis=0) + F.scale(
genL7b_f, mask_7b_f, axis=0)
genL8x_f = F.scale(genL8f_f, mask_8f_f, axis=0) + F.scale(
genL8b_f, mask_8b_f, axis=0)
xap_f = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, 5.0 * genL8x_f),
axis=1)
apf0f_b = F.reshape(xsp[48], (batchsize, 1, 128, 128))
apf1f_b = F.reshape(xsp[49], (batchsize, 1, 128, 128))
apf2f_b = F.reshape(xsp[50], (batchsize, 1, 128, 128))
apf3f_b = F.reshape(xsp[51], (batchsize, 1, 128, 128))
apf4f_b = F.reshape(xsp[52], (batchsize, 1, 128, 128))
apf5f_b = F.reshape(xsp[53], (batchsize, 1, 128, 128))
apf6f_b = F.reshape(xsp[54], (batchsize, 1, 128, 128))
apf7f_b = F.reshape(xsp[55], (batchsize, 1, 128, 128))
apf8f_b = F.reshape(xsp[56], (batchsize, 1, 128, 128))
apf9f_b = F.reshape(xsp[57], (batchsize, 1, 128, 128))
apf10f_b = F.reshape(xsp[58], (batchsize, 1, 128, 128))
apf11f_b = F.reshape(xsp[59], (batchsize, 1, 128, 128))
apf12f_b = F.reshape(xsp[60], (batchsize, 1, 128, 128))
apf13f_b = F.reshape(xsp[61], (batchsize, 1, 128, 128))
apf14f_b = F.reshape(xsp[62], (batchsize, 1, 128, 128))
apf15f_b = F.reshape(xsp[63], (batchsize, 1, 128, 128))
apf0b_b = F.reshape(xsp[64], (batchsize, 1, 128, 128))
apf1b_b = F.reshape(xsp[65], (batchsize, 1, 128, 128))
apf2b_b = F.reshape(xsp[66], (batchsize, 1, 128, 128))
apf3b_b = F.reshape(xsp[67], (batchsize, 1, 128, 128))
apf4b_b = F.reshape(xsp[68], (batchsize, 1, 128, 128))
apf5b_b = F.reshape(xsp[69], (batchsize, 1, 128, 128))
apf6b_b = F.reshape(xsp[70], (batchsize, 1, 128, 128))
apf7b_b = F.reshape(xsp[71], (batchsize, 1, 128, 128))
apf8b_b = F.reshape(xsp[72], (batchsize, 1, 128, 128))
apf9b_b = F.reshape(xsp[73], (batchsize, 1, 128, 128))
apf10b_b = F.reshape(xsp[74], (batchsize, 1, 128, 128))
apf11b_b = F.reshape(xsp[75], (batchsize, 1, 128, 128))
apf12b_b = F.reshape(xsp[76], (batchsize, 1, 128, 128))
apf13b_b = F.reshape(xsp[77], (batchsize, 1, 128, 128))
apf14b_b = F.reshape(xsp[78], (batchsize, 1, 128, 128))
apf15b_b = F.reshape(xsp[79], (batchsize, 1, 128, 128))
w1f_b = F.reshape(xsp[80], (batchsize, 1, 128, 128))
w2f_b = F.reshape(xsp[81], (batchsize, 1, 128, 128))
w3f_b = F.reshape(xsp[82], (batchsize, 1, 128, 128))
w4f_b = F.reshape(xsp[83], (batchsize, 1, 128, 128))
w5f_b = F.reshape(xsp[84], (batchsize, 1, 128, 128))
w6f_b = F.reshape(xsp[85], (batchsize, 1, 128, 128))
w7f_b = F.reshape(xsp[86], (batchsize, 1, 128, 128))
w8f_b = F.reshape(xsp[87], (batchsize, 1, 128, 128))
w1b_b = F.reshape(xsp[88], (batchsize, 1, 128, 128))
w2b_b = F.reshape(xsp[89], (batchsize, 1, 128, 128))
w3b_b = F.reshape(xsp[90], (batchsize, 1, 128, 128))
w4b_b = F.reshape(xsp[91], (batchsize, 1, 128, 128))
w5b_b = F.reshape(xsp[92], (batchsize, 1, 128, 128))
w6b_b = F.reshape(xsp[93], (batchsize, 1, 128, 128))
w7b_b = F.reshape(xsp[94], (batchsize, 1, 128, 128))
w8b_b = F.reshape(xsp[95], (batchsize, 1, 128, 128))
apf_b1f_b = F.concat((apf0f_b, apf1f_b), axis=1)
apf_b2f_b = F.concat((apf2f_b, apf3f_b), axis=1)
apf_b3f_b = F.concat((apf4f_b, apf5f_b), axis=1)
apf_b4f_b = F.concat((apf6f_b, apf7f_b), axis=1)
apf_b5f_b = F.concat((apf8f_b, apf9f_b), axis=1)
apf_b6f_b = F.concat((apf10f_b, apf11f_b), axis=1)
apf_b7f_b = F.concat((apf12f_b, apf13f_b), axis=1)
apf_b8f_b = F.concat((apf14f_b, apf15f_b), axis=1)
apf_b1b_b = F.concat((apf0b_b, apf1b_b), axis=1)
apf_b2b_b = F.concat((apf2b_b, apf3b_b), axis=1)
apf_b3b_b = F.concat((apf4b_b, apf5b_b), axis=1)
apf_b4b_b = F.concat((apf6b_b, apf7b_b), axis=1)
apf_b5b_b = F.concat((apf8b_b, apf9b_b), axis=1)
apf_b6b_b = F.concat((apf10b_b, apf11b_b), axis=1)
apf_b7b_b = F.concat((apf12b_b, apf13b_b), axis=1)
apf_b8b_b = F.concat((apf14b_b, apf15b_b), axis=1)
genL1f_b = F.spatial_transformer_sampler(xcgf, apf_b1f_b)
genL2f_b = F.spatial_transformer_sampler(xcgf, apf_b2f_b)
genL3f_b = F.spatial_transformer_sampler(xcgf, apf_b3f_b)
genL4f_b = F.spatial_transformer_sampler(xcgf, apf_b4f_b)
genL5f_b = F.spatial_transformer_sampler(xcgf, apf_b5f_b)
genL6f_b = F.spatial_transformer_sampler(xcgf, apf_b6f_b)
genL7f_b = F.spatial_transformer_sampler(xcgf, apf_b7f_b)
genL8f_b = F.spatial_transformer_sampler(xcgf, apf_b8f_b)
genL1b_b = F.spatial_transformer_sampler(xcgb, apf_b1b_b)
genL2b_b = F.spatial_transformer_sampler(xcgb, apf_b2b_b)
genL3b_b = F.spatial_transformer_sampler(xcgb, apf_b3b_b)
genL4b_b = F.spatial_transformer_sampler(xcgb, apf_b4b_b)
genL5b_b = F.spatial_transformer_sampler(xcgb, apf_b5b_b)
genL6b_b = F.spatial_transformer_sampler(xcgb, apf_b6b_b)
genL7b_b = F.spatial_transformer_sampler(xcgb, apf_b7b_b)
genL8b_b = F.spatial_transformer_sampler(xcgb, apf_b8b_b)
mask1_b = F.concat((w1f_b, w1b_b), axis=1)
mask2_b = F.concat((w2f_b, w2b_b), axis=1)
mask3_b = F.concat((w3f_b, w3b_b), axis=1)
mask4_b = F.concat((w4f_b, w4b_b), axis=1)
mask5_b = F.concat((w5f_b, w5b_b), axis=1)
mask6_b = F.concat((w6f_b, w6b_b), axis=1)
mask7_b = F.concat((w7f_b, w7b_b), axis=1)
mask8_b = F.concat((w8f_b, w8b_b), axis=1)
mask_soft1_b = F.softmax(mask1_b, axis=1)
mask_soft2_b = F.softmax(mask2_b, axis=1)
mask_soft3_b = F.softmax(mask3_b, axis=1)
mask_soft4_b = F.softmax(mask4_b, axis=1)
mask_soft5_b = F.softmax(mask5_b, axis=1)
mask_soft6_b = F.softmax(mask6_b, axis=1)
mask_soft7_b = F.softmax(mask7_b, axis=1)
mask_soft8_b = F.softmax(mask8_b, axis=1)
mask_sep1_b = F.separate(mask_soft1_b, axis=1)
mask_sep2_b = F.separate(mask_soft2_b, axis=1)
mask_sep3_b = F.separate(mask_soft3_b, axis=1)
mask_sep4_b = F.separate(mask_soft4_b, axis=1)
mask_sep5_b = F.separate(mask_soft5_b, axis=1)
mask_sep6_b = F.separate(mask_soft6_b, axis=1)
mask_sep7_b = F.separate(mask_soft7_b, axis=1)
mask_sep8_b = F.separate(mask_soft8_b, axis=1)
mask_1f_b = F.reshape(mask_sep1_b[0], (batchsize, 1, 128, 128))
mask_1b_b = F.reshape(mask_sep1_b[1], (batchsize, 1, 128, 128))
mask_2f_b = F.reshape(mask_sep2_b[0], (batchsize, 1, 128, 128))
mask_2b_b = F.reshape(mask_sep2_b[1], (batchsize, 1, 128, 128))
mask_3f_b = F.reshape(mask_sep3_b[0], (batchsize, 1, 128, 128))
mask_3b_b = F.reshape(mask_sep3_b[1], (batchsize, 1, 128, 128))
mask_4f_b = F.reshape(mask_sep4_b[0], (batchsize, 1, 128, 128))
mask_4b_b = F.reshape(mask_sep4_b[1], (batchsize, 1, 128, 128))
mask_5f_b = F.reshape(mask_sep5_b[0], (batchsize, 1, 128, 128))
mask_5b_b = F.reshape(mask_sep5_b[1], (batchsize, 1, 128, 128))
mask_6f_b = F.reshape(mask_sep6_b[0], (batchsize, 1, 128, 128))
mask_6b_b = F.reshape(mask_sep6_b[1], (batchsize, 1, 128, 128))
mask_7f_b = F.reshape(mask_sep7_b[0], (batchsize, 1, 128, 128))
mask_7b_b = F.reshape(mask_sep7_b[1], (batchsize, 1, 128, 128))
mask_8f_b = F.reshape(mask_sep8_b[0], (batchsize, 1, 128, 128))
mask_8b_b = F.reshape(mask_sep8_b[1], (batchsize, 1, 128, 128))
genL1x_b = F.scale(genL1f_b, mask_1f_b, axis=0) + F.scale(
genL1b_b, mask_1b_b, axis=0)
genL2x_b = F.scale(genL2f_b, mask_2f_b, axis=0) + F.scale(
genL2b_b, mask_2b_b, axis=0)
genL3x_b = F.scale(genL3f_b, mask_3f_b, axis=0) + F.scale(
genL3b_b, mask_3b_b, axis=0)
genL4x_b = F.scale(genL4f_b, mask_4f_b, axis=0) + F.scale(
genL4b_b, mask_4b_b, axis=0)
genL5x_b = F.scale(genL5f_b, mask_5f_b, axis=0) + F.scale(
genL5b_b, mask_5b_b, axis=0)
genL6x_b = F.scale(genL6f_b, mask_6f_b, axis=0) + F.scale(
genL6b_b, mask_6b_b, axis=0)
genL7x_b = F.scale(genL7f_b, mask_7f_b, axis=0) + F.scale(
genL7b_b, mask_7b_b, axis=0)
genL8x_b = F.scale(genL8f_b, mask_8f_b, axis=0) + F.scale(
genL8b_b, mask_8b_b, axis=0)
xap_b = F.concat((genL1x_b, genL2x_b, genL3x_b, genL4x_b, genL5x_b,
genL6x_b, genL7x_b, 5.0 * genL8x_b),
axis=1)
#VUNet-360 end
#GONet
xgonet = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, genL8x_f),
axis=0)
with chainer.using_config('train', False):
img_gen = gen(invg(xgonet))
dis_real = dis(xgonet)
dis_gen = dis(img_gen)
output = fl(xgonet - img_gen, dis_real - dis_gen, dis_real)
xrefy = F.concat((Variable(cur_img), Variable(goal_img)), axis=3)
xgonetx = F.concat((genL1x_f, genL2x_f, genL3x_f, genL4x_f, genL5x_f,
genL6x_f, genL7x_f, genL8x_f),
axis=3)
xgonety = F.concat((genL1x_b, genL2x_b, genL3x_b, genL4x_b, genL5x_b,
genL6x_b, genL7x_b, genL8x_b),
axis=3)
msg_pub = Twist()
#velocity cap and publish
vt = cuda.to_cpu(vresg.data[0][0])
wt = cuda.to_cpu(wresg.data[0][0])
if np.absolute(vt) < 0.2:
msg_pub.linear.x = vt
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = wt
else:
if np.absolute(wt) < 0.001:
msg_pub.linear.x = 0.2 * np.sign(vt)
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = 0.0
else:
rd = vt / wt
msg_pub.linear.x = 0.2 * np.sign(vt)
msg_pub.linear.y = 0.0
msg_pub.linear.z = 0.0
msg_pub.angular.x = 0.0
msg_pub.angular.y = 0.0
msg_pub.angular.z = 0.2 * np.sign(vt) / rd
#front predicted image
out_imgc = cuda.to_cpu(xgonetx.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 128 * 8))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_genf.publish(imgm)
#back predicted image
out_imgc = cuda.to_cpu(xgonety.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 128 * 8))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_genb.publish(imgm)
#current and goal image
out_imgc = cuda.to_cpu(xrefy.data)
imgb = np.fmin(255.0, np.fmax(0.0, out_imgc * 128 + 128))
imgc = np.reshape(imgb, (3, 128, 256 * 2))
imgd = imgc.transpose(1, 2, 0)
imge = imgd.astype(np.uint8)
imgm = bridge.cv2_to_imgmsg(imge)
image_ref.publish(imgm)
#velocities
msg_out.publish(msg_pub)
j = 0
def test_invalid_axis(self):
with self.assertRaises(TypeError):
functions.flip(self.x, 'a')
def check_type_error(self, x):
with self.assertRaises(type_check.InvalidType):
functions.flip(x, self.axis)
def forward(self, inputs, devices):
x, = inputs
y = functions.flip(x, self.axis)
return y,
def f(x):
x = functions.flip(x, axis)
return x * x
def check_backward(self, x_data, axis, y_grad):
gradient_check.check_backward(lambda x: functions.flip(x, axis),
x_data, y_grad, dtype=numpy.float64)
def f(x):
return functions.flip(x, axis)
def f(x):
return functions.flip(x, axis)
def irfft(real, imag):
real = F.concat((real, F.flip(real[..., 1:-1], -1)), -1)
imag = F.concat((imag, F.flip(-imag[..., 1:-1], -1)), -1)
ret, _ = F.ifft((real, imag))
return ret
