def check_invalid_option(self, xp):
x0 = xp.asarray(self.x0)
x1 = xp.asarray(self.x1)
t = xp.asarray(self.t)
with self.assertRaises(ValueError):
functions.contrastive(x0, x1, t, 1, 'invalid_option')
def batch(self):
a, b = self.load()
self.model.cleargrads()
y = tuple(self.executor.map(self.model, a + b))
loss = F.contrastive(y[0], y[1], [1]) +\
F.contrastive(y[2], y[3], [1]) +\
F.contrastive(y[0], y[2], [0]) +\
F.contrastive(y[1], y[3], [0])
loss.backward()
self.optimizer.update()
return loss.data.get()
def check_forward(self, x0_data, x1_data, t_data):
x0_val = chainer.Variable(x0_data)
x1_val = chainer.Variable(x1_data)
t_val = chainer.Variable(t_data)
loss = functions.contrastive(
x0_val, x1_val, t_val, self.margin, self.reduce)
self.assertEqual(loss.data.dtype, numpy.float32)
if self.reduce == 'mean':
self.assertEqual(loss.data.shape, ())
else:
self.assertEqual(loss.data.shape, (self.batchsize,))
loss_value = cuda.to_cpu(loss.data)
# Compute expected value
loss_expect = numpy.empty((self.batchsize,), numpy.float32)
for i in six.moves.range(self.x0.shape[0]):
x0d, x1d, td = self.x0[i], self.x1[i], self.t[i]
d = numpy.sum((x0d - x1d) ** 2)
if td == 1: # similar pair
loss_expect[i] = d
elif td == 0: # dissimilar pair
loss_expect[i] = max(self.margin - math.sqrt(d), 0) ** 2
loss_expect[i] /= 2.
if self.reduce == 'mean':
loss_expect = numpy.sum(loss_expect) / self.t.shape[0]
numpy.testing.assert_allclose(loss_expect, loss_value, rtol=1e-2)
def __call__(self, variables):
in_audio, curr_step, nx_step = variables
batchsize = in_audio.shape[0]
sequence = in_audio.shape[1]
self.loss = loss = 0
state = self.state
self.ot = xp.std(curr_step, axis=1)
for i in range(sequence):
h, state, y = self.forward(state, curr_step,
self.audiofeat(in_audio[:, i]), True)
loss += F.mean_squared_error(nx_step[:, i], y)
curr_step = y
ot = xp.std(nx_step[:, i], axis=1) * batchsize # y
delta_sgn = xp.sign(ot - self.ot)
if i > 0:
labels = xp.minimum(delta_sgn + self.delta_sgn, 1)
labels = xp.asarray(labels, dtype=xp.int32)
loss2 = F.contrastive(
h, self.h, labels
) / sequence # .mean_squared_error mean_absolute_error
if float(chainer.cuda.to_cpu(loss2.data)) > 0.:
loss += loss2 # F.mean_squared_error mean_absolute_error
self.h = h
self.ot = ot
self.delta_sgn = delta_sgn
self.loss = loss
stdout.write('loss={:.04f}\r'.format(
float(chainer.cuda.to_cpu(loss.data))))
stdout.flush()
return self.loss
def forward(self, x0, t0, x1, t1):
h0 = super(CNN_Siamese, self).forward(x0)
h1 = super(CNN_Siamese, self).forward(x1)
t = cp.equal(t0, t1).astype(int)
return F.contrastive(h0, h1, t)
def predict(self, v_feat, actions, reasons, label, ocr, layers=['dist']):
outputs = {}
# word embedding
ya_emb = self.lng_net.word_embedding(actions)
yr_emb = self.lng_net.word_embedding(reasons)
yocr_emb = self.lng_net.word_embedding(ocr)
yocr_emb = F.repeat(yocr_emb, 15, axis=0)
# action feat
h_act = self.lng_net(actions)
h_act = F.relu(self.act_l(h_act))
h_act = F.normalize(h_act)
# reason feat
h_rsn = self.lng_net(reasons)
h_rsn = F.relu(self.rsn_l(h_rsn))
h_rsn = F.normalize(h_rsn)
# attention over ocr words (action)
h_ocr_act = self.encode_ocr(yocr_emb, ya_emb)
h_ocr_act = F.normalize(h_ocr_act)
loss_act = F.contrastive(h_ocr_act, h_act, label, margin=self.margin)
# attention over ocr words (reason)
h_ocr_rsn = self.encode_ocr(yocr_emb, yr_emb)
h_ocr_rsn = F.normalize(h_ocr_rsn)
loss_rsn = F.contrastive(h_ocr_rsn, h_rsn, label, margin=self.margin)
outputs['loss'] = (loss_act + loss_rsn) * .5
# distance between ocr feature and target text
if 'dist' in layers:
dist_ocr = l2_distance(h_ocr_act, h_act)
dist_rsn = l2_distance(h_ocr_rsn, h_rsn)
dist = (dist_ocr + dist_rsn) * .5
outputs['dist'] = dist
return outputs
def check_backward(self, x0_data, x1_data, t_data):
x0 = chainer.Variable(x0_data)
x1 = chainer.Variable(x1_data)
t = chainer.Variable(t_data)
loss = functions.contrastive(x0, x1, t, self.margin)
loss.backward()
self.assertEqual(None, t.grad)
func = loss.creator
f = lambda: func.forward((x0.data, x1.data, t.data))
gx0, = gradient_check.numerical_grad(f, (x0.data, ), (1, ))
gx1, = gradient_check.numerical_grad(f, (x1.data, ), (1, ))
gradient_check.assert_allclose(gx0, x0.grad, rtol=1e-4, atol=1e-4)
gradient_check.assert_allclose(gx1, x1.grad, rtol=1e-4, atol=1e-4)
def check_backward(self, x0_data, x1_data, t_data):
x0 = chainer.Variable(x0_data)
x1 = chainer.Variable(x1_data)
t = chainer.Variable(t_data)
loss = functions.contrastive(x0, x1, t, self.margin)
loss.backward()
self.assertEqual(None, t.grad)
func = loss.creator
f = lambda: func.forward((x0.data, x1.data, t.data))
gx0, = gradient_check.numerical_grad(f, (x0.data,), (1,))
gx1, = gradient_check.numerical_grad(f, (x1.data,), (1,))
gradient_check.assert_allclose(gx0, x0.grad, rtol=1e-4, atol=1e-4)
gradient_check.assert_allclose(gx1, x1.grad, rtol=1e-4, atol=1e-4)
def metric_loss_average(model, x_data, t_data, num_batches, train):
# accuracies = []
losses = []
total_data = np.arange(len(x_data))
for indexes in np.array_split(total_data, num_batches):
X_batch = cuda.to_gpu(x_data[indexes])
T_batch = cuda.to_gpu(t_data[indexes])
# print("a")
with chainer.using_config('train', train):
# 順伝播させる
Y = model.__call__(X_batch, train)
Y1, Y2 = F.split_axis(Y, 2, axis=0)
T1, T2 = F.split_axis(T_batch, 2, axis=0)
T = (T1.array == T2.array).astype(np.int32)
loss = F.contrastive(Y1, Y2, T)
loss_cpu = cuda.to_cpu(loss.data)
losses.append(loss_cpu)
return np.mean(losses)
def check_forward(self, x0_data, x1_data, t_data):
x0_val = chainer.Variable(x0_data)
x1_val = chainer.Variable(x1_data)
t_val = chainer.Variable(t_data)
loss = functions.contrastive(x0_val, x1_val, t_val, self.margin)
self.assertEqual(loss.data.shape, ())
self.assertEqual(loss.data.dtype, numpy.float32)
loss_value = float(cuda.to_cpu(loss.data))
# Compute expected value
loss_expect = 0
for i in six.moves.range(self.x0.shape[0]):
x0d, x1d, td = self.x0[i], self.x1[i], self.t[i]
d = numpy.sum((x0d - x1d) ** 2)
if td == 1: # similar pair
loss_expect += d
elif td == 0: # dissimilar pair
loss_expect += max(self.margin - math.sqrt(d), 0) ** 2
loss_expect /= 2.0 * self.t.shape[0]
self.assertAlmostEqual(loss_expect, loss_value, places=5)
def check_forward(self, x0_data, x1_data, t_data):
x0_val = chainer.Variable(x0_data)
x1_val = chainer.Variable(x1_data)
t_val = chainer.Variable(t_data)
loss = functions.contrastive(x0_val, x1_val, t_val, self.margin)
self.assertEqual(loss.data.shape, ())
self.assertEqual(loss.data.dtype, numpy.float32)
loss_value = float(cuda.to_cpu(loss.data))
# Compute expected value
loss_expect = 0
for i in six.moves.range(self.x0.shape[0]):
x0d, x1d, td = self.x0[i], self.x1[i], self.t[i]
d = numpy.sum((x0d - x1d)**2)
if td == 1: # similar pair
loss_expect += d
elif td == 0: # dissimilar pair
loss_expect += max(self.margin - math.sqrt(d), 0)**2
loss_expect /= 2.0 * self.t.shape[0]
self.assertAlmostEqual(loss_expect, loss_value, places=5)
def train(view, set):
# 5分割されたデータのロード
gallery1, probe1 = load_res_invariant_dataset(view=view,
set=set,
type='SFDEI',
frame='1')
gallery2, probe2 = load_res_invariant_dataset(view=view,
set=set,
type='SFDEI',
frame='2')
# tensor-boardの定義
writer = SummaryWriter()
# 学習の設定
save_dir = '/home/common-ns/PycharmProjects/Multiscale_SFDEINet/4inputs/models' + view + '/set_' + str(
set)
# save_dir = '/home/common-ns/setoguchi/chainer_files/Two_by_two_in/SRGAN1-2/' + view + '/set_' + str(set)
os.mkdir(save_dir)
batch_size = 239
max_iteration = 50000
id = 1
# gpu_id = chainer.backends.cuda.get_device_from_id(id)
model = two_by_two_in()
model.to_gpu(id)
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
optimizer.setup(model)
# optimizer.add_hook(chainer.optimizer.WeightDecay(0.0005))
iteration = 1.0
i = 0
# 学習ループ
# for i in range(1, epock+1):
while iteration < max_iteration + 1:
accum_loss = 0.0
count = 0
# データから正例，負例のペアを構築．少なくとも全データの1割が正例になるようにする
# データに偏りが出ないよう，データのタイプを1/4の確率で決定
select_seed = np.random.randint(0, 2)
if select_seed == 0:
train_data = create_pair(gallery1, gallery2, probe1, probe2)
elif select_seed == 1:
train_data = create_pair(probe1, probe2, gallery1, gallery2)
elif select_seed == 2:
train_data = create_pair(gallery1, gallery2, gallery1, gallery2)
elif select_seed == 3:
train_data = create_pair(probe1, probe2, probe1, probe2)
# バッチをつくる前にシャッフル
shuffle = random.sample(train_data, len(train_data))
# ミニバッチに分割
mini_batches = make_batch_list(shuffle, batch_size)
for batch in mini_batches:
# リストから1つのnumpyバッチにする
pos1, pos2, neg1, neg2, signal = list2npy(batch, id)
# Forward
g_out, p_out = model(pos1, pos2, neg1, neg2)
# lossの計算
signal = F.flatten(signal) # contrastive_lossの仕様に合わせて1次元化
loss = F.contrastive(g_out, p_out, signal, margin=3, reduce='mean')
# print cuda.to_cpu(loss.data)
accum_loss += cuda.to_cpu(loss.data)
# backward
model.cleargrads()
loss.backward()
# パラメータ更新
optimizer.update()
if iteration % 10000 == 0:
optimizer.lr = optimizer.lr * 0.1
print('学習率がさがりました：{}'.format(optimizer.lr))
if iteration % 5000 == 0:
serializers.save_npz(
save_dir + '/model_snapshot_{}'.format(int(iteration)),
model)
iteration += 1.0
i += 1
print('epock:{}'.format(i), 'iteration:{}'.format(int(iteration)),
'accum_loss:{}'.format(accum_loss / float(len(mini_batches))))
writer.add_scalar('train/loss', accum_loss / float(len(mini_batches)),
i)
# 約1000イテレーション毎にモデルを保存
# if iteration % 10000 == 0:
# serializers.save_npz(save_dir + '/model_snapshot_{}'.format(i), model)
g = c.build_computational_graph(g_out[0])
with open(save_dir + '/graph.dot', 'w') as o:
o.write(g.dump())
def __call__(self, x0, x1, d):
y0 = self.predictor(x0)
y1 = self.predictor(x1)
self.loss = F.contrastive(y0, y1, d)
return self.loss
def f(x0, x1, t):
return functions.contrastive(x0, x1, t, self.margin, self.reduce)
def predict(self,
v_feat,
actions,
reasons,
label,
ocr,
layers=['dist'],
alpha=.9):
outputs = {}
ocr_on = self.xp.asarray([x.size > 0 for x in ocr],
dtype=np.float32).repeat(15)
# word embedding
ya_emb = self.lng_net.word_embedding(actions)
yr_emb = self.lng_net.word_embedding(reasons)
yocr_emb = self.lng_net.word_embedding(ocr)
yocr_emb = F.repeat(yocr_emb, 15, axis=0)
# action feat
h_act = self.lng_net(actions)
# reason feat
h_rsn = self.lng_net(reasons)
# attention over ocr words (action)
h_ocr_act = self.encode_ocr(yocr_emb, ya_emb)
# attention over ocr words (reason)
h_ocr_rsn = self.encode_ocr(yocr_emb, yr_emb)
y_ocr_act, y_act, y_ocr_rsn, y_rsn = self.ocr_mapping_net(
h_ocr_act, h_act, h_ocr_rsn, h_rsn)
# loss ocr
loss_act = F.contrastive(y_ocr_act,
y_act,
label,
margin=self.margin,
reduce='no')
loss_rsn = F.contrastive(y_ocr_rsn,
y_rsn,
label,
margin=self.margin,
reduce='no')
loss_ocr = ocr_on * (loss_act + loss_rsn) * .5
z_vis_act, z_act, z_vis_rsn, z_rsn = self.vis_mapping_net(
v_feat, h_act, h_rsn)
# loss visual
loss_act_vis = F.contrastive(z_vis_act,
z_act,
label,
margin=self.margin,
reduce='no')
loss_rsn_vis = F.contrastive(z_vis_rsn,
z_rsn,
label,
margin=self.margin,
reduce='no')
loss_vis = (loss_act_vis + loss_rsn_vis) * .5
loss = F.mean(alpha * loss_ocr + (1 - alpha) * loss_vis)
# loss = F.mean(loss_ocr)
if 'loss_ocr' in layers:
outputs['loss_ocr'] = loss_ocr
if 'loss_vis' in layers:
outputs['loss_vis'] = loss_vis
if 'loss' in layers:
outputs['loss'] = loss
# distance between ocr feature and target text
if 'dist' in layers:
dist_act = l2_distance(y_ocr_act, y_act)
dist_rsn = l2_distance(y_ocr_rsn, y_rsn)
dist_ocr = ocr_on * (dist_act + dist_rsn) * .5
dist_act = l2_distance(z_vis_act, z_act)
dist_rsn = l2_distance(z_vis_rsn, z_rsn)
dist_vis = (dist_act + dist_rsn) * .5
outputs['dist'] = alpha * dist_ocr + (1 - alpha) * dist_vis
if 'dist_ocr' in layers:
outputs['dist_ocr'] = dist_ocr
if 'dist_vis' in layers:
outputs['dist_vis'] = dist_vis
return outputs
def __call__(self, x0, x1, d):
y0 = self.predictor(x0)
y1 = self.predictor(x1)
self.loss = F.contrastive(y0, y1, d)
return self.loss
def f(x0, x1):
y = functions.contrastive(x0, x1, t_data, self.margin, self.reduce)
return y * y
time_start = time.time()
perm = np.random.permutation(num_train)
for batch_indexes in np.split(perm, num_batches):
x_batch = cuda.to_gpu(X_train[batch_indexes])
t_batch = cuda.to_gpu(T_train[batch_indexes])
# 勾配を初期化する
model.zerograds()
# 順伝播させる
y_batch = model.__call__(x_batch, True)
# contrastive lossに入力するy1,y2,tを取得する
y1, y2 = F.split_axis(y_batch, 2, axis=0)
t1, t2 = F.split_axis(t_batch, 2, axis=0)
t = (t1.array == t2.array).astype(np.int32)
loss = F.contrastive(y1, y2, t)
# print("loss:", loss)
# 逆伝播
loss.backward()
optimizer.update()
time_finish = time.time()
time_elapsed = time_finish - time_start
# print ("time_elapsed:", time_elapsed)
# 訓練データセットの交差エントロピー誤差を表示する
train_loss = M.metric_loss_average(model, X_train, T_train,
num_batches, False)
loss_train_history.append(train_loss)
print("[train] Loss:", train_loss)
# 訓練データからtop_kを求める
