def __call__(self, image, joints, is_valid_joints, crop_bbox, orig_bbox):
## predict joints, and calculate loss
y = self.predictor(image)
with get_device(image):
t = (joints * is_valid_joints).astype(joints.dtype)
loss = mean_squared_error(y, t)
joints = Variable(joints)
joints.to_cpu()
joints = joints.data.reshape(-1, 14, 2)
crop_bbox = Variable(crop_bbox)
crop_bbox.to_cpu()
crop_bbox = crop_bbox.data
y.to_cpu()
y = y.data.reshape(-1, 14, 2)
## evaluate with mPCP metrics
pcp_per_stick = calculate_metric(joints, y, crop_bbox,
self.dataset_name, 'PCP')
pcp_per_part, pcp_part_names = \
poseevaluation.pcp.average_pcp_left_right_limbs(pcp_per_stick)
## Report above meansured value
report({'loss': loss, 'mPCP': np.mean(pcp_per_part)}, self)
def to_device(elem: chainer.Variable, device=None):
if device is None:
return elem
elif device < 0:
elem.to_cpu()
else:
elem.to_gpu(device=device)
def __call__(self, x, t, dataset, train=True):
# Create variables
x = Variable(x)
x.to_gpu(self.gpu_id)
t = Variable(t)
t.to_gpu(self.gpu_id)
# Config mode
if len(t.shape) == 3:
config_mode = 'segmentation'
elif len(t.shape) == 2:
config_mode = 'recognition'
else:
raise ValueError('label format is not supported')
# Forward
with chainer.using_config('train', train):
with chainer.using_config('enable_backprop', train):
# InceptionV3 backbone
x = self.predictor(x)
# Classifiers
classifier_indx = self.args.dataset.split('+').index(dataset)
y = self.classifiers[classifier_indx](x, train)
# Loss
if config_mode == 'segmentation':
self.y = F.resize_images(y,
t.shape[-2:]) # Upsampling logits
self.loss = F.softmax_cross_entropy(self.y, t)
elif config_mode == 'recognition':
self.y = F.squeeze(F.average_pooling_2d(
y, ksize=y.shape[-2:]),
axis=(2, 3)) # Global Average Pooling
self.loss = F.sigmoid_cross_entropy(self.y, t)
# Backward
if train:
# Clear grads for uninitialized params
self.cleargrads()
# Backwards
self.loss.backward()
# Reporter
if config_mode == 'segmentation':
self.y = F.argmax(self.y, axis=1)
self.y.to_cpu()
t.to_cpu()
result = eval_semantic_segmentation(list(self.y.data),
list(t.data))
del result['iou'], result['class_accuracy']
result.update({'loss': self.loss.data.tolist()})
self.reporter.update({dataset: result})
elif config_mode == 'recognition':
self.reporter.update({
dataset: {
'loss': self.loss.data.tolist(),
'prediction': F.sigmoid(self.y).data.tolist(),
'groundtruth': t.data.tolist()
}
})
class LSTM(L.NStepLSTM):
def __init__(self, in_size, out_size, dropout=0, use_cudnn='auto'):
n_layers = 2
self.state_size = out_size
with chainer.configuration.using_config('use_cudnn', use_cudnn):
super(LSTM, self).__init__(n_layers, in_size, out_size, dropout)
with self.init_scope():
self.reset_state()
def __call__(self, xs, train=True):
batch = len(xs)
if self.hx is None:
xp = self.xp
with chainer.using_config('volatile', 'auto'):
self.hx = Variable(
xp.zeros((self.n_layers, batch, self.state_size),
dtype=xs[0].dtype))
if self.cx is None:
xp = self.xp
with chainer.using_config('volatile', 'auto'):
self.cx = Variable(
xp.zeros((self.n_layers, batch, self.state_size),
dtype=xs[0].dtype))
with chainer.configuration.using_config('train', train):
hy, cy, ys = super(LSTM, self).__call__(self.hx, self.cx, xs)
self.hx, self.cx = hy, cy
return ys
def reset_state(self):
self.hx = None
self.cx = None
def to_cpu(self):
super(LSTM, self).to_cpu()
if self.cx is not None:
self.cx.to_cpu()
if self.hx is not None:
self.hx.to_cpu()
def to_gpu(self, device=None):
super(LSTM, self).to_gpu(device)
if self.cx is not None:
self.cx.to_gpu(device)
if self.hx is not None:
self.hx.to_gpu(device)
def eval_func(self, label):
# chainer.functionのtype_checkを通過させるため、[]のwrapperをつける
xp = cuda.get_array_module(self.pool.variable.data)
mean_activations = Variable(xp.array(
[xp.mean(self.pool.variable.data, axis=0)],
dtype=xp.float32))
label = Variable(xp.array(label, dtype=xp.int32))
# cpuに移す
mean_activations.to_cpu()
label.to_cpu()
mean_activations = mean_activations.data
label = label.data
# top1, top5の算出
res_top1 = mean_activations.argmax()
res_top5 = mean_activations.argsort()[0][-5:]
top1 = 1.0 if label[0] == res_top1 else 0.0
top5 = 1.0 if label[0] in res_top5 else 0.0
report({'top1': top1, 'top5': top5})
def eval_func(self, label):
# chainer.functionのtype_checkを通過させるため、[]のwrapperをつける
xp = cuda.get_array_module(self.pool.variable.data)
mean_activations = Variable(xp.array(
[xp.mean(self.pool.variable.data, axis=0)],
dtype=xp.float32))
label = Variable(xp.array(label, dtype=xp.int32))
# cpuに移す
mean_activations.to_cpu()
label.to_cpu()
mean_activations = mean_activations.data
label = label.data
# top1, top5の算出
res_top1 = mean_activations.argmax()
res_top5 = mean_activations.argsort()[0][-5:]
# ndarray(numpy) -> list(pure python)
label.tolist()
res_top1 = int(res_top1)
res_top5.tolist()
# label[0] は imageに対応する元々のlabel
# label はcommon_root_labels
ref_top1 = 1.0 if res_top1 in label else 0.0
ref_top5 = 1.0 if set(res_top5) & set(label) else 0.0
org_top1 = 1.0 if label[0] == res_top1 else 0.0
org_top5 = 1.0 if label[0] in res_top5 else 0.0
report({
'ref/top1': ref_top1,
'ref/top5': ref_top5,
'org/top1': org_top1,
'org/top5': org_top5 })
x = Variable(cuda.to_gpu(x_train[perm[i:i+batch_size]]))#教師画像データ
t = Variable(cuda.to_gpu(t_train[perm[i:i+batch_size]]))#教師ラベル
y = model.forward(x)
loss = F.softmax_cross_entropy(y, t) # 活性化関数と損失関数
model.zerograds()
loss.backward()
#誤判別を特定するプログラム
acc, data, data2, index, pred, corre = F.accuracy(y, t)
acc.to_cpu()
data.to_cpu()
data2.to_cpu()
index.to_cpu()
pred.to_cpu()
corre.to_cpu()
t.to_cpu()
x.to_cpu()
if epoch == n_epoch:
for i in range(len(data.data)):
miss_train_total[data.data[i]] += 1
miss_train_judge[data.data[i]][data2.data[i]] += 1
for i in range(len(corre.data)):
correct_train_total[corre.data[i]] += 1
#誤判別した画像を出力するプログラム
for inde in index.data:
e = np.array(x.data[inde])
r = np.array(e[0])#赤色
g = np.array(e[1])#緑色
b = np.array(e[2])#青色
def __call__(self, x, t, dataset, train=True):
# Create variables
x = Variable(x)
x.to_gpu(self.gpu_id)
t = Variable(t)
t.to_gpu(self.gpu_id)
with chainer.using_config('train', False):
with chainer.using_config('enable_backprop', False):
xo = self.segmentation.predictor(x)
y = self.segmentation.classifiers[0](xo)
y = F.separate(F.softmax(y), axis=1)
# foreground, head, torso-hand, lower-body, shoes
segprob = F.stack(
(1.0 - y[0], y[1] + y[2] + y[4] + y[13],
y[5] + y[6] + y[7] + y[11] + y[10] + y[3] + y[14] + y[15],
y[9] + y[16] + y[17] + y[12], y[18] + y[19] + y[8]),
axis=1)
# Forward
with chainer.using_config('train', train):
with chainer.using_config('enable_backprop', train):
x = F.resize_images(x, self.args.scales_reid)
# InceptionV3 backbone
x = self.predictor(x)
x_a = F.average_pooling_2d(x, x.shape[-2:])
# Resize to segmentation map resolution
x = F.resize_images(x, segprob.shape[-2:])
# aggregate features at semantic parts
xl = F.scale(F.batch_matmul(
F.reshape(segprob,
(segprob.shape[0], segprob.shape[1], -1)),
F.reshape(x, (x.shape[0], x.shape[1], -1)),
transb=True),
1.0 / F.sum(segprob, axis=(2, 3)),
axis=0)
xfg, xl = F.split_axis(xl, [1], axis=1)
xl = F.max(xl, axis=1, keepdims=True)
x = F.concat((xfg, xl), axis=2)
# Classifiers
x_s = F.reshape(x, (-1, 2 * 2048, 1, 1))
x = F.concat((x_s, x_a), axis=1)
if train:
self.y_s = self.classifiers[0](x)
# Loss
self.loss = F.softmax_cross_entropy(
F.squeeze(self.y_s, axis=(2, 3)), t)
# Clear grads for uninitialized params
self.cleargrads()
# Backwards
self.loss.backward()
# Reporter
self.reporter.update(
{dataset: {
'loss': self.loss.data.tolist()
}})
else:
x = F.squeeze(x)
x.to_cpu()
self.reporter.update({dataset: {'features': x.data}})
out, layers=[layer])[layer]
# tv
dx = out[:, :, 1:, :-1] - out[:, :, :-1, :-1]
dy = out[:, :, :-1, 1:] - out[:, :, :-1, :-1]
tv = F.sqrt(dx**2 + dy**2 + 1e-8)
## cycle
with chainer.using_config('train', False):
if is_AE:
cycle = dec_i(enc_i(out))
else:
cycle = gen_i(out)
diff = cycle - imgs
# diff = gradimg(cycle)-gradimg(imgs)
img_disx.to_cpu()
img_disy.to_cpu()
imgs.to_cpu()
diff.to_cpu()
cycle.to_cpu()
tv.to_cpu()
perc_diff.to_cpu()
img_disx = img_disx.array
img_disy = img_disy.array
imgs = imgs.array
cycle_diff = diff.array
cycle = cycle.array
tv = tv.array
perc_diff = perc_diff.array
##
out.to_cpu()
out = out.array
x, y = yzmData.nextBatch(testOrTrain="train", batch_size=64)
x = Variable(x.astype(np.float32))
y = Variable(y.astype(np.int32))
x.to_gpu(0)
y.to_gpu(0)
opt.update(loss, x.data, y.data)
print("step:%d\ttotal_loss:%f\terro_rate:%f" % (step, model.loss.data, model.acc.data))
if step % 100 == 0:
model.save_model("./model.npz", save_format="npz")
serializers.save_npz("./opt.npz", opt)
test_x, test_t = yzmData.nextBatch(testOrTrain="test", batch_size=40)
test_x = Variable(test_x.astype(np.float32))
test_t = Variable(test_t.astype(np.int32))
test_x.to_gpu(0)
test_t.to_gpu(0)
infer_y = model.infer(test_x.data)
print(infer_y)
infer_y.to_cpu()
test_x.to_cpu()
showInputLabels(test_x.data, infer_y.data, yzmData.id2class)
break
# break
step += 1
def train(batch_size, epoch_count, lamda, datasetA_folder_path,
datasetB_folder_path, output_path):
print("Start load images data from " + datasetA_folder_path)
dataset_A = data_io.dataset_load(datasetA_folder_path)
print("Finish load images data from " + datasetA_folder_path)
print("Start load images data from " + datasetB_folder_path)
dataset_B = data_io.dataset_load(datasetB_folder_path)
print("Finish load images data from " + datasetB_folder_path)
if len(dataset_A) != len(dataset_B):
print("Error! Datasets are not paired data.")
exit()
gen = Generator()
dis = Discriminator()
gen.to_gpu(0)
dis.to_gpu(0)
opt_g = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_g.setup(gen)
opt_d = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_d.setup(dis)
iteration = 0
log_list = []
image_path = output_path + "image/"
dis_model_path = output_path + "dis/"
gen_model_path = output_path + "gen/"
os.mkdir(output_path)
os.mkdir(image_path)
os.mkdir(dis_model_path)
os.mkdir(gen_model_path)
dataset_num = min(dataset_A.shape[0], dataset_B.shape[0])
for epoch in range(epoch_count):
d_loss_list = []
g_loss_list = []
for i in range(dataset_num // batch_size):
input_image = dataset_A[i * batch_size:(i + 1) * batch_size]
input_image = Variable(input_image)
input_image.to_gpu(0)
correct_image = dataset_B[i * batch_size:(i + 1) * batch_size]
correct_image = Variable(correct_image)
correct_image.to_gpu(0)
fake_image = gen(input_image)
d_real_result = dis(correct_image)
d_fake_result = dis(fake_image)
loss_d = loss_dis(batch_size, d_real_result, d_fake_result)
dis.cleargrads()
loss_d.backward()
opt_d.update()
"""generatorのloss計算"""
loss_g = loss_gen(d_fake_result, fake_image, correct_image, lamda)
gen.cleargrads()
loss_g.backward()
opt_g.update()
loss_d.to_cpu()
loss_g.to_cpu()
iteration += batch_size
d_loss_list.append(loss_d.array)
g_loss_list.append(loss_g.array)
input_image.to_cpu()
correct_image.to_cpu()
fake_image.to_cpu()
input_images = input_image.array.transpose(0, 2, 3, 1)
correct_images = correct_image.array.transpose(0, 2, 3, 1)
fake_images = fake_image.array.transpose(0, 2, 3, 1)
data_io.output_images(image_path + str(epoch), input_images,
correct_images, fake_images)
print("epoch: " + str(epoch) + ", interation: " + str(iteration) + \
", d_loss: " + str(np.mean(d_loss_list)) + ", g_loss: " + str(np.mean(g_loss_list)))
log_json = {"epoch": str(epoch), "interation": str(iteration), \
"d_loss": str(np.mean(d_loss_list)),"g_loss": str(np.mean(g_loss_list))}
log_list.append(log_json)
with open(output_path + 'log.json', 'w') as log_file:
json.dump(log_list, log_file, indent=4)
if (epoch % 100 == 0):
gen.to_cpu()
dis.to_cpu()
save_npz(gen_model_path + str(epoch) + '.npz', gen)
save_npz(dis_model_path + str(epoch) + '.npz', dis)
gen.to_gpu(0)
dis.to_gpu(0)
gen.to_cpu()
dis.to_cpu()
save_npz(gen_model_path + 'last.npz', gen)
save_npz(dis_model_path + 'last.npz', dis)
def train(batch_size, epoch_count, lamda, datasetA_folder_path,
datasetB_folder_path, output_path):
dataset_A = data_io.dataset_load(datasetA_folder_path)
train_iter_A = SerialIterator(dataset_A,
batch_size,
repeat=True,
shuffle=True)
dataset_B = data_io.dataset_load(datasetB_folder_path)
train_iter_B = SerialIterator(dataset_B,
batch_size,
repeat=True,
shuffle=True)
g_ab = Generator()
g_ba = Generator()
d_a = Discriminator()
d_b = Discriminator()
g_ab.to_gpu(0)
g_ba.to_gpu(0)
d_a.to_gpu(0)
d_b.to_gpu(0)
opt_g_ab = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_g_ab.setup(g_ab)
opt_g_ba = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_g_ba.setup(g_ba)
opt_d_a = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_d_a.setup(d_a)
opt_d_b = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5)
opt_d_b.setup(d_b)
iteration = 0
train_iter_A.reset()
train_iter_B.reset()
log_list = []
image_path = output_path + "image/"
disA_model_path = output_path + "dis_A/"
disB_model_path = output_path + "dis_B/"
genAB_model_path = output_path + "gen_AB/"
genBA_model_path = output_path + "gen_BA/"
os.mkdir(output_path)
os.mkdir(image_path)
os.mkdir(disA_model_path)
os.mkdir(disB_model_path)
os.mkdir(genAB_model_path)
os.mkdir(genBA_model_path)
for epoch in range(epoch_count):
d_a_loss_list = []
d_b_loss_list = []
g_AB_loss_list = []
g_BA_loss_list = []
while True:
mini_batch_images_A = train_iter_A.next()
mini_batch_images_A = np.array(mini_batch_images_A)
mini_batch_images_A = (mini_batch_images_A - 128.0) / 128.0
real_a = Variable(np.array(mini_batch_images_A))
real_a.to_gpu(0)
mini_batch_images_B = train_iter_B.next()
mini_batch_images_B = np.array(mini_batch_images_B)
mini_batch_images_B = (mini_batch_images_B - 128.0) / 128.0
real_b = Variable(np.array(mini_batch_images_B))
real_b.to_gpu(0)
fake_b = g_ab(real_a)
fake_a = g_ba(real_b)
reconstr_a = g_ba(fake_b)
reconstr_b = g_ab(fake_a)
d_a_real_result = d_a(real_a)
d_a_fake_result = d_a(fake_a)
loss_d_a = loss_dis(batch_size, d_a_real_result, d_a_fake_result)
d_b_real_result = d_b(real_b)
d_b_fake_result = d_b(fake_b)
loss_d_b = loss_dis(batch_size, d_b_real_result, d_b_fake_result)
d_a.cleargrads()
loss_d_a.backward()
opt_d_a.update()
d_b.cleargrads()
loss_d_b.backward()
opt_d_b.update()
"""generatorのloss計算"""
loss_g_ab = loss_gen(batch_size, d_b_fake_result, real_a,
reconstr_a, lamda)
loss_g_ba = loss_gen(batch_size, d_a_fake_result, real_b,
reconstr_b, lamda)
g_ab.cleargrads()
loss_g_ab.backward()
opt_g_ab.update()
g_ba.cleargrads()
loss_g_ba.backward()
opt_g_ba.update()
loss_d_a.to_cpu()
loss_d_b.to_cpu()
loss_g_ab.to_cpu()
loss_g_ba.to_cpu()
iteration += batch_size
d_a_loss_list.append(loss_d_a.array)
d_b_loss_list.append(loss_d_b.array)
g_AB_loss_list.append(loss_g_ab.array)
g_BA_loss_list.append(loss_g_ba.array)
if train_iter_A.is_new_epoch or train_iter_B.is_new_epoch:
break
real_a.to_cpu()
fake_b.to_cpu()
reconstr_a.to_cpu()
real_b.to_cpu()
fake_a.to_cpu()
reconstr_b.to_cpu()
real_a_images = real_a.array.transpose(0, 2, 3, 1)
fake_b_images = fake_b.array.transpose(0, 2, 3, 1)
reconstr_a_images = reconstr_a.array.transpose(0, 2, 3, 1)
real_b_images = real_b.array.transpose(0, 2, 3, 1)
fake_a_images = fake_a.array.transpose(0, 2, 3, 1)
reconstr_b_images = reconstr_b.array.transpose(0, 2, 3, 1)
data_io.output_images(image_path + str(epoch), real_a_images,
fake_b_images, reconstr_a_images, real_b_images,
fake_a_images, reconstr_b_images)
print("epoch: " + str(epoch) + ", interation: " + str(iteration) + \
", d_A_loss: " + str(np.mean(d_a_loss_list)) + ", d_B_loss: " + str(np.mean(d_b_loss_list)) + \
", g_AB_loss: " + str(np.mean(g_AB_loss_list)) + ", g_BA_loss: " + str(np.mean(g_BA_loss_list)))
log_json = {"epoch": str(epoch), "interation": str(iteration), \
"d_A_loss": str(np.mean(d_a_loss_list)), "d_B_loss": str(np.mean(d_b_loss_list)), \
"g_AB_loss": str(np.mean(g_AB_loss_list)), "g_BA_loss": str(np.mean(g_BA_loss_list))}
log_list.append(log_json)
with open(output_path + 'log.json', 'w') as log_file:
json.dump(log_list, log_file, indent=4)
if (epoch % 100 == 0):
g_ab.to_cpu()
g_ba.to_cpu()
d_a.to_cpu()
d_b.to_cpu()
save_npz(genAB_model_path + str(epoch) + '.npz', g_ab)
save_npz(genBA_model_path + str(epoch) + '.npz', g_ba)
save_npz(disA_model_path + str(epoch) + '.npz', d_a)
save_npz(disB_model_path + str(epoch) + '.npz', d_b)
g_ab.to_gpu(0)
g_ba.to_gpu(0)
d_a.to_gpu(0)
d_b.to_gpu(0)
g_ab.to_cpu()
g_ba.to_cpu()
d_a.to_cpu()
d_b.to_cpu()
save_npz(genAB_model_path + 'last.npz', g_ab)
save_npz(genBA_model_path + 'last.npz', g_ba)
save_npz(disA_model_path + 'last.npz', d_a)
save_npz(disB_model_path + 'last.npz', d_b)
Dis.zerograds()
z = Gen.generate_hidden_variables(B)
x = Gen(Variable(xp.array(z)))
label_fake = Variable(xp.ones((B,1),dtype=xp.int32))
y, loss = Dis(x,label_fake)
loss_fake_gen += loss.data
loss.backward()
optG.update()
n_fake_gen += B
sys.stdout.write("\rtrain... epoch{}, {}/{}".format(epoch,n_real_dis+n_fake_dis+n_fake_gen,trainsize))
sys.stdout.flush()
z = Gen.generate_hidden_variables(batchsize)
x = Gen(Variable(xp.array(z))) #(B,3,64,64) B:batchsize
x.to_cpu()
tmp = np.transpose(x.data,(1,0,2,3)) #(3,B,64,64)
img_array=[]
for i in range(3):
img_array2=[]
for j in range(0,batchsize,8):
img=tmp[i][j:j+8]
img=np.transpose(img.reshape(64*8,64),(1,0))
img_array2.append(img)
img_array2=np.array(img_array2).reshape(int(batchsize/8*64),8*64)
img_array.append(np.transpose(img_array2,(1,0)))
img_array = np.array(img_array)
print("\nsave fig...")
save_img(img_array,save_path+"/{}.png".format(str(epoch).zfill(3)))
print("fake_gen_loss:{}(all/{}) fake_dis_loss:{}(all/{}), real_dis_loss:{}(all/{})".format(loss_fake_gen/float(n_fake_gen),n_fake_gen,loss_fake_dis/float(n_fake_dis),n_fake_dis,loss_real_dis/float(n_real_dis),n_real_dis)) #losses are approximated values
print('save model ...')
