def check_concat_dicts_padding(self, xp):
dicts = [
{'x': xp.random.rand(3, 4), 'y': xp.random.rand(2, 5)},
{'x': xp.random.rand(4, 4), 'y': xp.random.rand(3, 4)},
{'x': xp.random.rand(2, 5), 'y': xp.random.rand(2, 6)},
]
arrays = dataset.concat_examples(dicts, padding=0)
self.assertIn('x', arrays)
self.assertIn('y', arrays)
self.assertEqual(arrays['x'].shape, (3, 4, 5))
self.assertEqual(arrays['y'].shape, (3, 3, 6))
self.assertEqual(type(arrays['x']), type(dicts[0]['x']))
self.assertEqual(type(arrays['y']), type(dicts[0]['y']))
for d in dicts:
d['x'] = cuda.to_cpu(d['x'])
d['y'] = cuda.to_cpu(d['y'])
arrays = {'x': cuda.to_cpu(arrays['x']), 'y': cuda.to_cpu(arrays['y'])}
numpy.testing.assert_array_equal(arrays['x'][0, :3, :4], dicts[0]['x'])
numpy.testing.assert_array_equal(arrays['x'][0, 3:, :], 0)
numpy.testing.assert_array_equal(arrays['x'][0, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['x'][1, :4, :4], dicts[1]['x'])
numpy.testing.assert_array_equal(arrays['x'][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['x'][2, :2, :5], dicts[2]['x'])
numpy.testing.assert_array_equal(arrays['x'][2, 2:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][0, :2, :5], dicts[0]['y'])
numpy.testing.assert_array_equal(arrays['y'][0, 2:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][0, :, 5:], 0)
numpy.testing.assert_array_equal(arrays['y'][1, :3, :4], dicts[1]['y'])
numpy.testing.assert_array_equal(arrays['y'][1, 3:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['y'][2, :2, :6], dicts[2]['y'])
numpy.testing.assert_array_equal(arrays['y'][2, 2:, :], 0)
def check_concat_tuples_padding(self, xp):
tuples = [
(xp.random.rand(3, 4), xp.random.rand(2, 5)),
(xp.random.rand(4, 4), xp.random.rand(3, 4)),
(xp.random.rand(2, 5), xp.random.rand(2, 6)),
]
arrays = dataset.concat_examples(tuples, padding=0)
self.assertEqual(len(arrays), 2)
self.assertEqual(arrays[0].shape, (3, 4, 5))
self.assertEqual(arrays[1].shape, (3, 3, 6))
self.assertEqual(type(arrays[0]), type(tuples[0][0]))
self.assertEqual(type(arrays[1]), type(tuples[0][1]))
for i in range(len(tuples)):
tuples[i] = cuda.to_cpu(tuples[i][0]), cuda.to_cpu(tuples[i][1])
arrays = tuple(cuda.to_cpu(array) for array in arrays)
numpy.testing.assert_array_equal(arrays[0][0, :3, :4], tuples[0][0])
numpy.testing.assert_array_equal(arrays[0][0, 3:, :], 0)
numpy.testing.assert_array_equal(arrays[0][0, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[0][1, :4, :4], tuples[1][0])
numpy.testing.assert_array_equal(arrays[0][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[0][2, :2, :5], tuples[2][0])
numpy.testing.assert_array_equal(arrays[0][2, 2:, :], 0)
numpy.testing.assert_array_equal(arrays[1][0, :2, :5], tuples[0][1])
numpy.testing.assert_array_equal(arrays[1][0, 2:, :], 0)
numpy.testing.assert_array_equal(arrays[1][0, :, 5:], 0)
numpy.testing.assert_array_equal(arrays[1][1, :3, :4], tuples[1][1])
numpy.testing.assert_array_equal(arrays[1][1, 3:, :], 0)
numpy.testing.assert_array_equal(arrays[1][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[1][2, :2, :6], tuples[2][1])
numpy.testing.assert_array_equal(arrays[1][2, 2:, :], 0)
def check_concat_arrays(self, arrays, device=None):
array = dataset.concat_examples(arrays, device)
self.assertEqual(array.shape, (len(arrays),) + arrays[0].shape)
self.check_device(array, device)
for x, y in zip(array, arrays):
numpy.testing.assert_array_equal(
cuda.to_cpu(x), cuda.to_cpu(y))
def check_concat_arrays(self, arrays, device, expected_device):
array = dataset.concat_examples(arrays, device)
self.assertEqual(array.shape, (len(arrays),) + arrays[0].shape)
assert backend.get_device_from_array(array) == expected_device
np_array = backend.CpuDevice().send(array)
for x, y in zip(np_array, arrays):
numpy.testing.assert_array_equal(x, backend.CpuDevice().send(y))
def warmup(model, iterator, gpu_id=0):
batch = iterator.next()
img, img_info, bbox = concat_examples(batch, gpu_id)
img = chainer.Variable(img)
img_info = chainer.Variable(img_info)
bbox = chainer.Variable(bbox)
model.rcnn_train = True
model(img, img_info, bbox)
model.rpn_train = True
model(img, img_info, bbox)
def check_concat_arrays(
self, arrays, device, expected_device, expected_dtype):
array = dataset.concat_examples(arrays, device, self.padding)
self.assertEqual(array.shape, (len(arrays),))
self.check_device(array, device, expected_device)
np_array = backend.CpuDevice().send(array)
for x, y in zip(np_array, arrays):
assert x.dtype == expected_dtype
numpy.testing.assert_array_equal(
x, numpy.array(y, dtype=expected_dtype))
def check_concat_tuples(self, tuples, device=None):
arrays = dataset.concat_examples(tuples, device)
self.assertEqual(len(arrays), len(tuples[0]))
for i in range(len(arrays)):
shape = (len(tuples),) + tuples[0][i].shape
self.assertEqual(arrays[i].shape, shape)
self.check_device(arrays[i], device)
for x, y in zip(arrays[i], tuples):
numpy.testing.assert_array_equal(
cuda.to_cpu(x), cuda.to_cpu(y[i]))
def check_concat_dicts(self, dicts, device=None):
arrays = dataset.concat_examples(dicts, device)
self.assertEqual(frozenset(arrays.keys()), frozenset(dicts[0].keys()))
for key in arrays:
shape = (len(dicts),) + dicts[0][key].shape
self.assertEqual(arrays[key].shape, shape)
self.check_device(arrays[key], device)
for x, y in zip(arrays[key], dicts):
numpy.testing.assert_array_equal(
cuda.to_cpu(x), cuda.to_cpu(y[key]))
def check_concat_tuples(self, tuples, device, expected_device):
arrays = dataset.concat_examples(tuples, device)
self.assertEqual(len(arrays), len(tuples[0]))
for i in range(len(arrays)):
shape = (len(tuples),) + tuples[0][i].shape
self.assertEqual(arrays[i].shape, shape)
assert backend.get_device_from_array(arrays[i]) == expected_device
arr = backend.CpuDevice().send(arrays[i])
for x, y in zip(arr, tuples):
numpy.testing.assert_array_equal(
x, backend.CpuDevice().send(y[i]))
def check_concat_dicts(self, dicts, device, expected_device):
arrays = dataset.concat_examples(dicts, device)
self.assertEqual(frozenset(arrays.keys()), frozenset(dicts[0].keys()))
for key in arrays:
shape = (len(dicts),) + dicts[0][key].shape
self.assertEqual(arrays[key].shape, shape)
self.assertEqual(
backend.get_device_from_array(arrays[key]), expected_device)
arr = backend.CpuDevice().send(arrays[key])
for x, y in zip(arr, dicts):
numpy.testing.assert_array_equal(
x, backend.CpuDevice().send(y[key]))
def check_concat_arrays(self, arrays, device, expected_type):
array = dataset.concat_examples(arrays, device, self.padding)
self.assertEqual(array.shape, (len(arrays),))
self.check_device(array, device)
for x, y in zip(array, arrays):
if cuda.get_array_module(x) == numpy:
numpy.testing.assert_array_equal(
numpy.array(x),
numpy.array(y, dtype=expected_type))
else:
numpy.testing.assert_array_equal(
cuda.to_cpu(x),
numpy.array(y, dtype=expected_type))
def test(iterator, gpu, timesteps, encoder, decoder, rel_send, rel_rec, edge_types, temp, var):
nll_test = []
kl_test = []
edge_accuracies = []
node_mses = []
chainer.config.train = False
chainer.config.enable_backprop = False
while True:
inputs = iterator.next()
node_features, edge_labels = dataset.concat_examples(inputs, device=gpu)
data_encoder = node_features[:, :, :timesteps, :]
data_decoder = node_features[:, :, -timesteps:, :]
# logits: [batch_size, num_edges, edge_types]
logits = encoder(data_encoder, rel_send, rel_rec) # inverse func. of softmax
edges = F.gumbel_softmax(logits, tau=temp, axis=2)
edge_probs = F.softmax(logits, axis=2)
# edges, edge_probs: [batch_size, num_edges, edge_types]
# validation output uses teacher forcing
output = decoder(data_decoder, edges, rel_rec, rel_send, 1)
target = data_decoder[:, :, 1:, :]
num_nodes = node_features.shape[1]
loss_nll = get_nll_gaussian(output, target, var)
loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types)
nll_test.append(float(loss_nll.array))
kl_test.append(float(loss_kl.array))
edge_accuracy = get_edge_accuracy(logits.array, edge_labels)
edge_accuracies.append(edge_accuracy)
node_mse = float(F.mean_squared_error(output, target).array)
node_mses.append(node_mse)
if iterator.is_new_epoch:
break
put_log(iterator.epoch, np.mean(nll_test), np.mean(kl_test),
np.mean(edge_accuracies), np.mean(node_mses), 'test')
chainer.config.train = True
chainer.config.enable_backprop = True
def check_concat_arrays_padding(self, xp):
arrays = [xp.random.rand(3, 4),
xp.random.rand(2, 5),
xp.random.rand(4, 3)]
array = dataset.concat_examples(arrays, padding=0)
self.assertEqual(array.shape, (3, 4, 5))
self.assertEqual(type(array), type(arrays[0]))
arrays = [cuda.to_cpu(a) for a in arrays]
array = cuda.to_cpu(array)
numpy.testing.assert_array_equal(array[0, :3, :4], arrays[0])
numpy.testing.assert_array_equal(array[0, 3:, :], 0)
numpy.testing.assert_array_equal(array[0, :, 4:], 0)
numpy.testing.assert_array_equal(array[1, :2, :5], arrays[1])
numpy.testing.assert_array_equal(array[1, 2:, :], 0)
numpy.testing.assert_array_equal(array[2, :4, :3], arrays[2])
numpy.testing.assert_array_equal(array[2, :, 3:], 0)
def test_concat_arrays_padding(self, backend_config):
arrays = backend_config.get_array(
[numpy.random.rand(3, 4),
numpy.random.rand(2, 5),
numpy.random.rand(4, 3)])
array = dataset.concat_examples(arrays, padding=0)
self.assertEqual(array.shape, (3, 4, 5))
self.assertEqual(type(array), type(arrays[0]))
arrays = [backend.CpuDevice().send(a) for a in arrays]
array = backend.CpuDevice().send(array)
numpy.testing.assert_array_equal(array[0, :3, :4], arrays[0])
numpy.testing.assert_array_equal(array[0, 3:, :], 0)
numpy.testing.assert_array_equal(array[0, :, 4:], 0)
numpy.testing.assert_array_equal(array[1, :2, :5], arrays[1])
numpy.testing.assert_array_equal(array[1, 2:, :], 0)
numpy.testing.assert_array_equal(array[2, :4, :3], arrays[2])
numpy.testing.assert_array_equal(array[2, :, 3:], 0)
def test_concat_dicts_padding(self, backend_config):
dicts = [
{'x': numpy.random.rand(3, 4), 'y': numpy.random.rand(2, 5)},
{'x': numpy.random.rand(4, 4), 'y': numpy.random.rand(3, 4)},
{'x': numpy.random.rand(2, 5), 'y': numpy.random.rand(2, 6)},
]
dicts = [
{key: backend_config.get_array(arr) for key, arr in d.items()}
for d in dicts]
arrays = dataset.concat_examples(dicts, padding=0)
self.assertIn('x', arrays)
self.assertIn('y', arrays)
self.assertEqual(arrays['x'].shape, (3, 4, 5))
self.assertEqual(arrays['y'].shape, (3, 3, 6))
self.assertEqual(type(arrays['x']), type(dicts[0]['x']))
self.assertEqual(type(arrays['y']), type(dicts[0]['y']))
for d in dicts:
d['x'] = backend.CpuDevice().send(d['x'])
d['y'] = backend.CpuDevice().send(d['y'])
arrays = {
'x': backend.CpuDevice().send(arrays['x']),
'y': backend.CpuDevice().send(arrays['y'])}
numpy.testing.assert_array_equal(arrays['x'][0, :3, :4], dicts[0]['x'])
numpy.testing.assert_array_equal(arrays['x'][0, 3:, :], 0)
numpy.testing.assert_array_equal(arrays['x'][0, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['x'][1, :4, :4], dicts[1]['x'])
numpy.testing.assert_array_equal(arrays['x'][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['x'][2, :2, :5], dicts[2]['x'])
numpy.testing.assert_array_equal(arrays['x'][2, 2:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][0, :2, :5], dicts[0]['y'])
numpy.testing.assert_array_equal(arrays['y'][0, 2:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][0, :, 5:], 0)
numpy.testing.assert_array_equal(arrays['y'][1, :3, :4], dicts[1]['y'])
numpy.testing.assert_array_equal(arrays['y'][1, 3:, :], 0)
numpy.testing.assert_array_equal(arrays['y'][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays['y'][2, :2, :6], dicts[2]['y'])
numpy.testing.assert_array_equal(arrays['y'][2, 2:, :], 0)
def test_concat_tuples_padding(self, backend_config):
tuples = [
backend_config.get_array(
(numpy.random.rand(3, 4), numpy.random.rand(2, 5))),
backend_config.get_array(
(numpy.random.rand(4, 4), numpy.random.rand(3, 4))),
backend_config.get_array(
(numpy.random.rand(2, 5), numpy.random.rand(2, 6))),
]
arrays = dataset.concat_examples(tuples, padding=0)
self.assertEqual(len(arrays), 2)
self.assertEqual(arrays[0].shape, (3, 4, 5))
self.assertEqual(arrays[1].shape, (3, 3, 6))
self.assertEqual(type(arrays[0]), type(tuples[0][0]))
self.assertEqual(type(arrays[1]), type(tuples[0][1]))
for i in range(len(tuples)):
tuples[i] = (
backend.CpuDevice().send(tuples[i][0]),
backend.CpuDevice().send(tuples[i][1]))
arrays = tuple(backend.CpuDevice().send(array) for array in arrays)
numpy.testing.assert_array_equal(arrays[0][0, :3, :4], tuples[0][0])
numpy.testing.assert_array_equal(arrays[0][0, 3:, :], 0)
numpy.testing.assert_array_equal(arrays[0][0, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[0][1, :4, :4], tuples[1][0])
numpy.testing.assert_array_equal(arrays[0][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[0][2, :2, :5], tuples[2][0])
numpy.testing.assert_array_equal(arrays[0][2, 2:, :], 0)
numpy.testing.assert_array_equal(arrays[1][0, :2, :5], tuples[0][1])
numpy.testing.assert_array_equal(arrays[1][0, 2:, :], 0)
numpy.testing.assert_array_equal(arrays[1][0, :, 5:], 0)
numpy.testing.assert_array_equal(arrays[1][1, :3, :4], tuples[1][1])
numpy.testing.assert_array_equal(arrays[1][1, 3:, :], 0)
numpy.testing.assert_array_equal(arrays[1][1, :, 4:], 0)
numpy.testing.assert_array_equal(arrays[1][2, :2, :6], tuples[2][1])
numpy.testing.assert_array_equal(arrays[1][2, 2:, :], 0)
def accuracy(self):
image_test, target_test = concat_examples(self.iteration, -1)
prediction_test = self.model(image_test)
accuracy = F.accuracy(prediction_test, target_test)
return accuracy.data
def run(train_iter, val_iter, test_data, model, optimizer, max_epoch):
training_losses = []
validation_losses = []
while train_iter.epoch < max_epoch:
# Get next mini-batch
batch = train_iter.next()
image_train, target_train = concat_examples(batch)
# Prediction
prediction_train = model(image_train)
# Compute loss
loss = F.softmax_cross_entropy(prediction_train, target_train)
# Compute gradients
model.cleargrads()
loss.backward()
# Update variables
optimizer.update()
# Check the validation accuracy of prediction after every epoch
if train_iter.is_new_epoch: # If this iteration is the final iteration of the current epoch
# Display the training loss
print('epoch:{:02d} train_loss:{:.04f} '.format(
train_iter.epoch, float(loss.data)))
training_losses.append(float(loss.data))
val_losses = []
val_accuracies = []
while True:
val_batch = val_iter.next()
image_val, target_val = concat_examples(val_batch)
# Forward the validation data
prediction_val = model(image_val)
# Calculate the loss
loss_val = F.softmax_cross_entropy(prediction_val, target_val)
val_losses.append(loss_val.data)
# Calculate the accuracy
accuracy = F.accuracy(prediction_val, target_val)
val_accuracies.append(accuracy.data)
if val_iter.is_new_epoch:
val_iter.epoch = 0
val_iter.current_position = 0
val_iter.is_new_epoch = False
val_iter._pushed_position = None
validation_losses.append(np.mean(val_losses))
break
print('val_loss:{:.04f} val_accuracy:{:.04f}'.format(
np.mean(val_losses), np.mean(val_accuracies)))
# Predict full test set
image_test, target_test = concat_examples(test_data)
# Forward test data
prediction_test = model(image_test)
# Calculate loss and accuracy
loss_test = F.softmax_cross_entropy(prediction_test, target_test)
accuracy_test = F.accuracy(prediction_test, target_test)
print('test_loss: ' + str(loss_test.data) + ' test_accuracy: ' +
str(accuracy_test.data))
return training_losses, validation_losses
return self.l3(h2)
net = MLP()
from chainer import optimizers
optimizer = optimizers.SGD(lr=0.01)
optimizer.setup(net)
from chainer.dataset import concat_examples
max_epoch = 10
while train_iter.epoch < max_epoch:
train_batch = train_iter.next()
x, t = concat_examples(train_batch)
y = net(x)
loss = F.softmax_cross_entropy(y, t)
net.cleargrads()
loss.backward()
optimizer.update()
if train_iter.is_new_epoch:
print('epoch:{:02d} train_loss:{:.04f} '.format(
train_iter.epoch, float(loss.data)),
end='')
test_losses = []
test_accuracies = []
while True:
test_batch = test_iter.next()
x_test, t_test = concat_examples(test_batch)
def converter(batch, device):
x, t = concat_examples(batch, device, 0)
return x, t
train_iter = chainer.iterators.SerialIterator(train, batchsize)
test_iter = chainer.iterators.SerialIterator(test,
batchsize,
repeat=False,
shuffle=False)
# Updater <- LSTM用にカスタマイズ
#updater = training.StandardUpdater(train_iter, optimizer)
# Trainerとそのextensions
epoch = 3000
while train_iter.epoch < epoch:
train_batch = train_iter.next()
import pdb
pdb.set_trace()
x, t = concat_examples(train_batch)
train = model(x, t)
model, cleargrads()
loss.backwrd()
optimizer.update()
#trainer = training.Trainer(updater, (epoch, 'epoch'), out='result')
# 評価データで評価
#trainer.extend(extensions.Evaluator(test_iter, model,device = -1))
# 学習結果の途中を表示する
#trainer.extend(extensions.LogReport(trigger=(1, 'epoch')))
# １エポックごとに、trainデータに対するlossと、testデータに対するlossを出力させる
#trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time']), trigger=(1, 'epoch'))
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='end_to_end_result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--need_validate',
action='store_true',
help='do or not validate during training')
parser.add_argument('--mean',
default=config.ROOT_PATH +
"BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="mobilenet_v1",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='SGD',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model_rgb',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument(
'--pretrained_model_of',
help=
"path of optical flow pretrained model (may be single stream OF model)"
)
parser.add_argument('--pretrained_model_args',
nargs='+',
type=float,
help='you can pass in "1.0 224" or "0.75 224"')
parser.add_argument('--spatial_edge_mode',
type=SpatialEdgeMode,
choices=list(SpatialEdgeMode),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument('--spatial_sequence_type',
type=SpatialSequenceType,
choices=list(SpatialSequenceType),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument(
'--temporal_edge_mode',
type=TemporalEdgeMode,
choices=list(TemporalEdgeMode),
help='1:rnn, 2:attention_block, 3.point-wise feed forward(no temporal)'
)
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='spatial/ temporal/ spatial_temporal')
parser.add_argument('--conv_rnn_type',
type=ConvRNNType,
choices=list(ConvRNNType),
help='conv_lstm or conv_sru')
parser.add_argument("--bi_lstm",
action="store_true",
help="whether to use bi-lstm as Edge/Node RNN")
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--layers", type=int, default=1)
parser.add_argument("--label_win_size", type=int, default=3)
parser.add_argument("--fix",
action="store_true",
help="fix parameter of conv2 update when finetune")
parser.add_argument("--x_win_size", type=int, default=1)
parser.add_argument("--use_label_dependency",
action="store_true",
help="use label dependency layer after conv_lstm")
parser.add_argument("--dynamic_backbone",
action="store_true",
help="use dynamic backbone: conv lstm as backbone")
parser.add_argument("--ld_rnn_dropout", type=float, default=0.4)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument(
"--roi_align",
action="store_true",
help="whether to use roi align or roi pooling layer in CNN")
parser.add_argument("--debug",
action="store_true",
help="debug mode for 1/50 dataset")
parser.add_argument("--sample_frame", '-sample', type=int, default=10)
parser.add_argument(
"--snap_individual",
action="store_true",
help="whether to snapshot each individual epoch/iteration")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument("--fetch_mode", type=int, default=1)
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
# with open(pid_file_path, "w") as file_obj:
# file_obj.write(pid)
# file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
paper_report_label, class_num = squeeze_label_num_report(
args.database, args.use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
use_feature_map_res45 = (args.conv_rnn_type != ConvRNNType.conv_rcnn) and (
args.conv_rnn_type != ConvRNNType.fc_lstm)
use_au_rcnn_loss = (args.conv_rnn_type == ConvRNNType.conv_rcnn)
au_rcnn_train_chain_list = []
if args.backbone == 'vgg':
au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
elif args.backbone == 'resnet101':
if args.two_stream_mode != TwoStreamMode.spatial_temporal:
pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_of
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=(
args.two_stream_mode == TwoStreamMode.optical_flow),
temporal_length=args.sample_frame)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
else:
au_rcnn_rgb = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=False,
temporal_length=args.sample_frame)
au_rcnn_optical_flow = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_of,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=True,
temporal_length=args.sample_frame)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
elif args.backbone == "mobilenet_v1":
au_rcnn = AU_RCNN_MobilenetV1(
pretrained_model_type=args.pretrained_model_args,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
if use_au_rcnn_loss:
au_rcnn_train_loss = AU_RCNN_TrainChainLoss()
loss_head_module = au_rcnn_train_loss
elif args.conv_rnn_type == ConvRNNType.conv_lstm:
label_dependency_layer = None
if args.use_label_dependency:
label_dependency_layer = LabelDependencyRNNLayer(
args.database,
in_size=2048,
class_num=class_num,
train_mode=True,
label_win_size=args.label_win_size)
space_time_conv_lstm = SpaceTimeConv(
label_dependency_layer,
args.use_label_dependency,
class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode,
conv_rnn_type=args.conv_rnn_type)
loss_head_module = space_time_conv_lstm
elif args.conv_rnn_type == ConvRNNType.sep_conv_lstm:
space_time_sep_conv_lstm = SpaceTimeSepConv(
database=args.database,
class_num=class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode)
loss_head_module = space_time_sep_conv_lstm
elif args.conv_rnn_type == ConvRNNType.fc_lstm:
space_time_fc_lstm = SpaceTimeSepFcLSTM(
database=args.database,
class_num=class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode)
loss_head_module = space_time_fc_lstm
model = Wrapper(au_rcnn_train_chain_list,
loss_head_module,
args.database,
args.sample_frame,
use_feature_map=use_feature_map_res45,
two_stream_mode=args.two_stream_mode)
batch_size = args.batch_size
img_dataset = AUDataset(database=args.database,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False)
train_video_data = AU_video_dataset(
au_image_dataset=img_dataset,
sample_frame=args.sample_frame,
train_mode=(args.two_stream_mode != TwoStreamMode.optical_flow),
paper_report_label_idx=paper_report_label_idx,
)
Transform = Transform3D
train_video_data = TransformDataset(train_video_data,
Transform(au_rcnn, mirror=False))
if args.proc_num == 1:
train_iter = SerialIterator(train_video_data,
batch_size * args.sample_frame,
repeat=True,
shuffle=False)
else:
train_iter = MultiprocessIterator(train_video_data,
batch_size=batch_size *
args.sample_frame,
n_processes=args.proc_num,
repeat=True,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
if len(args.gpu) > 1:
for gpu in args.gpu:
chainer.cuda.get_device_from_id(gpu).use()
else:
chainer.cuda.get_device_from_id(args.gpu[0]).use()
model.to_gpu(args.gpu[0])
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
key_str = "{0}_fold_{1}".format(args.fold, args.split_idx)
file_list = []
file_list.extend(os.listdir(args.out))
snapshot_model_file_name = args.out + os.sep + filter_last_checkpoint_filename(
file_list, "model", key_str)
# BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz
use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label"
roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling"
label_dependency_layer_key_str = "label_dep_layer" if args.use_label_dependency else "no_label_dep"
single_model_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_model.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.spatial_edge_mode,
args.temporal_edge_mode,
use_paper_key_str, roi_align_key_str,
label_dependency_layer_key_str,
args.conv_rnn_type,args.sample_frame )#, args.label_win_size)
print(single_model_file_name)
pretrained_optimizer_file_name = args.out + os.sep +\
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_optimizer.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.spatial_edge_mode,
args.temporal_edge_mode,
use_paper_key_str, roi_align_key_str,
label_dependency_layer_key_str,
args.conv_rnn_type, args.sample_frame)# args.label_win_size)
print(pretrained_optimizer_file_name)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if args.snap_individual:
if os.path.exists(snapshot_model_file_name) and os.path.isfile(
snapshot_model_file_name):
print("loading pretrained snapshot:{}".format(
snapshot_model_file_name))
chainer.serializers.load_npz(snapshot_model_file_name, model)
else:
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(
single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model)
if args.fix:
au_rcnn = model.au_rcnn_train_chain.au_rcnn
au_rcnn.extractor.conv1.W.update_rule.enabled = False
au_rcnn.extractor.bn1.gamma.update_rule.enabled = False
au_rcnn.extractor.bn1.beta.update_rule.enabled = False
res2_names = ["a", "b1", "b2"]
for res2_name in res2_names:
if res2_name == "a":
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv4.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.beta.update_rule.enabled = False
elif res2_name.startswith("b"):
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
# if (args.spatial_edge_mode in [SpatialEdgeMode.ld_rnn, SpatialEdgeMode.bi_ld_rnn] or args.temporal_edge_mode in \
# [TemporalEdgeMode.ld_rnn, TemporalEdgeMode.bi_ld_rnn]) or (args.conv_rnn_type != ConvRNNType.conv_rcnn):
# updater = BPTTUpdater(train_iter, optimizer, converter=lambda batch, device: concat_examples(batch, device,
# padding=0), device=args.gpu[0])
if len(args.gpu) > 1:
gpu_dict = {"main": args.gpu[0]} # many gpu will use
parallel_models = {"parallel": model.au_rcnn_train_chain}
for slave_gpu in args.gpu[1:]:
gpu_dict[slave_gpu] = int(slave_gpu)
updater = PartialParallelUpdater(
train_iter,
optimizer,
args.database,
models=parallel_models,
devices=gpu_dict,
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
else:
print("only one GPU({0}) updater".format(args.gpu[0]))
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=args.gpu[0],
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
@training.make_extension(trigger=(1, "epoch"))
def reset_order(trainer):
print("reset dataset order after one epoch")
if args.debug:
trainer.updater._iterators[
"main"].dataset._dataset.reset_for_debug_mode()
else:
trainer.updater._iterators[
"main"].dataset._dataset.reset_for_train_mode()
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(reset_order)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
if not args.snap_individual:
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=os.path.basename(single_model_file_name)),
trigger=(args.snapshot, 'iteration'))
else:
snap_model_file_name = '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}sampleframe#{10}@win#{11}_'.format(
args.database, args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode, use_paper_key_str,
roi_align_key_str, label_dependency_layer_key_str,
args.conv_rnn_type, args.sample_frame, args.label_win_size)
snap_model_file_name = snap_model_file_name + "{.updater.iteration}.npz"
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=snap_model_file_name),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 10, 'iteration'
plot_interval = 10, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.1, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="log_{0}_fold_{1}_{2}@{3}@{4}@{5}.log".format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type)))
# trainer.reporter.add_observer("main_par", model.loss_head_module)
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name='loss_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name='accuracy_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def load_mnist():
train, test = get_mnist(ndim=3)
train = concat_examples(train)
test = concat_examples(test)
return train, test
def run(self):
loss_names = ['epoch']
loss_names.append('imitation_loss')
if self.learn_dx:
loss_names.append('sysid_loss')
fname = os.path.join(self.save, 'train_losses.csv')
train_loss_f = open(fname, 'w')
train_loss_f.write('{}\n'.format(','.join(loss_names)))
train_loss_f.flush()
fname = os.path.join(self.save, 'val_test_losses.csv')
vt_loss_f = open(fname, 'w')
loss_names = ['epoch']
loss_names += ['im_loss_val', 'im_loss_test']
vt_loss_f.write('{}\n'.format(','.join(loss_names)))
vt_loss_f.flush()
if self.learn_cost:
fname = os.path.join(self.save, 'cost_hist.csv')
cost_f = open(fname, 'w')
# first write answer
cost_f.write(','.join(
map(str,
xp.concatenate((self.true_q, self.true_p)).tolist())))
cost_f.write('\n')
cost_f.flush()
opt = chainer.optimizers.RMSprop(lr=1e-2, alpha=0.5)
opt.setup(self.net)
train_warm_start = xp.zeros((self.n_train, self.T, self.n_ctrl))
val_warm_start = xp.zeros(
(self.env.val_data.shape[0], self.T, self.n_ctrl))
test_warm_start = xp.zeros(
(self.env.test_data.shape[0], self.T, self.n_ctrl))
train_data, train_iter = self.make_data(
self.env.train_data[:self.n_train], shuffle=True)
val_data, val_iter = self.make_data(self.env.val_data)
test_data, test_iter = self.make_data(self.env.test_data)
best_val_loss = None
train_warmstart = xp.zeros((self.n_train, self.T, self.n_ctrl))
val_warmstart = xp.zeros(
(self.env.val_data.shape[0], self.T, self.n_ctrl))
test_warmstart = xp.zeros(
(self.env.test_data.shape[0], self.T, self.n_ctrl))
true_q, true_p = self.env.true_dx.get_true_obj()
cost_update_q = False
learn_cost_round_robin_interval = 10
while train_iter.epoch < self.n_epoch:
print("triain_iter.epoch", train_iter.epoch_detail)
if train_iter.epoch > 0 and train_iter.epoch % learn_cost_round_robin_interval == 0:
cost_update_q = not cost_update_q
self.net.cleargrads()
if train_iter.epoch % self.restart_warmstart_every == 0:
train_warm_start = xp.zeros_like(train_warm_start)
val_warm_start = xp.zeros_like(val_warm_start)
test_warm_start = xp.zeros_like(test_warm_start)
next_batch = train_iter.next()
next_batch = dataset.concat_examples(next_batch)
xinits = next_batch[0]
xs = next_batch[1]
us = next_batch[2]
idxs = next_batch[3]
if self.learn_dx:
assert False
else:
dx = self.env.true_dx
nom_x, nom_u = self.net(xinits, self.env, train_warm_start[idxs])
nom_u = F.transpose(nom_u, axes=(1, 0, 2))
'''
g = c.build_computational_graph([nom_u], remove_variable=True)
with open('graph.dot', 'w') as o:
o.write(g.dump())
assert False
'''
train_warmstart[idxs] = nom_u.array
assert type(us) != chainer.Variable
assert nom_u.shape == us.shape
squared_loss_u = (us - nom_u) * (us - nom_u)
im_loss_u = F.mean(squared_loss_u)
# print("im_loss shape", im_loss_u.shape)
nom_x = F.transpose(nom_x, axes=(1, 0, 2))
assert xs.shape == nom_x.shape, str(xs.shape) + " " + str(
nom_x.shape)
# squared_loss_x = (xs - nom_x) * (xs - nom_x)
# im_loss_x = F.mean(squared_loss_x)
# print("im_loss-x", im_loss_x.shape)
# im_loss = im_loss_x + im_loss_u
im_loss = im_loss_u
t = [train_iter.epoch_detail, im_loss.array]
t = ','.join(map(str, t))
train_loss_f.write(t + '\n')
train_loss_f.flush()
print("train imtation loss", im_loss.array)
im_loss.backward()
if cost_update_q:
print('only updating q')
self.net.learn_p.update_rule.enabled = False
self.net.learn_q_logit.update_rule.enabled = True
if self.is_lower_triangle:
self.net.lower_without_diag.update_rule.enabled = True
else:
print('only updating p')
self.net.learn_q_logit.update_rule.enabled = False
self.net.learn_p.update_rule.enabled = True
if self.is_lower_triangle:
self.net.lower_without_diag.update_rule.enabled = False
'''
if self.learn_cost:
true_cat = F.concat((true_q, true_p), axis=0)
_q = F.sigmoid(self.net.learn_q_logit).array
_p = xp.sqrt(_q) * self.net.learn_p
print("learn_q_logit", self.net.learn_q_logit)
print("learn_p", self.net.learn_p)
qp_cat = F.concat((_q, _p), axis=0)
print(xp.array_str(F.stack((true_cat, qp_cat)).array, precision=5, suppress_small=True))
cost_f.write(','.join(map(str, qp_cat.array)))
cost_f.write('\n')
cost_f.flush()
'''
opt.update()
if train_iter.is_new_epoch:
val_loss = self.dataset_loss(val_iter, val_warmstart)
test_loss = self.dataset_loss(test_iter, test_warmstart)
t = [train_iter.epoch, val_loss, test_loss]
t = ','.join(map(str, t))
vt_loss_f.write(t + '\n')
vt_loss_f.flush()
if best_val_loss is None or val_loss < best_val_loss:
best_val_loss = val_loss
fname = os.path.join(self.save, 'best.pkl')
print('Saving best model to {}'.format(fname))
with open(fname, 'wb') as f:
pkl.dump(self, f)
def converter(batch, device):
x, t = concat_examples(batch, device, 0)
return x, t
class MLP(Chain):
def __init__(self, n_units, n_out):
super(MLP, self).__init__()
with self.init_scope():
self.l1 = L.Linear(None, n_units)
self.l2 = L.Linear(None, n_units)
self.l3 = L.Linear(None, n_out)
def __call__(self, x):
h1 = F.relu(self.l1(x))
h2 = F.relu(self.l2(h1))
y = self.l3(h2)
return y
model = L.Classifier(MLP(100, 10))
optimizer = optimizers.SGD()
optimizer.setup(model)
model.to_gpu() # here!
batchsize = 100
x_train, y_train = concat_examples(train)
datasize = len(x_train)
for epoch in range(20):
print('epoch: %d' % epoch)
indexes = np.random.permutation(datasize)
for i in range(0, datasize, batchsize):
x = Variable(cuda.to_gpu(x_train[indexes[i:i + batchsize]])) # here!
t = Variable(cuda.to_gpu(y_train[indexes[i:i + batchsize]])) # here!
optimizer.update(model, x, t)
def dataset_loss(self, data_iter, warmstart=None):
""" calculate loss and set warm start
this loss is
:param loader:
:param warmstart:
:return:
"""
true_q, true_p = self.env.true_dx.get_true_obj()
losses = []
iter_before = data_iter.epoch
while data_iter.epoch < iter_before + 1:
next_batch = data_iter.next()
next_batch = dataset.concat_examples(next_batch)
x_inits = next_batch[0]
xs = next_batch[1]
us = next_batch[2]
idxs = next_batch[3]
n_batch = x_inits.shape[0]
dx = self.env.true_dx
if not self.is_lower_triangle and not self.is_strange_observation:
print("not lower triangle and not strange observation")
assert type(self.net) == Pendulum_Net_cost_logit
q = F.sigmoid(self.net.learn_q_logit)
p = F.sqrt(q) * self.net.learn_p
_, pred_u = self.env.mpc(self.env.true_dx,
x_inits,
q,
p,
u_init=xp.transpose(warmstart[idxs],
axes=(1, 0, 2)))
pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array
warmstart[idxs] = pred_u
elif not self.is_strange_observation:
print("lower triangle and not strange observation")
assert type(self.net) == Pendulum_Net_cost_lower_triangle
Q = xp.zeros((self.n_sc, self.n_sc))
index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool)
xp.fill_diagonal(index_diag, True)
index_not_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool)
index_not_diag[xp.tril_indices(self.n_sc, -1)] = True
Q = F.scatter_add(Q, xp.tril_indices(self.n_sc, -1),
self.net.lower_without_diag)
# index_not_diag = xp.zeros((self.n_state, self.n_state), dtype=bool)
# index_not_diag[xp.tril_indices(self.n_state, -1)] = True
# Q = F.scatter_add(Q, xp.tril_indices(self.n_state, -1), self.net.lower_without_diag)
diag_q = F.sigmoid(self.net.learn_q_logit)
Q = F.where(index_diag, diag_q, Q)
Q = Q @ Q.T
p = self.net.learn_p
p = F.concat((p, xp.array(0.0).reshape(1, )), axis=0)
_, pred_u = self.env.mpc_Q(self.env.true_dx,
x_inits,
Q,
p,
u_init=xp.transpose(warmstart[idxs],
axes=(1, 0, 2)))
pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array
warmstart[idxs] = pred_u
elif not self.is_lower_triangle:
print("not lower triangle and strange observation")
assert type(
self.net) == Pendulum_Net_cost_logit_strange_obervation
q = F.sigmoid(self.net.learn_q_logit)
p = F.sqrt(q) * self.net.learn_p
# p = self.learn_p
Q = xp.zeros((self.n_sc, self.n_sc))
index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool)
xp.fill_diagonal(index_diag, True)
Q = F.where(index_diag, q, Q)
Q = OBSERVATION_MATRIX.T @ Q @ OBSERVATION_MATRIX
p = p @ OBSERVATION_MATRIX
_, pred_u = self.env.mpc_Q(self.env.true_dx,
x_inits,
Q,
p,
u_init=xp.transpose(warmstart[idxs],
axes=(1, 0, 2)))
pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array
warmstart[idxs] = pred_u
else:
print("lower triangle and strange observation")
assert type(
self.net
) == Pendulum_Net_cost_lower_triangle_strange_obervation
Q = xp.zeros((self.n_sc, self.n_sc))
index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool)
xp.fill_diagonal(index_diag, True)
index_not_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool)
index_not_diag[xp.tril_indices(self.n_sc, -1)] = True
Q = F.scatter_add(Q, xp.tril_indices(self.n_sc, -1),
self.net.lower_without_diag)
diag_q = F.sigmoid(self.net.learn_q_logit)
Q = F.where(index_diag, diag_q, Q)
Q = Q @ Q.T
Q = OBSERVATION_MATRIX.T @ Q @ OBSERVATION_MATRIX
p = self.net.learn_p
p = p @ OBSERVATION_MATRIX
_, pred_u = self.env.mpc_Q(self.env.true_dx,
x_inits,
Q,
p,
u_init=xp.transpose(warmstart[idxs],
axes=(1, 0, 2)))
pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array
warmstart[idxs] = pred_u
assert pred_u.shape == us.shape
squared_loss = (us - pred_u) * (us - pred_u)
# print(squared_loss.shape)
loss = xp.mean(squared_loss)
losses.append(loss)
loss = xp.stack(losses).mean()
return loss
net = myNetwork.UNET_02()
if gpu_id >= 0:
net.to_gpu(gpu_id)
optimizer = optimizers.Adam().setup(net)
#--------------------
# Learning iteration
#--------------------
os.makedirs(savefile + "/models/")
while train_iter.epoch < max_epoch:
train_batch = train_iter.next()
x, t = concat_examples(train_batch, gpu_id)
y = net(x)
loss = F.mean_squared_error(y, t)
#loss = myLossfun.l1l2_norm_error(y, t, lam=3)
net.cleargrads()
loss.backward()
optimizer.update()
# Check
if train_iter.is_new_epoch:
print('epoch:{:02d} train_loss:{:.04f} '.format(
train_iter.epoch, float(to_cpu(loss.data))),
fig,ax = plt.subplots(nrows=6,ncols=8,sharex=True,sharey=True)
ax = ax.flatten()
for i in range(48):
img = xtrain[i].reshape(28,28)
ax[i].imshow(img,cmap='Greys',interpolation='none')
'''
"""<h3>ニューラルネットワークの学習を行います。GPUを用いて高速に学習ができます。</h3>"""
max_epoch = args_epoch
while train_iter.epoch < max_epoch:
train_accuracies = []
# ---------- One iteration of the training loop ----------
train_batch = train_iter.next()
image_train, target_train = concat_examples(train_batch, args_gpu)
# Calculate the prediction of the network
prediction_train = model(image_train)
# Calculate the loss with softmax_cross_entropy
loss = F.softmax_cross_entropy(prediction_train, target_train)
accuracy = F.accuracy(prediction_train, target_train)
accuracy.to_cpu()
train_accuracies.append(accuracy.data)
# Calculate the gradients in the network
model.cleargrads()
loss.backward()
# Update all the trainable parameters
optimizer.update()
def train(iterator, gpu, encoder, decoder, enc_optim, dec_optim, rel_send,
rel_rec, edge_types, temp, prediction_steps, var, out, benchmark,
lr_decay, gamma):
iter_i = 0
edge_accuracies = []
node_mses = []
nll_train = []
kl_train = []
logger = logging.getLogger(__name__)
while True:
inputs = iterator.next()
node_features, edge_labels = dataset.concat_examples(inputs,
device=gpu)
# logits: [batch_size, num_edges, edge_types]
logits = encoder(node_features, rel_send,
rel_rec) # inverse func. of softmax
edges = F.gumbel_softmax(logits, tau=temp, axis=2)
edge_probs = F.softmax(logits, axis=2)
# edges, edge_probs: [batch_size, num_edges, edge_types]
if isinstance(decoder, decoders.MLPDecoder):
output = decoder(node_features, edges, rel_rec, rel_send,
prediction_steps)
elif isinstance(decoder, decoders.RNNDecoder):
output = decoder(node_features,
edges,
rel_rec,
rel_send,
100,
burn_in=True,
burn_in_steps=args.timesteps -
args.prediction_steps)
target = node_features[:, :, 1:, :]
num_nodes = node_features.shape[1]
loss_nll = get_nll_gaussian(output, target, var)
loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types)
loss = loss_nll + loss_kl
nll_train.append(float(loss_nll.array))
kl_train.append(float(loss_kl.array))
edge_accuracy = get_edge_accuracy(logits.array, edge_labels)
edge_accuracies.append(edge_accuracy)
node_mse = float(F.mean_squared_error(output, target).array)
node_mses.append(node_mse)
encoder.cleargrads()
decoder.cleargrads()
loss.backward()
enc_optim.update()
dec_optim.update()
# Exit after 10 iterations when benchmark mode is ON
iter_i += 1
if benchmark:
put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train),
np.mean(edge_accuracies), np.mean(node_mses))
if iter_i == 10:
exit()
if iterator.is_new_epoch:
break
if not os.path.exists(os.path.join(out, 'graph.dot')):
with open(os.path.join(out, 'graph.dot'), 'w') as o:
g = computational_graph.build_computational_graph([loss])
o.write(g.dump())
if iterator.is_new_epoch:
put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train),
np.mean(edge_accuracies), np.mean(node_mses))
serializers.save_npz(
os.path.join(out, 'encoder_epoch-{}.npz'.format(iterator.epoch)),
encoder)
serializers.save_npz(
os.path.join(out, 'decoder_epoch-{}.npz'.format(iterator.epoch)),
decoder)
serializers.save_npz(
os.path.join(out, 'enc_optim_epoch-{}.npz'.format(iterator.epoch)),
enc_optim)
serializers.save_npz(
os.path.join(out, 'dec_optim_epoch-{}.npz'.format(iterator.epoch)),
dec_optim)
if iterator.epoch % lr_decay == 0:
enc_optim.alpha *= gamma
dec_optim.alpha *= gamma
logger.info('alpha of enc_optim: {}'.format(enc_optim.alpha))
logger.info('alpha of dec_optim: {}'.format(dec_optim.alpha))
def main():
parser = argparse.ArgumentParser(description='Chainer example: MNIST')
parser.add_argument('--batchsize',
'-b',
type=int,
default=100,
help='Number of images in each mini-batch')
parser.add_argument('--epochs',
'-e',
type=int,
default=20,
help='Number of sweeps over the dataset to train')
parser.add_argument('--output_dir',
'-o',
default='./outputs',
help='Directory to output the result')
parser.add_argument(
'--gpu_id',
'-g',
default=0,
help='ID of the GPU to be used. Set to -1 if you use CPU')
args = parser.parse_args()
# Download the MNIST data if you haven't downloaded it yet
train, test = datasets.mnist.get_mnist(withlabel=True, ndim=1)
gpu_id = args.gpu_id
batchsize = args.batchsize
epochs = args.epochs
run.log('Batch size', np.int(batchsize))
run.log('Epochs', np.int(epochs))
train_iter = iterators.SerialIterator(train, batchsize)
test_iter = iterators.SerialIterator(test,
batchsize,
repeat=False,
shuffle=False)
model = MyNetwork()
if gpu_id >= 0:
# Make a specified GPU current
chainer.backends.cuda.get_device_from_id(0).use()
model.to_gpu() # Copy the model to the GPU
# Choose an optimizer algorithm
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
# Give the optimizer a reference to the model so that it
# can locate the model's parameters.
optimizer.setup(model)
while train_iter.epoch < epochs:
# ---------- One iteration of the training loop ----------
train_batch = train_iter.next()
image_train, target_train = concat_examples(train_batch, gpu_id)
# Calculate the prediction of the network
prediction_train = model(image_train)
# Calculate the loss with softmax_cross_entropy
loss = F.softmax_cross_entropy(prediction_train, target_train)
# Calculate the gradients in the network
model.cleargrads()
loss.backward()
# Update all the trainable parameters
optimizer.update()
# --------------------- until here ---------------------
# Check the validation accuracy of prediction after every epoch
if train_iter.is_new_epoch: # If this iteration is the final iteration of the current epoch
# Display the training loss
print('epoch:{:02d} train_loss:{:.04f} '.format(
train_iter.epoch, float(to_cpu(loss.array))),
end='')
test_losses = []
test_accuracies = []
while True:
test_batch = test_iter.next()
image_test, target_test = concat_examples(test_batch, gpu_id)
# Forward the test data
prediction_test = model(image_test)
# Calculate the loss
loss_test = F.softmax_cross_entropy(prediction_test,
target_test)
test_losses.append(to_cpu(loss_test.array))
# Calculate the accuracy
accuracy = F.accuracy(prediction_test, target_test)
accuracy.to_cpu()
test_accuracies.append(accuracy.array)
if test_iter.is_new_epoch:
test_iter.epoch = 0
test_iter.current_position = 0
test_iter.is_new_epoch = False
test_iter._pushed_position = None
break
val_accuracy = np.mean(test_accuracies)
print('val_loss:{:.04f} val_accuracy:{:.04f}'.format(
np.mean(test_losses), val_accuracy))
run.log("Accuracy", np.float(val_accuracy))
def main():
print("chainer cudnn enabled: {}".format(chainer.cuda.cudnn_enabled))
parser = argparse.ArgumentParser(
description='Action Unit Intensity R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
default="0",
help='GPU ID, multiple GPU split by comma')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--seed', '-s', type=int, default=0)
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=20)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--feature_model',
default="resnet101",
help="vgg or resnet101 for train")
parser.add_argument('--extract_len', type=int, default=1000)
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='resnet101',
help='imagenet/vggface/resnet101/*.npz')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument(
"--is_pretrained",
action="store_true",
help="whether is to pretrain BP4D later will for DISFA dataset or not")
parser.add_argument(
"--pretrained_target",
'-pt',
default="",
help="whether pretrain label set will use DISFA or not")
parser.add_argument("--prefix",
'-prefix',
default="",
help="_beta, for example 3_fold_beta")
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
parser.add_argument("--img_resolution", type=int, default=512)
args = parser.parse_args()
if not os.path.exists(args.pid):
os.makedirs(args.pid)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
config.IMG_SIZE = (args.img_resolution, args.img_resolution)
print('GPU: {}'.format(args.gpu))
if args.is_pretrained:
adaptive_AU_database(args.pretrained_target)
else:
adaptive_AU_database(args.database)
np.random.seed(args.seed)
# 需要先构造一个list的txt文件:id_trainval_0.txt, 每一行是subject + "/" + emotion_seq + "/" frame
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
if args.feature_model == 'vgg':
faster_rcnn = FasterRCNNVGG16(
n_fg_class=len(config.AU_INTENSITY_DICT),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len
) # 可改为/home/nco/face_expr/result/snapshot_model.npz
elif args.feature_model == 'resnet101':
faster_rcnn = FasterRCNNResnet101(
n_fg_class=len(config.AU_INTENSITY_DICT),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len
) # 可改为/home/nco/face_expr/result/snapshot_model.npz
if args.eval_mode:
with chainer.no_backprop_mode(), chainer.using_config("train", False):
test_data = AUDataset(database=args.database,
fold=args.fold,
img_resolution=args.img_resolution,
split_name='test',
split_index=args.split_idx,
mc_manager=mc_manager,
prefix=args.prefix,
pretrained_target=args.pretrained_target)
test_data = TransformDataset(test_data,
Transform(faster_rcnn, mirror=False))
if args.proc_num == 1:
test_iter = SerialIterator(test_data,
1,
repeat=False,
shuffle=True)
else:
test_iter = MultiprocessIterator(test_data,
batch_size=1,
n_processes=args.proc_num,
repeat=False,
shuffle=True,
n_prefetch=10,
shared_mem=10000000)
gpu = int(args.gpu) if "," not in args.gpu else int(
args.gpu[:args.gpu.index(",")])
chainer.cuda.get_device_from_id(gpu).use()
faster_rcnn.to_gpu(gpu)
evaluator = AUEvaluator(
test_iter,
faster_rcnn,
lambda batch, device: concat_examples_not_none(
batch, device, padding=-99),
args.database,
"/home/machen/face_expr",
device=gpu)
observation = evaluator.evaluate()
with open(
args.out + os.path.sep +
"evaluation_intensity_split_{}_result_test_mode.json".
format(args.split_idx), "w") as file_obj:
file_obj.write(
json.dumps(observation, indent=4, separators=(',', ': ')))
file_obj.flush()
return
train_data = AUDataset(database=args.database,
img_resolution=args.img_resolution,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
prefix=args.prefix,
pretrained_target=args.pretrained_target)
train_data = TransformDataset(train_data,
Transform(faster_rcnn, mirror=True))
if args.proc_num == 1:
train_iter = SerialIterator(train_data, args.batch_size, True, True)
else:
train_iter = MultiprocessIterator(train_data,
batch_size=args.batch_size,
n_processes=args.proc_num,
repeat=True,
shuffle=True,
n_prefetch=10,
shared_mem=31457280)
model = FasterRCNNTrainChain(faster_rcnn)
if "," in args.gpu:
for gpu in args.gpu.split(","):
chainer.cuda.get_device_from_id(int(gpu)).use()
else:
chainer.cuda.get_device_from_id(int(args.gpu)).use()
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
print("using Adam")
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
print("using AdaDelta")
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
if not os.path.exists(args.out):
os.makedirs(args.out)
pretrained_optimizer_file_name = '{0}_fold_{1}_{2}_{3}_{4}_optimizer.npz'.format(
args.fold, args.split_idx, args.feature_model, "linear",
optimizer_name)
pretrained_optimizer_file_name = args.out + os.sep + pretrained_optimizer_file_name
single_model_file_name = args.out + os.sep + '{0}_fold_{1}_{2}_{3}_model.npz'.format(
args.fold, args.split_idx, args.feature_model, "linear")
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model.faster_rcnn)
if "," in args.gpu:
gpu_dict = {"main": int(args.gpu.split(",")[0])} # many gpu will use
for slave_gpu in args.gpu.split(",")[1:]:
gpu_dict[slave_gpu] = int(slave_gpu)
updater = chainer.training.ParallelUpdater(
train_iter,
optimizer,
devices=gpu_dict,
converter=lambda batch, device: concat_examples(
batch, device, padding=-99))
else:
print("only one GPU({0}) updater".format(args.gpu))
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=int(args.gpu),
converter=lambda batch, device: concat_examples(
batch, device, padding=-99))
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
snap_model_file_name = '{0}_fold_{1}_{2}_model.npz'.format(
args.fold, args.split_idx, args.feature_model)
trainer.extend(chainer.training.extensions.snapshot_object(
model.faster_rcnn, filename=snap_model_file_name),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 100, 'iteration'
plot_interval = 100, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
# use Adam
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.5, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="{0}_fold_{1}.log".format(args.fold, args.split_idx)))
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/pearson_correlation',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name='loss_{0}_fold_{1}.png'.format(args.fold,
args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/pearson_correlation'],
file_name='pearson_correlation_{0}_fold_{1}.png'.format(
args.fold, args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def concat_and_pad_examples(batch, device=None, padding=-10000):
return concat_examples(batch, device=device, padding=padding)
def __call__(self, x):
'''
Do a forward pass on the inference policy
Args:
x -- list (!) of actions / states up to final state,
i.e. [a_0,x_1,a_1,x_2, ..., a_{f-1}, x_f]
'''
#
# PREPROCESSING
# Sequence of (a_{t},x_{t+1}) pairs of actions and caused observations
# concat_examples(x) converts an "array of tuples" x into a
# "tuple of arrays" action_seq, input_observation_seq
# I.e. from x = [a_0,x_1,a_1,x_2, ..., a_{f-1}, x_f] to
# [a_0,a_1,...,a_{f-1}], [x_1,x_2,...,x_f]
# where the first is a single list, and the latter are two arrays.
action_seq, input_observation_seq = concat_examples(x)
input_observation_seq = Batch.to_array_and_type(
input_observation_seq, self.xp.float32)
trajectory_length = action_seq.size
# Create array of one-hot vectors for each action in the target sequence
# We only need this for actions a_{t-1} in the LSTM,
# so omitting last action from one-hot sequence
prev_action_seq_oh = self.xp.zeros((trajectory_length-1, self._output_space.n),\
dtype=self.xp.float32)
prev_action_seq_oh[self.xp.arange(trajectory_length - 1),
action_seq[0:-1]] = 1.0
# Wrap both input observations and target actions into variable object
# to keep track of computational graph.
# This is required for the backward pass / gradient computation later on
prev_action_seq_oh = Variable(prev_action_seq_oh)
input_observation_seq = Variable(input_observation_seq)
#
# FORWARD PASS
# Create embeddings u_t of observations x_t with last
# element = embedding of final observation x_f
u_seq = F.relu(self.lin_embed_p(input_observation_seq))
# Create sequence of concatenated
# (observation embedding at t, final observation embedding) pairs [u_t, u_f] for all t < f
# Excluding [u_f, u_f] as this is the state that we use to infer actions chosen
# at previous states u_t, t<f
# For concatenation, we need two arrays of same size f-1. Thus broadcasting u_f to fit size.
# u_f=u_seq[-1], i.e. the last element in the sequence of embeddings.
u_f = F.broadcast_to(u_seq[-1],
(trajectory_length - 1, u_seq[-1].size))
input_embedding_seq = F.concat((u_seq[0:-1], u_f), axis=1)
# Create joint embeddings v=RW[u_t,u_f] for each concatenated observation embedding
v_seq = F.relu(self.lin_embed_q(input_embedding_seq))
# Output from behaviour policy LSTM at different timesteps.
# TODO: Take actual output from policy LSTM (self.lstm_p) once that's implemented.
# Constant for now as we assume random policy for testing.
h_p = self.xp.full((trajectory_length-1, self._lstm_size),\
1.0/self._lstm_size, dtype=self.xp.float32)
# Inference LSTM input [h^p_t, v_t, a_{t-1}]
lstm_q_in = F.concat((h_p, v_seq, prev_action_seq_oh))
# Get into batch shape for N-step LSTM
# LSTM expects list with 2d arrays. 1st index = time,
# 2nd index = batch index, 3rd index = LSTM input elements
# Here we've only got one batch, thus first dimension = trajectory length / time,
# second dimension = 1, and third dimension is size of LSTM input.
lstm_q_in_list = []
for t in range(trajectory_length - 1):
lstm_q_in_list.append(F.expand_dims(lstm_q_in[t], axis=0))
# Feed it through n-step LSTM
# Returns a tuple: final hidden states at t=f, updated cell state at t=f,
# updated hidden states after each t
# We're only interested in the third, as this is the networks' prediction for time t
h_q = self.lstm_q(None, None, lstm_q_in_list)[2]
# Turn list of batches of hidden state vectors (list[2darray])
# into 2-d array of hidden state vectors
h_q = F.stack(h_q)
h_q = F.squeeze(h_q)
# Use another linear transformation to bring LSTM output h to the size of our action set
# This way, the size of our action set does not constrain the expressiveness of the LSTM
predictions = self.lin_pi_q(h_q)
# We return the outcome predictions and target action sequences
# Predictions = the unnormalised log-probabilities, as our inferred probabilities of
# actions performed at t, given observation x_t and final observation x_f
# Target action sequences = only the actions following the observations,
# i.e. first action in original sequence omitted
target_action_seq = action_seq[1:]
return (predictions, target_action_seq)
def concat_and_reshape(batch, device=None, padding=None):
x, y = dataset.concat_examples(batch, device, padding)
return x.reshape(len(x), 1, 784), y
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='output_two_stream',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean_rgb',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--mean_flow',
default=config.ROOT_PATH + "BP4D/idx/mean_flow.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="resnet101",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='SGD',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model_rgb',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument(
'--pretrained_model_flow',
help=
"path of optical flow pretrained model, can also use the same npz with rgb"
)
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='rgb_flow/ optical_flow/ rgb')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--fix",
action="store_true",
help="fix parameter of conv2 update when finetune")
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument(
"--roi_align",
action="store_true",
help="whether to use roi align or roi pooling layer in CNN")
parser.add_argument("--T", '-T', type=int, default=10)
parser.add_argument("--proc_num", "-proc", type=int, default=1)
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
paper_report_label, class_num = squeeze_label_num_report(
args.database, args.use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
au_rcnn_train_chain_list = []
if args.backbone == 'vgg':
au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
elif args.backbone == 'resnet101':
if args.two_stream_mode != TwoStreamMode.rgb_flow:
assert (args.pretrained_model_rgb == "" and args.pretrained_model_flow != "") or\
(args.pretrained_model_rgb != "" and args.pretrained_model_flow == "")
pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_flow
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=(
args.two_stream_mode == TwoStreamMode.optical_flow),
temporal_length=args.T)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
else: # rgb_flow mode
au_rcnn_rgb = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=False,
temporal_length=args.T)
au_rcnn_optical_flow = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_flow,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=True,
temporal_length=args.T)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
model = Wrapper(au_rcnn_train_chain_list,
class_num,
args.database,
args.T,
two_stream_mode=args.two_stream_mode,
gpus=args.gpu)
batch_size = args.batch_size
img_dataset = AUDataset(database=args.database,
L=args.T,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
two_stream_mode=args.two_stream_mode,
train_all_data=False,
paper_report_label_idx=paper_report_label_idx)
train_dataset = TransformDataset(
img_dataset,
Transform(L=args.T,
mirror=True,
mean_rgb_path=args.mean_rgb,
mean_flow_path=args.mean_flow))
if args.proc_num == 1:
train_iter = SerialIterator(train_dataset,
batch_size,
repeat=True,
shuffle=True)
else:
train_iter = MultiprocessIterator(train_dataset,
batch_size=batch_size,
n_processes=args.proc_num,
repeat=True,
shuffle=True,
n_prefetch=3,
shared_mem=53457280)
if len(args.gpu) > 1:
for gpu in args.gpu:
chainer.cuda.get_device_from_id(gpu).use()
else:
chainer.cuda.get_device_from_id(args.gpu[0]).use()
model.to_gpu(args.gpu[0])
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
# BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz
use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label"
roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling"
single_model_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_model.npz'.format(args.database,
args.fold,
args.split_idx,
args.backbone,
args.two_stream_mode,
use_paper_key_str,
roi_align_key_str,
args.T)
print(single_model_file_name)
pretrained_optimizer_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_optimizer.npz'.format(
args.database,
args.fold, args.split_idx,
args.backbone, args.two_stream_mode,
use_paper_key_str, roi_align_key_str,
args.T)
print(pretrained_optimizer_file_name)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model)
if args.fix:
au_rcnn = model.au_rcnn_train_chain.au_rcnn
au_rcnn.extractor.conv1.W.update_rule.enabled = False
au_rcnn.extractor.bn1.gamma.update_rule.enabled = False
au_rcnn.extractor.bn1.beta.update_rule.enabled = False
res2_names = ["a", "b1", "b2"]
for res2_name in res2_names:
if res2_name == "a":
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv4.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.beta.update_rule.enabled = False
elif res2_name.startswith("b"):
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=args.gpu[0],
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
trainer = training.Trainer(updater, (10, 'iteration'), out=args.out)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=os.path.basename(single_model_file_name)),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 100, 'iteration'
plot_interval = 10, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.1, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="log_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.log".format(
args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T)))
# trainer.reporter.add_observer("main_par", model.loss_head_module)
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name="loss_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png".format(
args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name="accuracy_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png".
format(args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T),
trigger=plot_interval),
trigger=plot_interval)
# trainer.run()
cProfile.runctx("trainer.run()", globals(), locals(), "Profile.prof")
s = pstats.Stats("Profile.prof")
s.strip_dirs().sort_stats("time").print_stats()
def train(model):
dir_path = 'mnist_mixup_' + str(datetime.date.today())
if not os.path.isdir(dir_path):
os.mkdir(dir_path)
train, test = mnist.get_mnist(withlabel=True, ndim=1)
train_iter = iterators.SerialIterator(train, args.batchsize, shuffle=False)
test_iter = iterators.SerialIterator(test,
args.batchsize,
repeat=False,
shuffle=False)
optimizer = optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
optimizer.use_cleargrads()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=args.decay))
result = []
train_losses = []
while train_iter.epoch < args.epoch:
train_batch = train_iter.next()
x, t = concat_examples(train_batch, gpu_id)
x, t_a, t_b, lam = mixup_data(x, t, args.alpha)
x = Variable(x)
t_a = Variable(t_a)
t_b = Variable(t_b)
y = model(x)
loss = mixup_criterion(y, t_a, t_b, lam)
train_losses.append(to_cpu(loss.data))
model.cleargrads()
loss.backward()
optimizer.update()
if train_iter.is_new_epoch:
if train_iter.epoch % 5 == 0:
print('epoch:{: 02d} train_loss:{: .04f}'.format(
train_iter.epoch, float(np.mean(train_losses))),
end=' ')
adjust_lr(optimizer, train_iter.epoch)
test_losses = []
test_accuracies = []
while True:
test_batch = test_iter.next()
x_test, t_test = concat_examples(test_batch, gpu_id)
y_test = model(x_test)
loss_test = F.softmax_cross_entropy(y_test, t_test)
test_losses.append(to_cpu(loss_test.data))
accuracy = F.accuracy(y_test, t_test)
accuracy.to_cpu()
test_accuracies.append(accuracy.data)
if test_iter.is_new_epoch:
test_iter.epoch = 0
test_iter.current_position = 0
test_iter.is_new_epoch = False
test_iter._pushed_position = None
break
if train_iter.epoch % 5 == 0:
print('val_loss:{: .04f} val_accuracy:{: .04f}'.format(
np.mean(test_losses), np.mean(test_accuracies)))
result.append([
train_iter.epoch,
float(np.mean(train_losses)),
np.mean(test_losses)
])
serializers.save_npz(
'mnist_mixup_' + str(datetime.date.today()) + '/epoch-' +
str(train_iter.epoch), model)
train_losses.clear()
result = np.asarray(result)
epoch = result[:, 0]
tr_loss = result[:, 1]
te_loss = result[:, 2]
plt.plot(epoch, tr_loss, color='blue', label="main/loss", marker="x")
plt.plot(epoch,
te_loss,
color='orange',
label="validation/main/loss",
marker="x")
plt.legend()
plt.savefig("mnist_mixup_" + str(datetime.date.today()) + "/loss.png")
shuffle=False)
model = MujiNN()
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
optimizer.setup(model)
max_epoch = 100
pLoss = 10.0
# while train_iter.epoch < max_epoch:
while pLoss > 0.0001:
# ---------- One iteration of the training loop ----------
train_batch = train_iter.next()
data_train, target_train = concat_examples(train_batch)
# Calculate the prediction of the network
prediction_train = model(data_train)
# Calculate the loss with softmax_cross_entropy
loss = F.softmax_cross_entropy(prediction_train, target_train)
pLoss = float(loss.data)
# Calculate the gradients in the network
model.cleargrads()
loss.backward()
# Update all the trainable paremters
optimizer.update()
# --------------------- until here ---------------------
def train(
iterator, gpu, encoder, decoder, enc_optim, dec_optim, rel_send, rel_rec, edge_types,
temp, prediction_steps, var, out, benchmark, lr_decay, gamma):
iter_i = 0
edge_accuracies = []
node_mses = []
nll_train = []
kl_train = []
logger = logging.getLogger(__name__)
while True:
inputs = iterator.next()
node_features, edge_labels = dataset.concat_examples(inputs, device=gpu)
# logits: [batch_size, num_edges, edge_types]
logits = encoder(node_features, rel_send, rel_rec) # inverse func. of softmax
edges = F.gumbel_softmax(logits, tau=temp, axis=2)
edge_probs = F.softmax(logits, axis=2)
# edges, edge_probs: [batch_size, num_edges, edge_types]
if isinstance(decoder, decoders.MLPDecoder):
output = decoder(
node_features, edges, rel_rec, rel_send, prediction_steps)
elif isinstance(decoder, decoders.RNNDecoder):
output = decoder(
node_features, edges, rel_rec, rel_send, 100,
burn_in=True,
burn_in_steps=args.timesteps - args.prediction_steps)
target = node_features[:, :, 1:, :]
num_nodes = node_features.shape[1]
loss_nll = get_nll_gaussian(output, target, var)
loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types)
loss = loss_nll + loss_kl
nll_train.append(float(loss_nll.array))
kl_train.append(float(loss_kl.array))
edge_accuracy = get_edge_accuracy(logits.array, edge_labels)
edge_accuracies.append(edge_accuracy)
node_mse = float(F.mean_squared_error(output, target).array)
node_mses.append(node_mse)
encoder.cleargrads()
decoder.cleargrads()
loss.backward()
enc_optim.update()
dec_optim.update()
# Exit after 10 iterations when benchmark mode is ON
iter_i += 1
if benchmark:
put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train),
np.mean(edge_accuracies), np.mean(node_mses))
if iter_i == 10:
exit()
if iterator.is_new_epoch:
break
if not os.path.exists(os.path.join(out, 'graph.dot')):
with open(os.path.join(out, 'graph.dot'), 'w') as o:
g = computational_graph.build_computational_graph([loss])
o.write(g.dump())
if iterator.is_new_epoch:
put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses))
serializers.save_npz(os.path.join(out, 'encoder_epoch-{}.npz'.format(iterator.epoch)), encoder)
serializers.save_npz(os.path.join(out, 'decoder_epoch-{}.npz'.format(iterator.epoch)), decoder)
serializers.save_npz(os.path.join(out, 'enc_optim_epoch-{}.npz'.format(iterator.epoch)), enc_optim)
serializers.save_npz(os.path.join(out, 'dec_optim_epoch-{}.npz'.format(iterator.epoch)), dec_optim)
if iterator.epoch % lr_decay == 0:
enc_optim.alpha *= gamma
dec_optim.alpha *= gamma
logger.info('alpha of enc_optim: {}'.format(enc_optim.alpha))
logger.info('alpha of dec_optim: {}'.format(dec_optim.alpha))
def evaluate(model, dataset, device=-1, flip=False):
batch_size = 50
data_iter = chainer.iterators.MultithreadIterator(dataset,
batch_size,
repeat=False,
shuffle=False)
corrects = list()
counts = list()
for it, batch in enumerate(data_iter):
# print progress
print(f'{batch_size*it:04d} / {len(dataset):04d}', end='\r')
img, label, idx, scale, shape = concat_examples(batch)
N, C, H, W = img.shape
if flip:
img = np.array((img, img[:, :, :, ::-1]))
img = img.reshape(N * 2, C, H, W)
if device >= 0:
img = cuda.to_gpu(img)
with chainer.no_backprop_mode():
# (N, 3, 256, 256) -> (N, 16, 64, 64)
_output, output = model(img)
output = output.array
if flip:
output = output.reshape((
2,
N,
) + output.shape[1:])
output_flipped = flip_heatmap(output[1], copy=True)
output = (output[0] + output_flipped) / 2
N, C, H, W = output.shape
keypoints = list()
# (N, 16, 64, 64) -> (N, 16, 2)
for i in range(N):
# (16, 64, 64) -> (16, -1)
out_reshaped = output[i].reshape(C, -1).argmax(axis=1)
out_reshaped = cuda.to_cpu(out_reshaped)
keypoint = np.unravel_index(out_reshaped, (H, W))
# (2, 16) -> (16, 2)
keypoint = np.array(keypoint).T
keypoint = transforms.resize_point(keypoint, (H, W), shape[i])
keypoints.append(keypoint)
else:
keypoints = np.array(keypoints)
correct, count = pckh_score(label, keypoints, idx, scale)
corrects.append(correct)
counts.append(count)
print()
corrects = np.sum(corrects, axis=0)
counts = np.sum(counts, axis=0)
# Head, Shoulder, Elbow, Wrist, Hip, Knee, Ankle
joints = {
'head': [8, 9],
'shoulder': [12, 13],
'elbow': [11, 14],
'wrist': [10, 15],
'hip': [2, 3],
'knee': [1, 4],
'ankle': [0, 5]
}
scores = dict()
for key, value in joints.items():
score = corrects[value].sum() / counts[value].sum()
scores.update({key: score})
return scores
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
config_names = ['data', 'param', 'arch', 'model', 'norm', 'gpu', 'layer']
config_lst = [
args.data_path, args.param_path, args.arch_path, args.model_path,
args.norm, args.gpu, args.layer
]
config_dic = cl.OrderedDict()
for n, c in zip(config_names, config_lst):
config_dic[n] = c
with open(os.path.join(args.out_dir, 'config.json'), 'w') as fp:
json.dump(config_dic, fp)
ds_info = get_dataset(args.data_path, norm=args.norm)
data = concat_examples(ds_info.__getitem__(slice(0, ds_info.__len__(), 1)),
device=args.gpu)
with open(args.param_path, 'r') as fp:
param = json.load(fp)
with open(args.arch_path, 'r') as fp:
arch = json.load(fp)
model = predict_four(arch, dr=param['dr'], bn=param['bn'])
serializers.load_npz(args.model_path, model)
if args.gpu >= 0:
cuda.get_device(data[0][0]).use()
model.to_gpu()
xp = cuda.get_array_module(data[0][0])
model.train = False
cor_dir = os.path.join(args.out_dir, 'correct')
if gpu_id >= 0:
model.to_gpu(gpu_id)
from chainer import optimizers
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
optimizer.setup(model)
# train loop
import numpy as np
from chainer.dataset import concat_examples
from chainer.backends.cuda import to_cpu
max_epoch = 10
while train_iter.epoch < max_epoch:
train_batch = train_iter.next()
image_train, target_train = concat_examples(train_batch, gpu_id)
prediction_train = model(image_train)
loss = F.softmax_cross_entropy(prediction_train, target_train)
model.cleargrads()
loss.backward()
optimizer.update()
if train_iter.is_new_epoch:
print('epoch:{:02d} train_loss{:04f}'.format(
train_iter.epoch, float(to_cpu(loss.data))),
end='')
test_losses = []
import chainer.iterators
from chainer.dataset import concat_examples
from chainer.cuda import to_cpu
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from load_model import forward, load_model
from Attack_Function.fgsm import fgsm
models = load_model()
chainer.config.train = False #turn model to eval mode
train, test = chainer.datasets.get_fashion_mnist(ndim=3)
train_iter = chainer.iterators.SerialIterator(test, 1)
i = 1
eps_list = [0.05, 0.1, 0.3, 0.6]
for eps in eps_list:
train_iter.current_position = 0
train_iter.epoch = 0
train_iter.is_new_epoch = False
train_iter._pushed_position = None #reset test_iter
test_batch = train_iter.next()
image_test, target_test = concat_examples(test_batch)
image_test = F.resize_images(image_test, (32, 32)).data
adv = fgsm(models, forward, image_test, eps=eps)
plt.subplot(1, 4, i)
plt.imshow(to_cpu(adv.squeeze()))
plt.title("Label:{}".format(to_cpu(F.argmax(forward(adv, models)).data)))
i = i + 1
plt.savefig("demo.jpg")
def save_results(dataset, saveflag, filename, lam, w, z):
for n in range(len(dataset.lens)):
in_vars = tuple(
Variable(x, volatile='on') for x in dataset_module.concat_examples(
dataset[sum(dataset.lens[0:n]):sum(dataset.lens[0:n + 1])]))
g0, g1, h0, h1 = loss.net(*in_vars, phi=loss.phi, train=False)
# prepare outputs
y0 = np.transpose(in_vars[0].data, axes=(1, 0, 2))[-1]
y1 = np.transpose(in_vars[1].data, axes=(1, 0, 2))[-1]
g0 = cuda.to_cpu(g0.data)
g1 = cuda.to_cpu(g1.data)
h0 = cuda.to_cpu(h0.data)
h1 = cuda.to_cpu(h1.data)
# do prediction in g-space
g_preds = [] # starts from prediction corresponding to g1
for j in range(args.horizon):
tmp = np.zeros(g0.shape) + 1j * np.zeros(g0.shape)
if j == 0:
g_prev = g0
else:
g_prev = g_preds[j - 1]
for i in range(g1.shape[0]):
tmp[i] = np.sum(np.dot(
w,
np.diag(np.dot(np.diag(lam), np.dot(z.conj().T,
g_prev[i])))),
axis=1)
g_preds.append(tmp.copy())
# back-project the prediction to y-space
y_preds = []
rmses = [0 for i in range(args.horizon)]
for j in range(args.horizon):
g_pred_var = Variable(np.real(g_preds[j]).astype(dtype=np.float32),
volatile='on')
tmp = loss.net.h(g_pred_var, train=False).data
y_preds.append(tmp.copy())
if j == 0:
tmp_end = tmp.shape[0]
else:
tmp_end = -j
rmses[j] = np.sqrt(np.mean(np.square(tmp[:tmp_end] - y1[j:])))
# save
if saveflag:
savemat(
os.path.join(args.outputdir, '{0}_{1:d}'.format(filename, n)),
{
'lam': lam,
'w': w,
'z': z,
'prediction_rmses': rmses,
'phi_W': loss.phi.l1.W.data,
'phi_b': loss.phi.l1.b.data,
'g_preds': g_preds,
'y_preds': y_preds,
'y0': y0,
'g0': g0,
'h0': h0,
'y1': y1,
'g1': g1,
'h1': h1
})
return rmses
def main():
parser = argparse.ArgumentParser(description='Chainer example: MNIST')
parser.add_argument('--batchsize',
'-b',
type=int,
default=100,
help='Number of images in each mini-batch')
parser.add_argument('--epoch',
'-e',
type=int,
default=20,
help='Number of sweeps over the dataset to train')
parser.add_argument('--frequency',
'-f',
type=int,
default=-1,
help='Frequency of taking a snapshot')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out',
'-o',
default='result',
help='Directory to output the result')
parser.add_argument('--resume',
'-r',
default='',
help='Resume the training from snapshot')
parser.add_argument('--unit',
'-u',
type=int,
default=50,
help='Number of units')
parser.add_argument('--example',
'-ex',
type=int,
default=3,
help='Example mode')
args = parser.parse_args()
print('GPU: {}'.format(args.gpu))
print('# unit: {}'.format(args.unit))
print('# Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('')
# Load the MNIST dataset
train, test = chainer.datasets.get_mnist()
model = SklearnWrapperClassifier(MLP(args.unit, 10), device=args.gpu)
if args.example == 1:
print("Example 1. fit with x, y numpy array (same with sklearn's fit)")
x, y = concat_examples(train)
model.fit(x, y)
elif args.example == 2:
print("Example 2. Train with Chainer's dataset")
# `train` is TupleDataset in this example
# Even this one line work! (but no validation)
model.fit(train)
else:
print("Example 3. Train with configuration")
model.fit(
train,
test=test,
batchsize=args.batchsize,
#iterator_class=chainer.iterators.SerialIterator,
optimizer=chainer.optimizers.Adam(),
epoch=args.epoch,
out=args.out,
snapshot_frequency=1,
#dump_graph=False
#log_report=True,
plot_report=False,
#print_report=True,
progress_report=False,
resume=args.resume)
# Save trained model
serializers.save_npz('{}/mlp.model'.format(args.out), model)
def evaluate(self, snapshot_name=''):
reporter = reporter_module.Reporter()
current_device = chainer.get_device(self.args.gpu)
summary = reporter_module.DictSummary()
with chainer.using_device(current_device), reporter, configuration.using_config('train', False):
for i, batch in enumerate(tqdm(self.data_iterator, total=len(self.data_loader) // self.args.batchsize)):
observation = {}
batch = concat_examples(batch, self.args.gpu)
image_size = Size._make(batch['image'].shape[-2:])
with reporter_module.report_scope(observation):
rois, bboxes, text_predictions, best_indices, chosen_prediction, scores = self.evaluator(return_predictions=True, **batch)
summary.add(observation)
if self.args.save_predictions:
assert self.args.batchsize == 1, "if you want to save predictions, batchsize must be 1!"
batch_size, num_predictions, num_bboxes, num_channels, height, width = rois.shape
base_image = self.bbox_plotter.array_to_image(batch['image'][0, 0])
chosen_word = self.data_loader.decode_chars(cuda.to_cpu(text_predictions[0, chosen_prediction[0]].squeeze()))
base_image = self.bbox_plotter.render_text(base_image, base_image, chosen_word, 0, bottom=True)
rendered_images = [base_image]
iterator = zip(
F.separate(self.localizer.xp.stack(
[batch['image'][i, best_indices[i]] for i in range(self.args.batchsize)]), axis=1),
F.separate(rois, axis=1), F.separate(bboxes, axis=1), F.separate(text_predictions, axis=1),
F.separate(scores, axis=1)
)
for image, roi, bbox, prediction, score in iterator:
image = image.array
roi = roi.array
bbox = bbox.array
prediction = prediction.array
score = score.array
bbox = self.localizer.xp.reshape(bbox, (-1, 2, height, width))
predicted_words = self.data_loader.decode_chars(cuda.to_cpu(prediction.squeeze()))
predicted_words = f"{predicted_words} {format(float(score[0]), '.4f')}"
if args.cut_bboxes:
cut_length = batch['num_words'][0] if 'num_words' in batch else len(predicted_words)
bbox = bbox[:cut_length, ...]
roi = roi[:cut_length, ...]
if args.render_no_boxes:
bbox = bbox[:1]
roi = roi[:1]
rendered_images.append(
self.render_roi(
[],
bbox,
None,
i,
image,
roi,
predicted_words
)
)
self.save_rois(rendered_images, i)
self.save_eval_results(snapshot_name, summary)
def grad_cam(model, val, gpu, save_dir, backward_label='true_label'):
xp = cp if gpu >= 0 else np
# batchサイズ(今回は1枚ずつ行うので1)
cam_iter = chainer.iterators.SerialIterator(val,
1,
repeat=False,
shuffle=False)
count = 0
while True:
# brakeするまで続ける
X_test_batch = cam_iter.next()
# テストデータをデータとラベルに分ける
X_test, y_test = concat_examples(X_test_batch, gpu)
with chainer.using_config("train", False):
pred = model.predictor(X_test)
probs = F.softmax(pred).data[0]
if gpu >= 0:
probs = chainer.cuda.to_cpu(probs)
top1 = np.argsort(probs)[::-1][0]
pred.zerograd()
pred.grad = xp.zero([1, 8], dtype=np.float32)
if backward_label == 'true_label':
backward_label = y_test[0]
else:
backward_label = top1
pred.grad[0, backward_label] = 1
pred.backward(True)
feature = model.predictor.cam.data[0]
grad = model.predictor.cam.grad[0]
cam = xp.ones(feature.shape[1:], dtype=xp.float32)
weights = grad.mean((1, 2)) * 1000
for i, w in enumerate(weights):
cam += feature[i] * w
if gpu >= 0:
cam = chainer.cuda.to_cpu(cam)
X_test = chainer.cuda.to_cpu(X_test)
cam = cv2.resize(cam, (224, 224))
cam = np.maximum(cam, 0)
heatmap = cam / np.mat(cam)
heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
###########################################################################
image = X_test[0, ::-1, :, :].transpose(1, 2, 0)
image -= np.min(image)
image = np.minimum(image, 255)
cam_img = np.float32(image)
cam_img = np.float32(heatmap) + np.float32(image)
cam_img = 255 * cam_img / np.max(cam_img)
save_img_dir = save_dir + '/visualize'
if not os.exists(save_img_dir):
os.mkdir(save_img_dir)
save_name = save_img_dir + '/cam_img_{}.png'.format(count)
cv2.imwrite(save_name, cam_img)
count += 1
if cam_iter.is_new_epoch:
cam_iter.epoch = 0
cam_iter.current_position = 0
cam_iter.is_new_epoch = False
cam_iter._pushed_position = None
break
def training_loop(model=None,
optimizer=None,
stats=None,
epochs=None,
train_iter=None,
test_iter=None,
lf=None,
models_folder=None,
epochs_so_far=0,
args=None):
train_losses = []
train_accs = []
while train_iter.epoch < epochs:
# ------------ One epoch of the training loop ------------
# ---------- One iteration of the training loop ----------
train_batch = train_iter.next()
image_train = concat_examples(train_batch, 0)
# Calculate the loss with softmax_cross_entropy
train_loss, train_rec_loss, train_label_loss, acc, _ = model.get_loss_func(
)(image_train)
train_losses.append(train_loss.array)
train_accs.append(acc.array)
# Calculate the gradients in the network
model.cleargrads()
train_loss.backward()
# Update all the trainable paremters
optimizer.update()
if train_iter.epoch % int(args.freq) == 0:
serializers.save_npz(
os.path.join(models_folder,
str(train_iter.epoch + epochs_so_far) + '.model'),
model)
# --------------------- iteration until here ---------------------
if train_iter.is_new_epoch:
test_losses = []
test_accs = []
test_rec_losses = []
test_label_losses = []
test_kl = []
while True:
test_batch = test_iter.next()
image_test = concat_examples(test_batch, 0)
loss, rec_loss, label_loss, label_acc, kl = model.get_loss_func(
)(image_test)
test_losses.append(loss.array)
test_rec_losses.append(rec_loss.array)
test_label_losses.append(label_loss.array)
test_accs.append(label_acc.array)
test_kl.append(kl.array)
if test_iter.is_new_epoch:
test_iter.epoch = 0
test_iter.current_position = 0
test_iter.is_new_epoch = False
test_iter._pushed_position = None
break
stats['train_loss'].append(np.mean(to_cpu(train_losses)))
stats['train_accs'].append(np.mean(to_cpu(train_accs)))
stats['valid_loss'].append(np.mean(to_cpu(test_losses)))
stats['valid_rec_loss'].append(np.mean(to_cpu(test_rec_losses)))
stats['valid_label_loss'].append(np.mean(
to_cpu(test_label_losses)))
stats['valid_label_acc'].append(np.mean(to_cpu(test_accs)))
stats['valid_kl'].append(np.mean(to_cpu(test_kl)))
print(("Epoch: {0} \t T_Loss: {1} \t V_Loss: {2} \t V_Rec_Loss: {3} \t V_Label_Loss: {4} \t " + \
"V_KL: {6} \t T_Acc: {7} \t V_Acc: {5}").format(train_iter.epoch,
round(stats['train_loss'][-1], 2),
round(stats['valid_loss'][-1], 2),
round(stats['valid_rec_loss'][-1], 2),
round(stats['valid_label_loss'][-1], 2),
round(stats['valid_label_acc'][-1], 2),
round(stats['valid_kl'][-1], 2),
round(stats['train_accs'][-1], 2)))
train_losses = []
train_accs = []
# --------------------- epoch until here ---------------------
return stats, model, optimizer, epochs
