def check_concat_dicts_padding(self, xp): dicts = [ {'x': xp.random.rand(3, 4), 'y': xp.random.rand(2, 5)}, {'x': xp.random.rand(4, 4), 'y': xp.random.rand(3, 4)}, {'x': xp.random.rand(2, 5), 'y': xp.random.rand(2, 6)}, ] arrays = dataset.concat_examples(dicts, padding=0) self.assertIn('x', arrays) self.assertIn('y', arrays) self.assertEqual(arrays['x'].shape, (3, 4, 5)) self.assertEqual(arrays['y'].shape, (3, 3, 6)) self.assertEqual(type(arrays['x']), type(dicts[0]['x'])) self.assertEqual(type(arrays['y']), type(dicts[0]['y'])) for d in dicts: d['x'] = cuda.to_cpu(d['x']) d['y'] = cuda.to_cpu(d['y']) arrays = {'x': cuda.to_cpu(arrays['x']), 'y': cuda.to_cpu(arrays['y'])} numpy.testing.assert_array_equal(arrays['x'][0, :3, :4], dicts[0]['x']) numpy.testing.assert_array_equal(arrays['x'][0, 3:, :], 0) numpy.testing.assert_array_equal(arrays['x'][0, :, 4:], 0) numpy.testing.assert_array_equal(arrays['x'][1, :4, :4], dicts[1]['x']) numpy.testing.assert_array_equal(arrays['x'][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays['x'][2, :2, :5], dicts[2]['x']) numpy.testing.assert_array_equal(arrays['x'][2, 2:, :], 0) numpy.testing.assert_array_equal(arrays['y'][0, :2, :5], dicts[0]['y']) numpy.testing.assert_array_equal(arrays['y'][0, 2:, :], 0) numpy.testing.assert_array_equal(arrays['y'][0, :, 5:], 0) numpy.testing.assert_array_equal(arrays['y'][1, :3, :4], dicts[1]['y']) numpy.testing.assert_array_equal(arrays['y'][1, 3:, :], 0) numpy.testing.assert_array_equal(arrays['y'][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays['y'][2, :2, :6], dicts[2]['y']) numpy.testing.assert_array_equal(arrays['y'][2, 2:, :], 0)
def check_concat_tuples_padding(self, xp): tuples = [ (xp.random.rand(3, 4), xp.random.rand(2, 5)), (xp.random.rand(4, 4), xp.random.rand(3, 4)), (xp.random.rand(2, 5), xp.random.rand(2, 6)), ] arrays = dataset.concat_examples(tuples, padding=0) self.assertEqual(len(arrays), 2) self.assertEqual(arrays[0].shape, (3, 4, 5)) self.assertEqual(arrays[1].shape, (3, 3, 6)) self.assertEqual(type(arrays[0]), type(tuples[0][0])) self.assertEqual(type(arrays[1]), type(tuples[0][1])) for i in range(len(tuples)): tuples[i] = cuda.to_cpu(tuples[i][0]), cuda.to_cpu(tuples[i][1]) arrays = tuple(cuda.to_cpu(array) for array in arrays) numpy.testing.assert_array_equal(arrays[0][0, :3, :4], tuples[0][0]) numpy.testing.assert_array_equal(arrays[0][0, 3:, :], 0) numpy.testing.assert_array_equal(arrays[0][0, :, 4:], 0) numpy.testing.assert_array_equal(arrays[0][1, :4, :4], tuples[1][0]) numpy.testing.assert_array_equal(arrays[0][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays[0][2, :2, :5], tuples[2][0]) numpy.testing.assert_array_equal(arrays[0][2, 2:, :], 0) numpy.testing.assert_array_equal(arrays[1][0, :2, :5], tuples[0][1]) numpy.testing.assert_array_equal(arrays[1][0, 2:, :], 0) numpy.testing.assert_array_equal(arrays[1][0, :, 5:], 0) numpy.testing.assert_array_equal(arrays[1][1, :3, :4], tuples[1][1]) numpy.testing.assert_array_equal(arrays[1][1, 3:, :], 0) numpy.testing.assert_array_equal(arrays[1][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays[1][2, :2, :6], tuples[2][1]) numpy.testing.assert_array_equal(arrays[1][2, 2:, :], 0)
def check_concat_arrays(self, arrays, device=None): array = dataset.concat_examples(arrays, device) self.assertEqual(array.shape, (len(arrays),) + arrays[0].shape) self.check_device(array, device) for x, y in zip(array, arrays): numpy.testing.assert_array_equal( cuda.to_cpu(x), cuda.to_cpu(y))
def check_concat_arrays(self, arrays, device, expected_device): array = dataset.concat_examples(arrays, device) self.assertEqual(array.shape, (len(arrays),) + arrays[0].shape) assert backend.get_device_from_array(array) == expected_device np_array = backend.CpuDevice().send(array) for x, y in zip(np_array, arrays): numpy.testing.assert_array_equal(x, backend.CpuDevice().send(y))
def warmup(model, iterator, gpu_id=0): batch = iterator.next() img, img_info, bbox = concat_examples(batch, gpu_id) img = chainer.Variable(img) img_info = chainer.Variable(img_info) bbox = chainer.Variable(bbox) model.rcnn_train = True model(img, img_info, bbox) model.rpn_train = True model(img, img_info, bbox)
def check_concat_arrays( self, arrays, device, expected_device, expected_dtype): array = dataset.concat_examples(arrays, device, self.padding) self.assertEqual(array.shape, (len(arrays),)) self.check_device(array, device, expected_device) np_array = backend.CpuDevice().send(array) for x, y in zip(np_array, arrays): assert x.dtype == expected_dtype numpy.testing.assert_array_equal( x, numpy.array(y, dtype=expected_dtype))
def check_concat_tuples(self, tuples, device=None): arrays = dataset.concat_examples(tuples, device) self.assertEqual(len(arrays), len(tuples[0])) for i in range(len(arrays)): shape = (len(tuples),) + tuples[0][i].shape self.assertEqual(arrays[i].shape, shape) self.check_device(arrays[i], device) for x, y in zip(arrays[i], tuples): numpy.testing.assert_array_equal( cuda.to_cpu(x), cuda.to_cpu(y[i]))
def check_concat_dicts(self, dicts, device=None): arrays = dataset.concat_examples(dicts, device) self.assertEqual(frozenset(arrays.keys()), frozenset(dicts[0].keys())) for key in arrays: shape = (len(dicts),) + dicts[0][key].shape self.assertEqual(arrays[key].shape, shape) self.check_device(arrays[key], device) for x, y in zip(arrays[key], dicts): numpy.testing.assert_array_equal( cuda.to_cpu(x), cuda.to_cpu(y[key]))
def check_concat_tuples(self, tuples, device, expected_device): arrays = dataset.concat_examples(tuples, device) self.assertEqual(len(arrays), len(tuples[0])) for i in range(len(arrays)): shape = (len(tuples),) + tuples[0][i].shape self.assertEqual(arrays[i].shape, shape) assert backend.get_device_from_array(arrays[i]) == expected_device arr = backend.CpuDevice().send(arrays[i]) for x, y in zip(arr, tuples): numpy.testing.assert_array_equal( x, backend.CpuDevice().send(y[i]))
def check_concat_dicts(self, dicts, device, expected_device): arrays = dataset.concat_examples(dicts, device) self.assertEqual(frozenset(arrays.keys()), frozenset(dicts[0].keys())) for key in arrays: shape = (len(dicts),) + dicts[0][key].shape self.assertEqual(arrays[key].shape, shape) self.assertEqual( backend.get_device_from_array(arrays[key]), expected_device) arr = backend.CpuDevice().send(arrays[key]) for x, y in zip(arr, dicts): numpy.testing.assert_array_equal( x, backend.CpuDevice().send(y[key]))
def check_concat_arrays(self, arrays, device, expected_type): array = dataset.concat_examples(arrays, device, self.padding) self.assertEqual(array.shape, (len(arrays),)) self.check_device(array, device) for x, y in zip(array, arrays): if cuda.get_array_module(x) == numpy: numpy.testing.assert_array_equal( numpy.array(x), numpy.array(y, dtype=expected_type)) else: numpy.testing.assert_array_equal( cuda.to_cpu(x), numpy.array(y, dtype=expected_type))
def test(iterator, gpu, timesteps, encoder, decoder, rel_send, rel_rec, edge_types, temp, var): nll_test = [] kl_test = [] edge_accuracies = [] node_mses = [] chainer.config.train = False chainer.config.enable_backprop = False while True: inputs = iterator.next() node_features, edge_labels = dataset.concat_examples(inputs, device=gpu) data_encoder = node_features[:, :, :timesteps, :] data_decoder = node_features[:, :, -timesteps:, :] # logits: [batch_size, num_edges, edge_types] logits = encoder(data_encoder, rel_send, rel_rec) # inverse func. of softmax edges = F.gumbel_softmax(logits, tau=temp, axis=2) edge_probs = F.softmax(logits, axis=2) # edges, edge_probs: [batch_size, num_edges, edge_types] # validation output uses teacher forcing output = decoder(data_decoder, edges, rel_rec, rel_send, 1) target = data_decoder[:, :, 1:, :] num_nodes = node_features.shape[1] loss_nll = get_nll_gaussian(output, target, var) loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types) nll_test.append(float(loss_nll.array)) kl_test.append(float(loss_kl.array)) edge_accuracy = get_edge_accuracy(logits.array, edge_labels) edge_accuracies.append(edge_accuracy) node_mse = float(F.mean_squared_error(output, target).array) node_mses.append(node_mse) if iterator.is_new_epoch: break put_log(iterator.epoch, np.mean(nll_test), np.mean(kl_test), np.mean(edge_accuracies), np.mean(node_mses), 'test') chainer.config.train = True chainer.config.enable_backprop = True
def check_concat_arrays_padding(self, xp): arrays = [xp.random.rand(3, 4), xp.random.rand(2, 5), xp.random.rand(4, 3)] array = dataset.concat_examples(arrays, padding=0) self.assertEqual(array.shape, (3, 4, 5)) self.assertEqual(type(array), type(arrays[0])) arrays = [cuda.to_cpu(a) for a in arrays] array = cuda.to_cpu(array) numpy.testing.assert_array_equal(array[0, :3, :4], arrays[0]) numpy.testing.assert_array_equal(array[0, 3:, :], 0) numpy.testing.assert_array_equal(array[0, :, 4:], 0) numpy.testing.assert_array_equal(array[1, :2, :5], arrays[1]) numpy.testing.assert_array_equal(array[1, 2:, :], 0) numpy.testing.assert_array_equal(array[2, :4, :3], arrays[2]) numpy.testing.assert_array_equal(array[2, :, 3:], 0)
def test_concat_arrays_padding(self, backend_config): arrays = backend_config.get_array( [numpy.random.rand(3, 4), numpy.random.rand(2, 5), numpy.random.rand(4, 3)]) array = dataset.concat_examples(arrays, padding=0) self.assertEqual(array.shape, (3, 4, 5)) self.assertEqual(type(array), type(arrays[0])) arrays = [backend.CpuDevice().send(a) for a in arrays] array = backend.CpuDevice().send(array) numpy.testing.assert_array_equal(array[0, :3, :4], arrays[0]) numpy.testing.assert_array_equal(array[0, 3:, :], 0) numpy.testing.assert_array_equal(array[0, :, 4:], 0) numpy.testing.assert_array_equal(array[1, :2, :5], arrays[1]) numpy.testing.assert_array_equal(array[1, 2:, :], 0) numpy.testing.assert_array_equal(array[2, :4, :3], arrays[2]) numpy.testing.assert_array_equal(array[2, :, 3:], 0)
def test_concat_dicts_padding(self, backend_config): dicts = [ {'x': numpy.random.rand(3, 4), 'y': numpy.random.rand(2, 5)}, {'x': numpy.random.rand(4, 4), 'y': numpy.random.rand(3, 4)}, {'x': numpy.random.rand(2, 5), 'y': numpy.random.rand(2, 6)}, ] dicts = [ {key: backend_config.get_array(arr) for key, arr in d.items()} for d in dicts] arrays = dataset.concat_examples(dicts, padding=0) self.assertIn('x', arrays) self.assertIn('y', arrays) self.assertEqual(arrays['x'].shape, (3, 4, 5)) self.assertEqual(arrays['y'].shape, (3, 3, 6)) self.assertEqual(type(arrays['x']), type(dicts[0]['x'])) self.assertEqual(type(arrays['y']), type(dicts[0]['y'])) for d in dicts: d['x'] = backend.CpuDevice().send(d['x']) d['y'] = backend.CpuDevice().send(d['y']) arrays = { 'x': backend.CpuDevice().send(arrays['x']), 'y': backend.CpuDevice().send(arrays['y'])} numpy.testing.assert_array_equal(arrays['x'][0, :3, :4], dicts[0]['x']) numpy.testing.assert_array_equal(arrays['x'][0, 3:, :], 0) numpy.testing.assert_array_equal(arrays['x'][0, :, 4:], 0) numpy.testing.assert_array_equal(arrays['x'][1, :4, :4], dicts[1]['x']) numpy.testing.assert_array_equal(arrays['x'][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays['x'][2, :2, :5], dicts[2]['x']) numpy.testing.assert_array_equal(arrays['x'][2, 2:, :], 0) numpy.testing.assert_array_equal(arrays['y'][0, :2, :5], dicts[0]['y']) numpy.testing.assert_array_equal(arrays['y'][0, 2:, :], 0) numpy.testing.assert_array_equal(arrays['y'][0, :, 5:], 0) numpy.testing.assert_array_equal(arrays['y'][1, :3, :4], dicts[1]['y']) numpy.testing.assert_array_equal(arrays['y'][1, 3:, :], 0) numpy.testing.assert_array_equal(arrays['y'][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays['y'][2, :2, :6], dicts[2]['y']) numpy.testing.assert_array_equal(arrays['y'][2, 2:, :], 0)
def test_concat_tuples_padding(self, backend_config): tuples = [ backend_config.get_array( (numpy.random.rand(3, 4), numpy.random.rand(2, 5))), backend_config.get_array( (numpy.random.rand(4, 4), numpy.random.rand(3, 4))), backend_config.get_array( (numpy.random.rand(2, 5), numpy.random.rand(2, 6))), ] arrays = dataset.concat_examples(tuples, padding=0) self.assertEqual(len(arrays), 2) self.assertEqual(arrays[0].shape, (3, 4, 5)) self.assertEqual(arrays[1].shape, (3, 3, 6)) self.assertEqual(type(arrays[0]), type(tuples[0][0])) self.assertEqual(type(arrays[1]), type(tuples[0][1])) for i in range(len(tuples)): tuples[i] = ( backend.CpuDevice().send(tuples[i][0]), backend.CpuDevice().send(tuples[i][1])) arrays = tuple(backend.CpuDevice().send(array) for array in arrays) numpy.testing.assert_array_equal(arrays[0][0, :3, :4], tuples[0][0]) numpy.testing.assert_array_equal(arrays[0][0, 3:, :], 0) numpy.testing.assert_array_equal(arrays[0][0, :, 4:], 0) numpy.testing.assert_array_equal(arrays[0][1, :4, :4], tuples[1][0]) numpy.testing.assert_array_equal(arrays[0][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays[0][2, :2, :5], tuples[2][0]) numpy.testing.assert_array_equal(arrays[0][2, 2:, :], 0) numpy.testing.assert_array_equal(arrays[1][0, :2, :5], tuples[0][1]) numpy.testing.assert_array_equal(arrays[1][0, 2:, :], 0) numpy.testing.assert_array_equal(arrays[1][0, :, 5:], 0) numpy.testing.assert_array_equal(arrays[1][1, :3, :4], tuples[1][1]) numpy.testing.assert_array_equal(arrays[1][1, 3:, :], 0) numpy.testing.assert_array_equal(arrays[1][1, :, 4:], 0) numpy.testing.assert_array_equal(arrays[1][2, :2, :6], tuples[2][1]) numpy.testing.assert_array_equal(arrays[1][2, 2:, :], 0)
def accuracy(self): image_test, target_test = concat_examples(self.iteration, -1) prediction_test = self.model(image_test) accuracy = F.accuracy(prediction_test, target_test) return accuracy.data
def run(train_iter, val_iter, test_data, model, optimizer, max_epoch): training_losses = [] validation_losses = [] while train_iter.epoch < max_epoch: # Get next mini-batch batch = train_iter.next() image_train, target_train = concat_examples(batch) # Prediction prediction_train = model(image_train) # Compute loss loss = F.softmax_cross_entropy(prediction_train, target_train) # Compute gradients model.cleargrads() loss.backward() # Update variables optimizer.update() # Check the validation accuracy of prediction after every epoch if train_iter.is_new_epoch: # If this iteration is the final iteration of the current epoch # Display the training loss print('epoch:{:02d} train_loss:{:.04f} '.format( train_iter.epoch, float(loss.data))) training_losses.append(float(loss.data)) val_losses = [] val_accuracies = [] while True: val_batch = val_iter.next() image_val, target_val = concat_examples(val_batch) # Forward the validation data prediction_val = model(image_val) # Calculate the loss loss_val = F.softmax_cross_entropy(prediction_val, target_val) val_losses.append(loss_val.data) # Calculate the accuracy accuracy = F.accuracy(prediction_val, target_val) val_accuracies.append(accuracy.data) if val_iter.is_new_epoch: val_iter.epoch = 0 val_iter.current_position = 0 val_iter.is_new_epoch = False val_iter._pushed_position = None validation_losses.append(np.mean(val_losses)) break print('val_loss:{:.04f} val_accuracy:{:.04f}'.format( np.mean(val_losses), np.mean(val_accuracies))) # Predict full test set image_test, target_test = concat_examples(test_data) # Forward test data prediction_test = model(image_test) # Calculate loss and accuracy loss_test = F.softmax_cross_entropy(prediction_test, target_test) accuracy_test = F.accuracy(prediction_test, target_test) print('test_loss: ' + str(loss_test.data) + ' test_accuracy: ' + str(accuracy_test.data)) return training_losses, validation_losses
return self.l3(h2) net = MLP() from chainer import optimizers optimizer = optimizers.SGD(lr=0.01) optimizer.setup(net) from chainer.dataset import concat_examples max_epoch = 10 while train_iter.epoch < max_epoch: train_batch = train_iter.next() x, t = concat_examples(train_batch) y = net(x) loss = F.softmax_cross_entropy(y, t) net.cleargrads() loss.backward() optimizer.update() if train_iter.is_new_epoch: print('epoch:{:02d} train_loss:{:.04f} '.format( train_iter.epoch, float(loss.data)), end='') test_losses = [] test_accuracies = [] while True: test_batch = test_iter.next() x_test, t_test = concat_examples(test_batch)
def converter(batch, device): x, t = concat_examples(batch, device, 0) return x, t
train_iter = chainer.iterators.SerialIterator(train, batchsize) test_iter = chainer.iterators.SerialIterator(test, batchsize, repeat=False, shuffle=False) # Updater <- LSTM用にカスタマイズ #updater = training.StandardUpdater(train_iter, optimizer) # Trainerとそのextensions epoch = 3000 while train_iter.epoch < epoch: train_batch = train_iter.next() import pdb pdb.set_trace() x, t = concat_examples(train_batch) train = model(x, t) model, cleargrads() loss.backwrd() optimizer.update() #trainer = training.Trainer(updater, (epoch, 'epoch'), out='result') # 評価データで評価 #trainer.extend(extensions.Evaluator(test_iter, model,device = -1)) # 学習結果の途中を表示する #trainer.extend(extensions.LogReport(trigger=(1, 'epoch'))) # 1エポックごとに、trainデータに対するlossと、testデータに対するlossを出力させる #trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time']), trigger=(1, 'epoch'))
def main(): parser = argparse.ArgumentParser( description='Space Time Action Unit R-CNN training example:') parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/') parser.add_argument('--gpu', '-g', nargs='+', type=int, help='GPU ID, multiple GPU split by space') parser.add_argument('--lr', '-l', type=float, default=0.001) parser.add_argument('--out', '-o', default='end_to_end_result', help='Output directory') parser.add_argument('--database', default='BP4D', help='Output directory: BP4D/DISFA/BP4D_DISFA') parser.add_argument('--iteration', '-i', type=int, default=70000) parser.add_argument('--epoch', '-e', type=int, default=20) parser.add_argument('--batch_size', '-bs', type=int, default=1) parser.add_argument('--snapshot', '-snap', type=int, default=1000) parser.add_argument('--need_validate', action='store_true', help='do or not validate during training') parser.add_argument('--mean', default=config.ROOT_PATH + "BP4D/idx/mean_no_enhance.npy", help='image mean .npy file') parser.add_argument('--backbone', default="mobilenet_v1", help="vgg/resnet101/mobilenet_v1 for train") parser.add_argument('--optimizer', default='SGD', help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta') parser.add_argument('--pretrained_model_rgb', help='imagenet/mobilenet_v1/resnet101/*.npz') parser.add_argument( '--pretrained_model_of', help= "path of optical flow pretrained model (may be single stream OF model)" ) parser.add_argument('--pretrained_model_args', nargs='+', type=float, help='you can pass in "1.0 224" or "0.75 224"') parser.add_argument('--spatial_edge_mode', type=SpatialEdgeMode, choices=list(SpatialEdgeMode), help='1:all_edge, 2:configure_edge, 3:no_edge') parser.add_argument('--spatial_sequence_type', type=SpatialSequenceType, choices=list(SpatialSequenceType), help='1:all_edge, 2:configure_edge, 3:no_edge') parser.add_argument( '--temporal_edge_mode', type=TemporalEdgeMode, choices=list(TemporalEdgeMode), help='1:rnn, 2:attention_block, 3.point-wise feed forward(no temporal)' ) parser.add_argument('--two_stream_mode', type=TwoStreamMode, choices=list(TwoStreamMode), help='spatial/ temporal/ spatial_temporal') parser.add_argument('--conv_rnn_type', type=ConvRNNType, choices=list(ConvRNNType), help='conv_lstm or conv_sru') parser.add_argument("--bi_lstm", action="store_true", help="whether to use bi-lstm as Edge/Node RNN") parser.add_argument( '--use_memcached', action='store_true', help='whether use memcached to boost speed of fetch crop&mask') # parser.add_argument('--memcached_host', default='127.0.0.1') parser.add_argument("--fold", '-fd', type=int, default=3) parser.add_argument("--layers", type=int, default=1) parser.add_argument("--label_win_size", type=int, default=3) parser.add_argument("--fix", action="store_true", help="fix parameter of conv2 update when finetune") parser.add_argument("--x_win_size", type=int, default=1) parser.add_argument("--use_label_dependency", action="store_true", help="use label dependency layer after conv_lstm") parser.add_argument("--dynamic_backbone", action="store_true", help="use dynamic backbone: conv lstm as backbone") parser.add_argument("--ld_rnn_dropout", type=float, default=0.4) parser.add_argument("--split_idx", '-sp', type=int, default=1) parser.add_argument("--use_paper_num_label", action="store_true", help="only to use paper reported number of labels" " to train") parser.add_argument( "--roi_align", action="store_true", help="whether to use roi align or roi pooling layer in CNN") parser.add_argument("--debug", action="store_true", help="debug mode for 1/50 dataset") parser.add_argument("--sample_frame", '-sample', type=int, default=10) parser.add_argument( "--snap_individual", action="store_true", help="whether to snapshot each individual epoch/iteration") parser.add_argument("--proc_num", "-proc", type=int, default=1) parser.add_argument("--fetch_mode", type=int, default=1) parser.add_argument('--eval_mode', action='store_true', help='Use test datasets for evaluation metric') args = parser.parse_args() os.makedirs(args.pid, exist_ok=True) os.makedirs(args.out, exist_ok=True) pid = str(os.getpid()) pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format( args.database, args.fold, args.split_idx) # with open(pid_file_path, "w") as file_obj: # file_obj.write(pid) # file_obj.flush() print('GPU: {}'.format(",".join(list(map(str, args.gpu))))) adaptive_AU_database(args.database) mc_manager = None if args.use_memcached: from collections_toolkit.memcached_manager import PyLibmcManager mc_manager = PyLibmcManager(args.memcached_host) if mc_manager is None: raise IOError("no memcached found listen in {}".format( args.memcached_host)) paper_report_label, class_num = squeeze_label_num_report( args.database, args.use_paper_num_label) paper_report_label_idx = list(paper_report_label.keys()) use_feature_map_res45 = (args.conv_rnn_type != ConvRNNType.conv_rcnn) and ( args.conv_rnn_type != ConvRNNType.fc_lstm) use_au_rcnn_loss = (args.conv_rnn_type == ConvRNNType.conv_rcnn) au_rcnn_train_chain_list = [] if args.backbone == 'vgg': au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], mean_file=args.mean, use_roi_align=args.roi_align) au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn) au_rcnn_train_chain_list.append(au_rcnn_train_chain) elif args.backbone == 'resnet101': if args.two_stream_mode != TwoStreamMode.spatial_temporal: pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_of au_rcnn = AU_RCNN_Resnet101( pretrained_model=pretrained_model, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], mean_file=args.mean, classify_mode=use_au_rcnn_loss, n_class=class_num, use_roi_align=args.roi_align, use_feature_map_res45=use_feature_map_res45, use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm or args.conv_rnn_type == ConvRNNType.sep_conv_lstm), use_optical_flow_input=( args.two_stream_mode == TwoStreamMode.optical_flow), temporal_length=args.sample_frame) au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn) au_rcnn_train_chain_list.append(au_rcnn_train_chain) else: au_rcnn_rgb = AU_RCNN_Resnet101( pretrained_model=args.pretrained_model_rgb, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], mean_file=args.mean, classify_mode=use_au_rcnn_loss, n_class=class_num, use_roi_align=args.roi_align, use_feature_map_res45=use_feature_map_res45, use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm or args.conv_rnn_type == ConvRNNType.sep_conv_lstm), use_optical_flow_input=False, temporal_length=args.sample_frame) au_rcnn_optical_flow = AU_RCNN_Resnet101( pretrained_model=args.pretrained_model_of, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], mean_file=args.mean, classify_mode=use_au_rcnn_loss, n_class=class_num, use_roi_align=args.roi_align, use_feature_map_res45=use_feature_map_res45, use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm or args.conv_rnn_type == ConvRNNType.sep_conv_lstm), use_optical_flow_input=True, temporal_length=args.sample_frame) au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb) au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor( au_rcnn_optical_flow) au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb) au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow) elif args.backbone == "mobilenet_v1": au_rcnn = AU_RCNN_MobilenetV1( pretrained_model_type=args.pretrained_model_args, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], mean_file=args.mean, classify_mode=use_au_rcnn_loss, n_class=class_num, use_roi_align=args.roi_align) au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn) if use_au_rcnn_loss: au_rcnn_train_loss = AU_RCNN_TrainChainLoss() loss_head_module = au_rcnn_train_loss elif args.conv_rnn_type == ConvRNNType.conv_lstm: label_dependency_layer = None if args.use_label_dependency: label_dependency_layer = LabelDependencyRNNLayer( args.database, in_size=2048, class_num=class_num, train_mode=True, label_win_size=args.label_win_size) space_time_conv_lstm = SpaceTimeConv( label_dependency_layer, args.use_label_dependency, class_num, spatial_edge_mode=args.spatial_edge_mode, temporal_edge_mode=args.temporal_edge_mode, conv_rnn_type=args.conv_rnn_type) loss_head_module = space_time_conv_lstm elif args.conv_rnn_type == ConvRNNType.sep_conv_lstm: space_time_sep_conv_lstm = SpaceTimeSepConv( database=args.database, class_num=class_num, spatial_edge_mode=args.spatial_edge_mode, temporal_edge_mode=args.temporal_edge_mode) loss_head_module = space_time_sep_conv_lstm elif args.conv_rnn_type == ConvRNNType.fc_lstm: space_time_fc_lstm = SpaceTimeSepFcLSTM( database=args.database, class_num=class_num, spatial_edge_mode=args.spatial_edge_mode, temporal_edge_mode=args.temporal_edge_mode) loss_head_module = space_time_fc_lstm model = Wrapper(au_rcnn_train_chain_list, loss_head_module, args.database, args.sample_frame, use_feature_map=use_feature_map_res45, two_stream_mode=args.two_stream_mode) batch_size = args.batch_size img_dataset = AUDataset(database=args.database, fold=args.fold, split_name='trainval', split_index=args.split_idx, mc_manager=mc_manager, train_all_data=False) train_video_data = AU_video_dataset( au_image_dataset=img_dataset, sample_frame=args.sample_frame, train_mode=(args.two_stream_mode != TwoStreamMode.optical_flow), paper_report_label_idx=paper_report_label_idx, ) Transform = Transform3D train_video_data = TransformDataset(train_video_data, Transform(au_rcnn, mirror=False)) if args.proc_num == 1: train_iter = SerialIterator(train_video_data, batch_size * args.sample_frame, repeat=True, shuffle=False) else: train_iter = MultiprocessIterator(train_video_data, batch_size=batch_size * args.sample_frame, n_processes=args.proc_num, repeat=True, shuffle=False, n_prefetch=10, shared_mem=10000000) if len(args.gpu) > 1: for gpu in args.gpu: chainer.cuda.get_device_from_id(gpu).use() else: chainer.cuda.get_device_from_id(args.gpu[0]).use() model.to_gpu(args.gpu[0]) optimizer = None if args.optimizer == 'AdaGrad': optimizer = chainer.optimizers.AdaGrad( lr=args.lr ) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题,改为AdaGrad elif args.optimizer == 'RMSprop': optimizer = chainer.optimizers.RMSprop(lr=args.lr) elif args.optimizer == 'Adam': optimizer = chainer.optimizers.Adam(alpha=args.lr) elif args.optimizer == 'SGD': optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) elif args.optimizer == "AdaDelta": optimizer = chainer.optimizers.AdaDelta() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) optimizer_name = args.optimizer key_str = "{0}_fold_{1}".format(args.fold, args.split_idx) file_list = [] file_list.extend(os.listdir(args.out)) snapshot_model_file_name = args.out + os.sep + filter_last_checkpoint_filename( file_list, "model", key_str) # BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label" roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling" label_dependency_layer_key_str = "label_dep_layer" if args.use_label_dependency else "no_label_dep" single_model_file_name = args.out + os.sep + \ '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_model.npz'.format(args.database, args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, use_paper_key_str, roi_align_key_str, label_dependency_layer_key_str, args.conv_rnn_type,args.sample_frame )#, args.label_win_size) print(single_model_file_name) pretrained_optimizer_file_name = args.out + os.sep +\ '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_optimizer.npz'.format(args.database, args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, use_paper_key_str, roi_align_key_str, label_dependency_layer_key_str, args.conv_rnn_type, args.sample_frame)# args.label_win_size) print(pretrained_optimizer_file_name) if os.path.exists(pretrained_optimizer_file_name): print("loading optimizer snatshot:{}".format( pretrained_optimizer_file_name)) chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer) if args.snap_individual: if os.path.exists(snapshot_model_file_name) and os.path.isfile( snapshot_model_file_name): print("loading pretrained snapshot:{}".format( snapshot_model_file_name)) chainer.serializers.load_npz(snapshot_model_file_name, model) else: if os.path.exists(single_model_file_name): print("loading pretrained snapshot:{}".format( single_model_file_name)) chainer.serializers.load_npz(single_model_file_name, model) if args.fix: au_rcnn = model.au_rcnn_train_chain.au_rcnn au_rcnn.extractor.conv1.W.update_rule.enabled = False au_rcnn.extractor.bn1.gamma.update_rule.enabled = False au_rcnn.extractor.bn1.beta.update_rule.enabled = False res2_names = ["a", "b1", "b2"] for res2_name in res2_names: if res2_name == "a": getattr(au_rcnn.extractor.res2, res2_name).conv1.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv2.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv3.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv4.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn4.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn4.beta.update_rule.enabled = False elif res2_name.startswith("b"): getattr(au_rcnn.extractor.res2, res2_name).conv1.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv2.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv3.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.beta.update_rule.enabled = False # if (args.spatial_edge_mode in [SpatialEdgeMode.ld_rnn, SpatialEdgeMode.bi_ld_rnn] or args.temporal_edge_mode in \ # [TemporalEdgeMode.ld_rnn, TemporalEdgeMode.bi_ld_rnn]) or (args.conv_rnn_type != ConvRNNType.conv_rcnn): # updater = BPTTUpdater(train_iter, optimizer, converter=lambda batch, device: concat_examples(batch, device, # padding=0), device=args.gpu[0]) if len(args.gpu) > 1: gpu_dict = {"main": args.gpu[0]} # many gpu will use parallel_models = {"parallel": model.au_rcnn_train_chain} for slave_gpu in args.gpu[1:]: gpu_dict[slave_gpu] = int(slave_gpu) updater = PartialParallelUpdater( train_iter, optimizer, args.database, models=parallel_models, devices=gpu_dict, converter=lambda batch, device: concat_examples( batch, device, padding=0)) else: print("only one GPU({0}) updater".format(args.gpu[0])) updater = chainer.training.StandardUpdater( train_iter, optimizer, device=args.gpu[0], converter=lambda batch, device: concat_examples( batch, device, padding=0)) @training.make_extension(trigger=(1, "epoch")) def reset_order(trainer): print("reset dataset order after one epoch") if args.debug: trainer.updater._iterators[ "main"].dataset._dataset.reset_for_debug_mode() else: trainer.updater._iterators[ "main"].dataset._dataset.reset_for_train_mode() trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(reset_order) trainer.extend(chainer.training.extensions.snapshot_object( optimizer, filename=os.path.basename(pretrained_optimizer_file_name)), trigger=(args.snapshot, 'iteration')) if not args.snap_individual: trainer.extend(chainer.training.extensions.snapshot_object( model, filename=os.path.basename(single_model_file_name)), trigger=(args.snapshot, 'iteration')) else: snap_model_file_name = '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}sampleframe#{10}@win#{11}_'.format( args.database, args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, use_paper_key_str, roi_align_key_str, label_dependency_layer_key_str, args.conv_rnn_type, args.sample_frame, args.label_win_size) snap_model_file_name = snap_model_file_name + "{.updater.iteration}.npz" trainer.extend(chainer.training.extensions.snapshot_object( model, filename=snap_model_file_name), trigger=(args.snapshot, 'iteration')) log_interval = 100, 'iteration' print_interval = 10, 'iteration' plot_interval = 10, 'iteration' if args.optimizer != "Adam" and args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1), trigger=(10, 'epoch')) elif args.optimizer == "Adam": trainer.extend(chainer.training.extensions.ExponentialShift( "alpha", 0.1, optimizer=optimizer), trigger=(10, 'epoch')) if args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend( chainer.training.extensions.LogReport( trigger=log_interval, log_name="log_{0}_fold_{1}_{2}@{3}@{4}@{5}.log".format( args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, args.conv_rnn_type))) # trainer.reporter.add_observer("main_par", model.loss_head_module) trainer.extend(chainer.training.extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/accuracy', ]), trigger=print_interval) trainer.extend( chainer.training.extensions.ProgressBar(update_interval=100)) if chainer.training.extensions.PlotReport.available(): trainer.extend(chainer.training.extensions.PlotReport( ['main/loss'], file_name='loss_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format( args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, args.conv_rnn_type), trigger=plot_interval), trigger=plot_interval) trainer.extend(chainer.training.extensions.PlotReport( ['main/accuracy'], file_name='accuracy_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format( args.fold, args.split_idx, args.backbone, args.spatial_edge_mode, args.temporal_edge_mode, args.conv_rnn_type), trigger=plot_interval), trigger=plot_interval) trainer.run()
def load_mnist(): train, test = get_mnist(ndim=3) train = concat_examples(train) test = concat_examples(test) return train, test
def run(self): loss_names = ['epoch'] loss_names.append('imitation_loss') if self.learn_dx: loss_names.append('sysid_loss') fname = os.path.join(self.save, 'train_losses.csv') train_loss_f = open(fname, 'w') train_loss_f.write('{}\n'.format(','.join(loss_names))) train_loss_f.flush() fname = os.path.join(self.save, 'val_test_losses.csv') vt_loss_f = open(fname, 'w') loss_names = ['epoch'] loss_names += ['im_loss_val', 'im_loss_test'] vt_loss_f.write('{}\n'.format(','.join(loss_names))) vt_loss_f.flush() if self.learn_cost: fname = os.path.join(self.save, 'cost_hist.csv') cost_f = open(fname, 'w') # first write answer cost_f.write(','.join( map(str, xp.concatenate((self.true_q, self.true_p)).tolist()))) cost_f.write('\n') cost_f.flush() opt = chainer.optimizers.RMSprop(lr=1e-2, alpha=0.5) opt.setup(self.net) train_warm_start = xp.zeros((self.n_train, self.T, self.n_ctrl)) val_warm_start = xp.zeros( (self.env.val_data.shape[0], self.T, self.n_ctrl)) test_warm_start = xp.zeros( (self.env.test_data.shape[0], self.T, self.n_ctrl)) train_data, train_iter = self.make_data( self.env.train_data[:self.n_train], shuffle=True) val_data, val_iter = self.make_data(self.env.val_data) test_data, test_iter = self.make_data(self.env.test_data) best_val_loss = None train_warmstart = xp.zeros((self.n_train, self.T, self.n_ctrl)) val_warmstart = xp.zeros( (self.env.val_data.shape[0], self.T, self.n_ctrl)) test_warmstart = xp.zeros( (self.env.test_data.shape[0], self.T, self.n_ctrl)) true_q, true_p = self.env.true_dx.get_true_obj() cost_update_q = False learn_cost_round_robin_interval = 10 while train_iter.epoch < self.n_epoch: print("triain_iter.epoch", train_iter.epoch_detail) if train_iter.epoch > 0 and train_iter.epoch % learn_cost_round_robin_interval == 0: cost_update_q = not cost_update_q self.net.cleargrads() if train_iter.epoch % self.restart_warmstart_every == 0: train_warm_start = xp.zeros_like(train_warm_start) val_warm_start = xp.zeros_like(val_warm_start) test_warm_start = xp.zeros_like(test_warm_start) next_batch = train_iter.next() next_batch = dataset.concat_examples(next_batch) xinits = next_batch[0] xs = next_batch[1] us = next_batch[2] idxs = next_batch[3] if self.learn_dx: assert False else: dx = self.env.true_dx nom_x, nom_u = self.net(xinits, self.env, train_warm_start[idxs]) nom_u = F.transpose(nom_u, axes=(1, 0, 2)) ''' g = c.build_computational_graph([nom_u], remove_variable=True) with open('graph.dot', 'w') as o: o.write(g.dump()) assert False ''' train_warmstart[idxs] = nom_u.array assert type(us) != chainer.Variable assert nom_u.shape == us.shape squared_loss_u = (us - nom_u) * (us - nom_u) im_loss_u = F.mean(squared_loss_u) # print("im_loss shape", im_loss_u.shape) nom_x = F.transpose(nom_x, axes=(1, 0, 2)) assert xs.shape == nom_x.shape, str(xs.shape) + " " + str( nom_x.shape) # squared_loss_x = (xs - nom_x) * (xs - nom_x) # im_loss_x = F.mean(squared_loss_x) # print("im_loss-x", im_loss_x.shape) # im_loss = im_loss_x + im_loss_u im_loss = im_loss_u t = [train_iter.epoch_detail, im_loss.array] t = ','.join(map(str, t)) train_loss_f.write(t + '\n') train_loss_f.flush() print("train imtation loss", im_loss.array) im_loss.backward() if cost_update_q: print('only updating q') self.net.learn_p.update_rule.enabled = False self.net.learn_q_logit.update_rule.enabled = True if self.is_lower_triangle: self.net.lower_without_diag.update_rule.enabled = True else: print('only updating p') self.net.learn_q_logit.update_rule.enabled = False self.net.learn_p.update_rule.enabled = True if self.is_lower_triangle: self.net.lower_without_diag.update_rule.enabled = False ''' if self.learn_cost: true_cat = F.concat((true_q, true_p), axis=0) _q = F.sigmoid(self.net.learn_q_logit).array _p = xp.sqrt(_q) * self.net.learn_p print("learn_q_logit", self.net.learn_q_logit) print("learn_p", self.net.learn_p) qp_cat = F.concat((_q, _p), axis=0) print(xp.array_str(F.stack((true_cat, qp_cat)).array, precision=5, suppress_small=True)) cost_f.write(','.join(map(str, qp_cat.array))) cost_f.write('\n') cost_f.flush() ''' opt.update() if train_iter.is_new_epoch: val_loss = self.dataset_loss(val_iter, val_warmstart) test_loss = self.dataset_loss(test_iter, test_warmstart) t = [train_iter.epoch, val_loss, test_loss] t = ','.join(map(str, t)) vt_loss_f.write(t + '\n') vt_loss_f.flush() if best_val_loss is None or val_loss < best_val_loss: best_val_loss = val_loss fname = os.path.join(self.save, 'best.pkl') print('Saving best model to {}'.format(fname)) with open(fname, 'wb') as f: pkl.dump(self, f)
class MLP(Chain): def __init__(self, n_units, n_out): super(MLP, self).__init__() with self.init_scope(): self.l1 = L.Linear(None, n_units) self.l2 = L.Linear(None, n_units) self.l3 = L.Linear(None, n_out) def __call__(self, x): h1 = F.relu(self.l1(x)) h2 = F.relu(self.l2(h1)) y = self.l3(h2) return y model = L.Classifier(MLP(100, 10)) optimizer = optimizers.SGD() optimizer.setup(model) model.to_gpu() # here! batchsize = 100 x_train, y_train = concat_examples(train) datasize = len(x_train) for epoch in range(20): print('epoch: %d' % epoch) indexes = np.random.permutation(datasize) for i in range(0, datasize, batchsize): x = Variable(cuda.to_gpu(x_train[indexes[i:i + batchsize]])) # here! t = Variable(cuda.to_gpu(y_train[indexes[i:i + batchsize]])) # here! optimizer.update(model, x, t)
def dataset_loss(self, data_iter, warmstart=None): """ calculate loss and set warm start this loss is :param loader: :param warmstart: :return: """ true_q, true_p = self.env.true_dx.get_true_obj() losses = [] iter_before = data_iter.epoch while data_iter.epoch < iter_before + 1: next_batch = data_iter.next() next_batch = dataset.concat_examples(next_batch) x_inits = next_batch[0] xs = next_batch[1] us = next_batch[2] idxs = next_batch[3] n_batch = x_inits.shape[0] dx = self.env.true_dx if not self.is_lower_triangle and not self.is_strange_observation: print("not lower triangle and not strange observation") assert type(self.net) == Pendulum_Net_cost_logit q = F.sigmoid(self.net.learn_q_logit) p = F.sqrt(q) * self.net.learn_p _, pred_u = self.env.mpc(self.env.true_dx, x_inits, q, p, u_init=xp.transpose(warmstart[idxs], axes=(1, 0, 2))) pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array warmstart[idxs] = pred_u elif not self.is_strange_observation: print("lower triangle and not strange observation") assert type(self.net) == Pendulum_Net_cost_lower_triangle Q = xp.zeros((self.n_sc, self.n_sc)) index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool) xp.fill_diagonal(index_diag, True) index_not_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool) index_not_diag[xp.tril_indices(self.n_sc, -1)] = True Q = F.scatter_add(Q, xp.tril_indices(self.n_sc, -1), self.net.lower_without_diag) # index_not_diag = xp.zeros((self.n_state, self.n_state), dtype=bool) # index_not_diag[xp.tril_indices(self.n_state, -1)] = True # Q = F.scatter_add(Q, xp.tril_indices(self.n_state, -1), self.net.lower_without_diag) diag_q = F.sigmoid(self.net.learn_q_logit) Q = F.where(index_diag, diag_q, Q) Q = Q @ Q.T p = self.net.learn_p p = F.concat((p, xp.array(0.0).reshape(1, )), axis=0) _, pred_u = self.env.mpc_Q(self.env.true_dx, x_inits, Q, p, u_init=xp.transpose(warmstart[idxs], axes=(1, 0, 2))) pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array warmstart[idxs] = pred_u elif not self.is_lower_triangle: print("not lower triangle and strange observation") assert type( self.net) == Pendulum_Net_cost_logit_strange_obervation q = F.sigmoid(self.net.learn_q_logit) p = F.sqrt(q) * self.net.learn_p # p = self.learn_p Q = xp.zeros((self.n_sc, self.n_sc)) index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool) xp.fill_diagonal(index_diag, True) Q = F.where(index_diag, q, Q) Q = OBSERVATION_MATRIX.T @ Q @ OBSERVATION_MATRIX p = p @ OBSERVATION_MATRIX _, pred_u = self.env.mpc_Q(self.env.true_dx, x_inits, Q, p, u_init=xp.transpose(warmstart[idxs], axes=(1, 0, 2))) pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array warmstart[idxs] = pred_u else: print("lower triangle and strange observation") assert type( self.net ) == Pendulum_Net_cost_lower_triangle_strange_obervation Q = xp.zeros((self.n_sc, self.n_sc)) index_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool) xp.fill_diagonal(index_diag, True) index_not_diag = xp.zeros((self.n_sc, self.n_sc), dtype=bool) index_not_diag[xp.tril_indices(self.n_sc, -1)] = True Q = F.scatter_add(Q, xp.tril_indices(self.n_sc, -1), self.net.lower_without_diag) diag_q = F.sigmoid(self.net.learn_q_logit) Q = F.where(index_diag, diag_q, Q) Q = Q @ Q.T Q = OBSERVATION_MATRIX.T @ Q @ OBSERVATION_MATRIX p = self.net.learn_p p = p @ OBSERVATION_MATRIX _, pred_u = self.env.mpc_Q(self.env.true_dx, x_inits, Q, p, u_init=xp.transpose(warmstart[idxs], axes=(1, 0, 2))) pred_u = F.transpose(pred_u, axes=(1, 0, 2)).array warmstart[idxs] = pred_u assert pred_u.shape == us.shape squared_loss = (us - pred_u) * (us - pred_u) # print(squared_loss.shape) loss = xp.mean(squared_loss) losses.append(loss) loss = xp.stack(losses).mean() return loss
net = myNetwork.UNET_02() if gpu_id >= 0: net.to_gpu(gpu_id) optimizer = optimizers.Adam().setup(net) #-------------------- # Learning iteration #-------------------- os.makedirs(savefile + "/models/") while train_iter.epoch < max_epoch: train_batch = train_iter.next() x, t = concat_examples(train_batch, gpu_id) y = net(x) loss = F.mean_squared_error(y, t) #loss = myLossfun.l1l2_norm_error(y, t, lam=3) net.cleargrads() loss.backward() optimizer.update() # Check if train_iter.is_new_epoch: print('epoch:{:02d} train_loss:{:.04f} '.format( train_iter.epoch, float(to_cpu(loss.data))),
fig,ax = plt.subplots(nrows=6,ncols=8,sharex=True,sharey=True) ax = ax.flatten() for i in range(48): img = xtrain[i].reshape(28,28) ax[i].imshow(img,cmap='Greys',interpolation='none') ''' """<h3>ニューラルネットワークの学習を行います。GPUを用いて高速に学習ができます。</h3>""" max_epoch = args_epoch while train_iter.epoch < max_epoch: train_accuracies = [] # ---------- One iteration of the training loop ---------- train_batch = train_iter.next() image_train, target_train = concat_examples(train_batch, args_gpu) # Calculate the prediction of the network prediction_train = model(image_train) # Calculate the loss with softmax_cross_entropy loss = F.softmax_cross_entropy(prediction_train, target_train) accuracy = F.accuracy(prediction_train, target_train) accuracy.to_cpu() train_accuracies.append(accuracy.data) # Calculate the gradients in the network model.cleargrads() loss.backward() # Update all the trainable parameters optimizer.update()
def train(iterator, gpu, encoder, decoder, enc_optim, dec_optim, rel_send, rel_rec, edge_types, temp, prediction_steps, var, out, benchmark, lr_decay, gamma): iter_i = 0 edge_accuracies = [] node_mses = [] nll_train = [] kl_train = [] logger = logging.getLogger(__name__) while True: inputs = iterator.next() node_features, edge_labels = dataset.concat_examples(inputs, device=gpu) # logits: [batch_size, num_edges, edge_types] logits = encoder(node_features, rel_send, rel_rec) # inverse func. of softmax edges = F.gumbel_softmax(logits, tau=temp, axis=2) edge_probs = F.softmax(logits, axis=2) # edges, edge_probs: [batch_size, num_edges, edge_types] if isinstance(decoder, decoders.MLPDecoder): output = decoder(node_features, edges, rel_rec, rel_send, prediction_steps) elif isinstance(decoder, decoders.RNNDecoder): output = decoder(node_features, edges, rel_rec, rel_send, 100, burn_in=True, burn_in_steps=args.timesteps - args.prediction_steps) target = node_features[:, :, 1:, :] num_nodes = node_features.shape[1] loss_nll = get_nll_gaussian(output, target, var) loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types) loss = loss_nll + loss_kl nll_train.append(float(loss_nll.array)) kl_train.append(float(loss_kl.array)) edge_accuracy = get_edge_accuracy(logits.array, edge_labels) edge_accuracies.append(edge_accuracy) node_mse = float(F.mean_squared_error(output, target).array) node_mses.append(node_mse) encoder.cleargrads() decoder.cleargrads() loss.backward() enc_optim.update() dec_optim.update() # Exit after 10 iterations when benchmark mode is ON iter_i += 1 if benchmark: put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses)) if iter_i == 10: exit() if iterator.is_new_epoch: break if not os.path.exists(os.path.join(out, 'graph.dot')): with open(os.path.join(out, 'graph.dot'), 'w') as o: g = computational_graph.build_computational_graph([loss]) o.write(g.dump()) if iterator.is_new_epoch: put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses)) serializers.save_npz( os.path.join(out, 'encoder_epoch-{}.npz'.format(iterator.epoch)), encoder) serializers.save_npz( os.path.join(out, 'decoder_epoch-{}.npz'.format(iterator.epoch)), decoder) serializers.save_npz( os.path.join(out, 'enc_optim_epoch-{}.npz'.format(iterator.epoch)), enc_optim) serializers.save_npz( os.path.join(out, 'dec_optim_epoch-{}.npz'.format(iterator.epoch)), dec_optim) if iterator.epoch % lr_decay == 0: enc_optim.alpha *= gamma dec_optim.alpha *= gamma logger.info('alpha of enc_optim: {}'.format(enc_optim.alpha)) logger.info('alpha of dec_optim: {}'.format(dec_optim.alpha))
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epochs', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--output_dir', '-o', default='./outputs', help='Directory to output the result') parser.add_argument( '--gpu_id', '-g', default=0, help='ID of the GPU to be used. Set to -1 if you use CPU') args = parser.parse_args() # Download the MNIST data if you haven't downloaded it yet train, test = datasets.mnist.get_mnist(withlabel=True, ndim=1) gpu_id = args.gpu_id batchsize = args.batchsize epochs = args.epochs run.log('Batch size', np.int(batchsize)) run.log('Epochs', np.int(epochs)) train_iter = iterators.SerialIterator(train, batchsize) test_iter = iterators.SerialIterator(test, batchsize, repeat=False, shuffle=False) model = MyNetwork() if gpu_id >= 0: # Make a specified GPU current chainer.backends.cuda.get_device_from_id(0).use() model.to_gpu() # Copy the model to the GPU # Choose an optimizer algorithm optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9) # Give the optimizer a reference to the model so that it # can locate the model's parameters. optimizer.setup(model) while train_iter.epoch < epochs: # ---------- One iteration of the training loop ---------- train_batch = train_iter.next() image_train, target_train = concat_examples(train_batch, gpu_id) # Calculate the prediction of the network prediction_train = model(image_train) # Calculate the loss with softmax_cross_entropy loss = F.softmax_cross_entropy(prediction_train, target_train) # Calculate the gradients in the network model.cleargrads() loss.backward() # Update all the trainable parameters optimizer.update() # --------------------- until here --------------------- # Check the validation accuracy of prediction after every epoch if train_iter.is_new_epoch: # If this iteration is the final iteration of the current epoch # Display the training loss print('epoch:{:02d} train_loss:{:.04f} '.format( train_iter.epoch, float(to_cpu(loss.array))), end='') test_losses = [] test_accuracies = [] while True: test_batch = test_iter.next() image_test, target_test = concat_examples(test_batch, gpu_id) # Forward the test data prediction_test = model(image_test) # Calculate the loss loss_test = F.softmax_cross_entropy(prediction_test, target_test) test_losses.append(to_cpu(loss_test.array)) # Calculate the accuracy accuracy = F.accuracy(prediction_test, target_test) accuracy.to_cpu() test_accuracies.append(accuracy.array) if test_iter.is_new_epoch: test_iter.epoch = 0 test_iter.current_position = 0 test_iter.is_new_epoch = False test_iter._pushed_position = None break val_accuracy = np.mean(test_accuracies) print('val_loss:{:.04f} val_accuracy:{:.04f}'.format( np.mean(test_losses), val_accuracy)) run.log("Accuracy", np.float(val_accuracy))
def main(): print("chainer cudnn enabled: {}".format(chainer.cuda.cudnn_enabled)) parser = argparse.ArgumentParser( description='Action Unit Intensity R-CNN training example:') parser.add_argument('--pid', '-pp', default='/tmp/AU_R_CNN/') parser.add_argument('--gpu', '-g', default="0", help='GPU ID, multiple GPU split by comma') parser.add_argument('--lr', '-l', type=float, default=0.001) parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--database', default='BP4D', help='Output directory: BP4D/DISFA/BP4D_DISFA') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--iteration', '-i', type=int, default=70000) parser.add_argument('--epoch', '-e', type=int, default=20) parser.add_argument('--batch_size', '-bs', type=int, default=20) parser.add_argument('--snapshot', '-snap', type=int, default=1000) parser.add_argument('--mean', default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy", help='image mean .npy file') parser.add_argument('--feature_model', default="resnet101", help="vgg or resnet101 for train") parser.add_argument('--extract_len', type=int, default=1000) parser.add_argument('--optimizer', default='RMSprop', help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta') parser.add_argument('--pretrained_model', default='resnet101', help='imagenet/vggface/resnet101/*.npz') parser.add_argument( '--use_memcached', action='store_true', help='whether use memcached to boost speed of fetch crop&mask') # parser.add_argument('--memcached_host', default='127.0.0.1') parser.add_argument("--fold", '-fd', type=int, default=3) parser.add_argument("--split_idx", '-sp', type=int, default=1) parser.add_argument("--proc_num", "-proc", type=int, default=1) parser.add_argument( "--is_pretrained", action="store_true", help="whether is to pretrain BP4D later will for DISFA dataset or not") parser.add_argument( "--pretrained_target", '-pt', default="", help="whether pretrain label set will use DISFA or not") parser.add_argument("--prefix", '-prefix', default="", help="_beta, for example 3_fold_beta") parser.add_argument('--eval_mode', action='store_true', help='Use test datasets for evaluation metric') parser.add_argument("--img_resolution", type=int, default=512) args = parser.parse_args() if not os.path.exists(args.pid): os.makedirs(args.pid) pid = str(os.getpid()) pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format( args.database, args.fold, args.split_idx) with open(pid_file_path, "w") as file_obj: file_obj.write(pid) file_obj.flush() config.IMG_SIZE = (args.img_resolution, args.img_resolution) print('GPU: {}'.format(args.gpu)) if args.is_pretrained: adaptive_AU_database(args.pretrained_target) else: adaptive_AU_database(args.database) np.random.seed(args.seed) # 需要先构造一个list的txt文件:id_trainval_0.txt, 每一行是subject + "/" + emotion_seq + "/" frame mc_manager = None if args.use_memcached: from collections_toolkit.memcached_manager import PyLibmcManager mc_manager = PyLibmcManager(args.memcached_host) if mc_manager is None: raise IOError("no memcached found listen in {}".format( args.memcached_host)) if args.feature_model == 'vgg': faster_rcnn = FasterRCNNVGG16( n_fg_class=len(config.AU_INTENSITY_DICT), pretrained_model=args.pretrained_model, mean_file=args.mean, min_size=args.img_resolution, max_size=args.img_resolution, extract_len=args.extract_len ) # 可改为/home/nco/face_expr/result/snapshot_model.npz elif args.feature_model == 'resnet101': faster_rcnn = FasterRCNNResnet101( n_fg_class=len(config.AU_INTENSITY_DICT), pretrained_model=args.pretrained_model, mean_file=args.mean, min_size=args.img_resolution, max_size=args.img_resolution, extract_len=args.extract_len ) # 可改为/home/nco/face_expr/result/snapshot_model.npz if args.eval_mode: with chainer.no_backprop_mode(), chainer.using_config("train", False): test_data = AUDataset(database=args.database, fold=args.fold, img_resolution=args.img_resolution, split_name='test', split_index=args.split_idx, mc_manager=mc_manager, prefix=args.prefix, pretrained_target=args.pretrained_target) test_data = TransformDataset(test_data, Transform(faster_rcnn, mirror=False)) if args.proc_num == 1: test_iter = SerialIterator(test_data, 1, repeat=False, shuffle=True) else: test_iter = MultiprocessIterator(test_data, batch_size=1, n_processes=args.proc_num, repeat=False, shuffle=True, n_prefetch=10, shared_mem=10000000) gpu = int(args.gpu) if "," not in args.gpu else int( args.gpu[:args.gpu.index(",")]) chainer.cuda.get_device_from_id(gpu).use() faster_rcnn.to_gpu(gpu) evaluator = AUEvaluator( test_iter, faster_rcnn, lambda batch, device: concat_examples_not_none( batch, device, padding=-99), args.database, "/home/machen/face_expr", device=gpu) observation = evaluator.evaluate() with open( args.out + os.path.sep + "evaluation_intensity_split_{}_result_test_mode.json". format(args.split_idx), "w") as file_obj: file_obj.write( json.dumps(observation, indent=4, separators=(',', ': '))) file_obj.flush() return train_data = AUDataset(database=args.database, img_resolution=args.img_resolution, fold=args.fold, split_name='trainval', split_index=args.split_idx, mc_manager=mc_manager, prefix=args.prefix, pretrained_target=args.pretrained_target) train_data = TransformDataset(train_data, Transform(faster_rcnn, mirror=True)) if args.proc_num == 1: train_iter = SerialIterator(train_data, args.batch_size, True, True) else: train_iter = MultiprocessIterator(train_data, batch_size=args.batch_size, n_processes=args.proc_num, repeat=True, shuffle=True, n_prefetch=10, shared_mem=31457280) model = FasterRCNNTrainChain(faster_rcnn) if "," in args.gpu: for gpu in args.gpu.split(","): chainer.cuda.get_device_from_id(int(gpu)).use() else: chainer.cuda.get_device_from_id(int(args.gpu)).use() optimizer = None if args.optimizer == 'AdaGrad': optimizer = chainer.optimizers.AdaGrad( lr=args.lr ) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题,改为AdaGrad elif args.optimizer == 'RMSprop': optimizer = chainer.optimizers.RMSprop(lr=args.lr) elif args.optimizer == 'Adam': print("using Adam") optimizer = chainer.optimizers.Adam(alpha=args.lr) elif args.optimizer == 'SGD': optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) elif args.optimizer == "AdaDelta": print("using AdaDelta") optimizer = chainer.optimizers.AdaDelta() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) optimizer_name = args.optimizer if not os.path.exists(args.out): os.makedirs(args.out) pretrained_optimizer_file_name = '{0}_fold_{1}_{2}_{3}_{4}_optimizer.npz'.format( args.fold, args.split_idx, args.feature_model, "linear", optimizer_name) pretrained_optimizer_file_name = args.out + os.sep + pretrained_optimizer_file_name single_model_file_name = args.out + os.sep + '{0}_fold_{1}_{2}_{3}_model.npz'.format( args.fold, args.split_idx, args.feature_model, "linear") if os.path.exists(pretrained_optimizer_file_name): print("loading optimizer snatshot:{}".format( pretrained_optimizer_file_name)) chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer) if os.path.exists(single_model_file_name): print("loading pretrained snapshot:{}".format(single_model_file_name)) chainer.serializers.load_npz(single_model_file_name, model.faster_rcnn) if "," in args.gpu: gpu_dict = {"main": int(args.gpu.split(",")[0])} # many gpu will use for slave_gpu in args.gpu.split(",")[1:]: gpu_dict[slave_gpu] = int(slave_gpu) updater = chainer.training.ParallelUpdater( train_iter, optimizer, devices=gpu_dict, converter=lambda batch, device: concat_examples( batch, device, padding=-99)) else: print("only one GPU({0}) updater".format(args.gpu)) updater = chainer.training.StandardUpdater( train_iter, optimizer, device=int(args.gpu), converter=lambda batch, device: concat_examples( batch, device, padding=-99)) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(chainer.training.extensions.snapshot_object( optimizer, filename=os.path.basename(pretrained_optimizer_file_name)), trigger=(args.snapshot, 'iteration')) snap_model_file_name = '{0}_fold_{1}_{2}_model.npz'.format( args.fold, args.split_idx, args.feature_model) trainer.extend(chainer.training.extensions.snapshot_object( model.faster_rcnn, filename=snap_model_file_name), trigger=(args.snapshot, 'iteration')) log_interval = 100, 'iteration' print_interval = 100, 'iteration' plot_interval = 100, 'iteration' if args.optimizer != "Adam" and args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1), trigger=(10, 'epoch')) elif args.optimizer == "Adam": # use Adam trainer.extend(chainer.training.extensions.ExponentialShift( "alpha", 0.5, optimizer=optimizer), trigger=(10, 'epoch')) if args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend( chainer.training.extensions.LogReport( trigger=log_interval, log_name="{0}_fold_{1}.log".format(args.fold, args.split_idx))) trainer.extend(chainer.training.extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/pearson_correlation', ]), trigger=print_interval) trainer.extend( chainer.training.extensions.ProgressBar(update_interval=100)) if chainer.training.extensions.PlotReport.available(): trainer.extend(chainer.training.extensions.PlotReport( ['main/loss'], file_name='loss_{0}_fold_{1}.png'.format(args.fold, args.split_idx), trigger=plot_interval), trigger=plot_interval) trainer.extend(chainer.training.extensions.PlotReport( ['main/pearson_correlation'], file_name='pearson_correlation_{0}_fold_{1}.png'.format( args.fold, args.split_idx), trigger=plot_interval), trigger=plot_interval) trainer.run()
def concat_and_pad_examples(batch, device=None, padding=-10000): return concat_examples(batch, device=device, padding=padding)
def __call__(self, x): ''' Do a forward pass on the inference policy Args: x -- list (!) of actions / states up to final state, i.e. [a_0,x_1,a_1,x_2, ..., a_{f-1}, x_f] ''' # # PREPROCESSING # Sequence of (a_{t},x_{t+1}) pairs of actions and caused observations # concat_examples(x) converts an "array of tuples" x into a # "tuple of arrays" action_seq, input_observation_seq # I.e. from x = [a_0,x_1,a_1,x_2, ..., a_{f-1}, x_f] to # [a_0,a_1,...,a_{f-1}], [x_1,x_2,...,x_f] # where the first is a single list, and the latter are two arrays. action_seq, input_observation_seq = concat_examples(x) input_observation_seq = Batch.to_array_and_type( input_observation_seq, self.xp.float32) trajectory_length = action_seq.size # Create array of one-hot vectors for each action in the target sequence # We only need this for actions a_{t-1} in the LSTM, # so omitting last action from one-hot sequence prev_action_seq_oh = self.xp.zeros((trajectory_length-1, self._output_space.n),\ dtype=self.xp.float32) prev_action_seq_oh[self.xp.arange(trajectory_length - 1), action_seq[0:-1]] = 1.0 # Wrap both input observations and target actions into variable object # to keep track of computational graph. # This is required for the backward pass / gradient computation later on prev_action_seq_oh = Variable(prev_action_seq_oh) input_observation_seq = Variable(input_observation_seq) # # FORWARD PASS # Create embeddings u_t of observations x_t with last # element = embedding of final observation x_f u_seq = F.relu(self.lin_embed_p(input_observation_seq)) # Create sequence of concatenated # (observation embedding at t, final observation embedding) pairs [u_t, u_f] for all t < f # Excluding [u_f, u_f] as this is the state that we use to infer actions chosen # at previous states u_t, t<f # For concatenation, we need two arrays of same size f-1. Thus broadcasting u_f to fit size. # u_f=u_seq[-1], i.e. the last element in the sequence of embeddings. u_f = F.broadcast_to(u_seq[-1], (trajectory_length - 1, u_seq[-1].size)) input_embedding_seq = F.concat((u_seq[0:-1], u_f), axis=1) # Create joint embeddings v=RW[u_t,u_f] for each concatenated observation embedding v_seq = F.relu(self.lin_embed_q(input_embedding_seq)) # Output from behaviour policy LSTM at different timesteps. # TODO: Take actual output from policy LSTM (self.lstm_p) once that's implemented. # Constant for now as we assume random policy for testing. h_p = self.xp.full((trajectory_length-1, self._lstm_size),\ 1.0/self._lstm_size, dtype=self.xp.float32) # Inference LSTM input [h^p_t, v_t, a_{t-1}] lstm_q_in = F.concat((h_p, v_seq, prev_action_seq_oh)) # Get into batch shape for N-step LSTM # LSTM expects list with 2d arrays. 1st index = time, # 2nd index = batch index, 3rd index = LSTM input elements # Here we've only got one batch, thus first dimension = trajectory length / time, # second dimension = 1, and third dimension is size of LSTM input. lstm_q_in_list = [] for t in range(trajectory_length - 1): lstm_q_in_list.append(F.expand_dims(lstm_q_in[t], axis=0)) # Feed it through n-step LSTM # Returns a tuple: final hidden states at t=f, updated cell state at t=f, # updated hidden states after each t # We're only interested in the third, as this is the networks' prediction for time t h_q = self.lstm_q(None, None, lstm_q_in_list)[2] # Turn list of batches of hidden state vectors (list[2darray]) # into 2-d array of hidden state vectors h_q = F.stack(h_q) h_q = F.squeeze(h_q) # Use another linear transformation to bring LSTM output h to the size of our action set # This way, the size of our action set does not constrain the expressiveness of the LSTM predictions = self.lin_pi_q(h_q) # We return the outcome predictions and target action sequences # Predictions = the unnormalised log-probabilities, as our inferred probabilities of # actions performed at t, given observation x_t and final observation x_f # Target action sequences = only the actions following the observations, # i.e. first action in original sequence omitted target_action_seq = action_seq[1:] return (predictions, target_action_seq)
def concat_and_reshape(batch, device=None, padding=None): x, y = dataset.concat_examples(batch, device, padding) return x.reshape(len(x), 1, 784), y
def main(): parser = argparse.ArgumentParser( description='Space Time Action Unit R-CNN training example:') parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/') parser.add_argument('--gpu', '-g', nargs='+', type=int, help='GPU ID, multiple GPU split by space') parser.add_argument('--lr', '-l', type=float, default=0.001) parser.add_argument('--out', '-o', default='output_two_stream', help='Output directory') parser.add_argument('--database', default='BP4D', help='Output directory: BP4D/DISFA/BP4D_DISFA') parser.add_argument('--iteration', '-i', type=int, default=70000) parser.add_argument('--epoch', '-e', type=int, default=20) parser.add_argument('--batch_size', '-bs', type=int, default=1) parser.add_argument('--snapshot', '-snap', type=int, default=1000) parser.add_argument('--mean_rgb', default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy", help='image mean .npy file') parser.add_argument('--mean_flow', default=config.ROOT_PATH + "BP4D/idx/mean_flow.npy", help='image mean .npy file') parser.add_argument('--backbone', default="resnet101", help="vgg/resnet101/mobilenet_v1 for train") parser.add_argument('--optimizer', default='SGD', help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta') parser.add_argument('--pretrained_model_rgb', help='imagenet/mobilenet_v1/resnet101/*.npz') parser.add_argument( '--pretrained_model_flow', help= "path of optical flow pretrained model, can also use the same npz with rgb" ) parser.add_argument('--two_stream_mode', type=TwoStreamMode, choices=list(TwoStreamMode), help='rgb_flow/ optical_flow/ rgb') parser.add_argument( '--use_memcached', action='store_true', help='whether use memcached to boost speed of fetch crop&mask') # parser.add_argument('--memcached_host', default='127.0.0.1') parser.add_argument("--fold", '-fd', type=int, default=3) parser.add_argument("--fix", action="store_true", help="fix parameter of conv2 update when finetune") parser.add_argument("--split_idx", '-sp', type=int, default=1) parser.add_argument("--use_paper_num_label", action="store_true", help="only to use paper reported number of labels" " to train") parser.add_argument( "--roi_align", action="store_true", help="whether to use roi align or roi pooling layer in CNN") parser.add_argument("--T", '-T', type=int, default=10) parser.add_argument("--proc_num", "-proc", type=int, default=1) args = parser.parse_args() os.makedirs(args.pid, exist_ok=True) os.makedirs(args.out, exist_ok=True) pid = str(os.getpid()) pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format( args.database, args.fold, args.split_idx) with open(pid_file_path, "w") as file_obj: file_obj.write(pid) file_obj.flush() print('GPU: {}'.format(",".join(list(map(str, args.gpu))))) adaptive_AU_database(args.database) mc_manager = None if args.use_memcached: from collections_toolkit.memcached_manager import PyLibmcManager mc_manager = PyLibmcManager(args.memcached_host) if mc_manager is None: raise IOError("no memcached found listen in {}".format( args.memcached_host)) paper_report_label, class_num = squeeze_label_num_report( args.database, args.use_paper_num_label) paper_report_label_idx = list(paper_report_label.keys()) au_rcnn_train_chain_list = [] if args.backbone == 'vgg': au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], use_roi_align=args.roi_align) au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn) au_rcnn_train_chain_list.append(au_rcnn_train_chain) elif args.backbone == 'resnet101': if args.two_stream_mode != TwoStreamMode.rgb_flow: assert (args.pretrained_model_rgb == "" and args.pretrained_model_flow != "") or\ (args.pretrained_model_rgb != "" and args.pretrained_model_flow == "") pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_flow au_rcnn = AU_RCNN_Resnet101( pretrained_model=pretrained_model, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], use_roi_align=args.roi_align, use_optical_flow_input=( args.two_stream_mode == TwoStreamMode.optical_flow), temporal_length=args.T) au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn) au_rcnn_train_chain_list.append(au_rcnn_train_chain) else: # rgb_flow mode au_rcnn_rgb = AU_RCNN_Resnet101( pretrained_model=args.pretrained_model_rgb, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], use_roi_align=args.roi_align, use_optical_flow_input=False, temporal_length=args.T) au_rcnn_optical_flow = AU_RCNN_Resnet101( pretrained_model=args.pretrained_model_flow, min_size=config.IMG_SIZE[0], max_size=config.IMG_SIZE[1], use_roi_align=args.roi_align, use_optical_flow_input=True, temporal_length=args.T) au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb) au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor( au_rcnn_optical_flow) au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb) au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow) model = Wrapper(au_rcnn_train_chain_list, class_num, args.database, args.T, two_stream_mode=args.two_stream_mode, gpus=args.gpu) batch_size = args.batch_size img_dataset = AUDataset(database=args.database, L=args.T, fold=args.fold, split_name='trainval', split_index=args.split_idx, mc_manager=mc_manager, two_stream_mode=args.two_stream_mode, train_all_data=False, paper_report_label_idx=paper_report_label_idx) train_dataset = TransformDataset( img_dataset, Transform(L=args.T, mirror=True, mean_rgb_path=args.mean_rgb, mean_flow_path=args.mean_flow)) if args.proc_num == 1: train_iter = SerialIterator(train_dataset, batch_size, repeat=True, shuffle=True) else: train_iter = MultiprocessIterator(train_dataset, batch_size=batch_size, n_processes=args.proc_num, repeat=True, shuffle=True, n_prefetch=3, shared_mem=53457280) if len(args.gpu) > 1: for gpu in args.gpu: chainer.cuda.get_device_from_id(gpu).use() else: chainer.cuda.get_device_from_id(args.gpu[0]).use() model.to_gpu(args.gpu[0]) optimizer = None if args.optimizer == 'AdaGrad': optimizer = chainer.optimizers.AdaGrad( lr=args.lr ) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题,改为AdaGrad elif args.optimizer == 'RMSprop': optimizer = chainer.optimizers.RMSprop(lr=args.lr) elif args.optimizer == 'Adam': optimizer = chainer.optimizers.Adam(alpha=args.lr) elif args.optimizer == 'SGD': optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9) elif args.optimizer == "AdaDelta": optimizer = chainer.optimizers.AdaDelta() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) optimizer_name = args.optimizer # BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label" roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling" single_model_file_name = args.out + os.sep + \ '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_model.npz'.format(args.database, args.fold, args.split_idx, args.backbone, args.two_stream_mode, use_paper_key_str, roi_align_key_str, args.T) print(single_model_file_name) pretrained_optimizer_file_name = args.out + os.sep + \ '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_optimizer.npz'.format( args.database, args.fold, args.split_idx, args.backbone, args.two_stream_mode, use_paper_key_str, roi_align_key_str, args.T) print(pretrained_optimizer_file_name) if os.path.exists(pretrained_optimizer_file_name): print("loading optimizer snatshot:{}".format( pretrained_optimizer_file_name)) chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer) if os.path.exists(single_model_file_name): print("loading pretrained snapshot:{}".format(single_model_file_name)) chainer.serializers.load_npz(single_model_file_name, model) if args.fix: au_rcnn = model.au_rcnn_train_chain.au_rcnn au_rcnn.extractor.conv1.W.update_rule.enabled = False au_rcnn.extractor.bn1.gamma.update_rule.enabled = False au_rcnn.extractor.bn1.beta.update_rule.enabled = False res2_names = ["a", "b1", "b2"] for res2_name in res2_names: if res2_name == "a": getattr(au_rcnn.extractor.res2, res2_name).conv1.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv2.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv3.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv4.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn4.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn4.beta.update_rule.enabled = False elif res2_name.startswith("b"): getattr(au_rcnn.extractor.res2, res2_name).conv1.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn1.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv2.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).conv3.W.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn2.beta.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.gamma.update_rule.enabled = False getattr(au_rcnn.extractor.res2, res2_name).bn3.beta.update_rule.enabled = False updater = chainer.training.StandardUpdater( train_iter, optimizer, device=args.gpu[0], converter=lambda batch, device: concat_examples( batch, device, padding=0)) trainer = training.Trainer(updater, (10, 'iteration'), out=args.out) trainer.extend(chainer.training.extensions.snapshot_object( optimizer, filename=os.path.basename(pretrained_optimizer_file_name)), trigger=(args.snapshot, 'iteration')) trainer.extend(chainer.training.extensions.snapshot_object( model, filename=os.path.basename(single_model_file_name)), trigger=(args.snapshot, 'iteration')) log_interval = 100, 'iteration' print_interval = 100, 'iteration' plot_interval = 10, 'iteration' if args.optimizer != "Adam" and args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1), trigger=(10, 'epoch')) elif args.optimizer == "Adam": trainer.extend(chainer.training.extensions.ExponentialShift( "alpha", 0.1, optimizer=optimizer), trigger=(10, 'epoch')) if args.optimizer != "AdaDelta": trainer.extend(chainer.training.extensions.observe_lr(), trigger=log_interval) trainer.extend( chainer.training.extensions.LogReport( trigger=log_interval, log_name="log_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.log".format( args.database, args.fold, args.split_idx, args.backbone, args.two_stream_mode, use_paper_key_str, roi_align_key_str, args.T))) # trainer.reporter.add_observer("main_par", model.loss_head_module) trainer.extend(chainer.training.extensions.PrintReport([ 'iteration', 'epoch', 'elapsed_time', 'lr', 'main/loss', 'main/accuracy', ]), trigger=print_interval) trainer.extend( chainer.training.extensions.ProgressBar(update_interval=100)) if chainer.training.extensions.PlotReport.available(): trainer.extend(chainer.training.extensions.PlotReport( ['main/loss'], file_name="loss_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png".format( args.database, args.fold, args.split_idx, args.backbone, args.two_stream_mode, use_paper_key_str, roi_align_key_str, args.T), trigger=plot_interval), trigger=plot_interval) trainer.extend(chainer.training.extensions.PlotReport( ['main/accuracy'], file_name="accuracy_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png". format(args.database, args.fold, args.split_idx, args.backbone, args.two_stream_mode, use_paper_key_str, roi_align_key_str, args.T), trigger=plot_interval), trigger=plot_interval) # trainer.run() cProfile.runctx("trainer.run()", globals(), locals(), "Profile.prof") s = pstats.Stats("Profile.prof") s.strip_dirs().sort_stats("time").print_stats()
def train(model): dir_path = 'mnist_mixup_' + str(datetime.date.today()) if not os.path.isdir(dir_path): os.mkdir(dir_path) train, test = mnist.get_mnist(withlabel=True, ndim=1) train_iter = iterators.SerialIterator(train, args.batchsize, shuffle=False) test_iter = iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) optimizer = optimizers.MomentumSGD(lr=args.lr, momentum=0.9) optimizer.use_cleargrads() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=args.decay)) result = [] train_losses = [] while train_iter.epoch < args.epoch: train_batch = train_iter.next() x, t = concat_examples(train_batch, gpu_id) x, t_a, t_b, lam = mixup_data(x, t, args.alpha) x = Variable(x) t_a = Variable(t_a) t_b = Variable(t_b) y = model(x) loss = mixup_criterion(y, t_a, t_b, lam) train_losses.append(to_cpu(loss.data)) model.cleargrads() loss.backward() optimizer.update() if train_iter.is_new_epoch: if train_iter.epoch % 5 == 0: print('epoch:{: 02d} train_loss:{: .04f}'.format( train_iter.epoch, float(np.mean(train_losses))), end=' ') adjust_lr(optimizer, train_iter.epoch) test_losses = [] test_accuracies = [] while True: test_batch = test_iter.next() x_test, t_test = concat_examples(test_batch, gpu_id) y_test = model(x_test) loss_test = F.softmax_cross_entropy(y_test, t_test) test_losses.append(to_cpu(loss_test.data)) accuracy = F.accuracy(y_test, t_test) accuracy.to_cpu() test_accuracies.append(accuracy.data) if test_iter.is_new_epoch: test_iter.epoch = 0 test_iter.current_position = 0 test_iter.is_new_epoch = False test_iter._pushed_position = None break if train_iter.epoch % 5 == 0: print('val_loss:{: .04f} val_accuracy:{: .04f}'.format( np.mean(test_losses), np.mean(test_accuracies))) result.append([ train_iter.epoch, float(np.mean(train_losses)), np.mean(test_losses) ]) serializers.save_npz( 'mnist_mixup_' + str(datetime.date.today()) + '/epoch-' + str(train_iter.epoch), model) train_losses.clear() result = np.asarray(result) epoch = result[:, 0] tr_loss = result[:, 1] te_loss = result[:, 2] plt.plot(epoch, tr_loss, color='blue', label="main/loss", marker="x") plt.plot(epoch, te_loss, color='orange', label="validation/main/loss", marker="x") plt.legend() plt.savefig("mnist_mixup_" + str(datetime.date.today()) + "/loss.png")
shuffle=False) model = MujiNN() optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) max_epoch = 100 pLoss = 10.0 # while train_iter.epoch < max_epoch: while pLoss > 0.0001: # ---------- One iteration of the training loop ---------- train_batch = train_iter.next() data_train, target_train = concat_examples(train_batch) # Calculate the prediction of the network prediction_train = model(data_train) # Calculate the loss with softmax_cross_entropy loss = F.softmax_cross_entropy(prediction_train, target_train) pLoss = float(loss.data) # Calculate the gradients in the network model.cleargrads() loss.backward() # Update all the trainable paremters optimizer.update() # --------------------- until here ---------------------
def train( iterator, gpu, encoder, decoder, enc_optim, dec_optim, rel_send, rel_rec, edge_types, temp, prediction_steps, var, out, benchmark, lr_decay, gamma): iter_i = 0 edge_accuracies = [] node_mses = [] nll_train = [] kl_train = [] logger = logging.getLogger(__name__) while True: inputs = iterator.next() node_features, edge_labels = dataset.concat_examples(inputs, device=gpu) # logits: [batch_size, num_edges, edge_types] logits = encoder(node_features, rel_send, rel_rec) # inverse func. of softmax edges = F.gumbel_softmax(logits, tau=temp, axis=2) edge_probs = F.softmax(logits, axis=2) # edges, edge_probs: [batch_size, num_edges, edge_types] if isinstance(decoder, decoders.MLPDecoder): output = decoder( node_features, edges, rel_rec, rel_send, prediction_steps) elif isinstance(decoder, decoders.RNNDecoder): output = decoder( node_features, edges, rel_rec, rel_send, 100, burn_in=True, burn_in_steps=args.timesteps - args.prediction_steps) target = node_features[:, :, 1:, :] num_nodes = node_features.shape[1] loss_nll = get_nll_gaussian(output, target, var) loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types) loss = loss_nll + loss_kl nll_train.append(float(loss_nll.array)) kl_train.append(float(loss_kl.array)) edge_accuracy = get_edge_accuracy(logits.array, edge_labels) edge_accuracies.append(edge_accuracy) node_mse = float(F.mean_squared_error(output, target).array) node_mses.append(node_mse) encoder.cleargrads() decoder.cleargrads() loss.backward() enc_optim.update() dec_optim.update() # Exit after 10 iterations when benchmark mode is ON iter_i += 1 if benchmark: put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses)) if iter_i == 10: exit() if iterator.is_new_epoch: break if not os.path.exists(os.path.join(out, 'graph.dot')): with open(os.path.join(out, 'graph.dot'), 'w') as o: g = computational_graph.build_computational_graph([loss]) o.write(g.dump()) if iterator.is_new_epoch: put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses)) serializers.save_npz(os.path.join(out, 'encoder_epoch-{}.npz'.format(iterator.epoch)), encoder) serializers.save_npz(os.path.join(out, 'decoder_epoch-{}.npz'.format(iterator.epoch)), decoder) serializers.save_npz(os.path.join(out, 'enc_optim_epoch-{}.npz'.format(iterator.epoch)), enc_optim) serializers.save_npz(os.path.join(out, 'dec_optim_epoch-{}.npz'.format(iterator.epoch)), dec_optim) if iterator.epoch % lr_decay == 0: enc_optim.alpha *= gamma dec_optim.alpha *= gamma logger.info('alpha of enc_optim: {}'.format(enc_optim.alpha)) logger.info('alpha of dec_optim: {}'.format(dec_optim.alpha))
def evaluate(model, dataset, device=-1, flip=False): batch_size = 50 data_iter = chainer.iterators.MultithreadIterator(dataset, batch_size, repeat=False, shuffle=False) corrects = list() counts = list() for it, batch in enumerate(data_iter): # print progress print(f'{batch_size*it:04d} / {len(dataset):04d}', end='\r') img, label, idx, scale, shape = concat_examples(batch) N, C, H, W = img.shape if flip: img = np.array((img, img[:, :, :, ::-1])) img = img.reshape(N * 2, C, H, W) if device >= 0: img = cuda.to_gpu(img) with chainer.no_backprop_mode(): # (N, 3, 256, 256) -> (N, 16, 64, 64) _output, output = model(img) output = output.array if flip: output = output.reshape(( 2, N, ) + output.shape[1:]) output_flipped = flip_heatmap(output[1], copy=True) output = (output[0] + output_flipped) / 2 N, C, H, W = output.shape keypoints = list() # (N, 16, 64, 64) -> (N, 16, 2) for i in range(N): # (16, 64, 64) -> (16, -1) out_reshaped = output[i].reshape(C, -1).argmax(axis=1) out_reshaped = cuda.to_cpu(out_reshaped) keypoint = np.unravel_index(out_reshaped, (H, W)) # (2, 16) -> (16, 2) keypoint = np.array(keypoint).T keypoint = transforms.resize_point(keypoint, (H, W), shape[i]) keypoints.append(keypoint) else: keypoints = np.array(keypoints) correct, count = pckh_score(label, keypoints, idx, scale) corrects.append(correct) counts.append(count) print() corrects = np.sum(corrects, axis=0) counts = np.sum(counts, axis=0) # Head, Shoulder, Elbow, Wrist, Hip, Knee, Ankle joints = { 'head': [8, 9], 'shoulder': [12, 13], 'elbow': [11, 14], 'wrist': [10, 15], 'hip': [2, 3], 'knee': [1, 4], 'ankle': [0, 5] } scores = dict() for key, value in joints.items(): score = corrects[value].sum() / counts[value].sum() scores.update({key: score}) return scores
if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) config_names = ['data', 'param', 'arch', 'model', 'norm', 'gpu', 'layer'] config_lst = [ args.data_path, args.param_path, args.arch_path, args.model_path, args.norm, args.gpu, args.layer ] config_dic = cl.OrderedDict() for n, c in zip(config_names, config_lst): config_dic[n] = c with open(os.path.join(args.out_dir, 'config.json'), 'w') as fp: json.dump(config_dic, fp) ds_info = get_dataset(args.data_path, norm=args.norm) data = concat_examples(ds_info.__getitem__(slice(0, ds_info.__len__(), 1)), device=args.gpu) with open(args.param_path, 'r') as fp: param = json.load(fp) with open(args.arch_path, 'r') as fp: arch = json.load(fp) model = predict_four(arch, dr=param['dr'], bn=param['bn']) serializers.load_npz(args.model_path, model) if args.gpu >= 0: cuda.get_device(data[0][0]).use() model.to_gpu() xp = cuda.get_array_module(data[0][0]) model.train = False cor_dir = os.path.join(args.out_dir, 'correct')
if gpu_id >= 0: model.to_gpu(gpu_id) from chainer import optimizers optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) # train loop import numpy as np from chainer.dataset import concat_examples from chainer.backends.cuda import to_cpu max_epoch = 10 while train_iter.epoch < max_epoch: train_batch = train_iter.next() image_train, target_train = concat_examples(train_batch, gpu_id) prediction_train = model(image_train) loss = F.softmax_cross_entropy(prediction_train, target_train) model.cleargrads() loss.backward() optimizer.update() if train_iter.is_new_epoch: print('epoch:{:02d} train_loss{:04f}'.format( train_iter.epoch, float(to_cpu(loss.data))), end='') test_losses = []
import chainer.iterators from chainer.dataset import concat_examples from chainer.cuda import to_cpu import numpy as np from PIL import Image import matplotlib.pyplot as plt from load_model import forward, load_model from Attack_Function.fgsm import fgsm models = load_model() chainer.config.train = False #turn model to eval mode train, test = chainer.datasets.get_fashion_mnist(ndim=3) train_iter = chainer.iterators.SerialIterator(test, 1) i = 1 eps_list = [0.05, 0.1, 0.3, 0.6] for eps in eps_list: train_iter.current_position = 0 train_iter.epoch = 0 train_iter.is_new_epoch = False train_iter._pushed_position = None #reset test_iter test_batch = train_iter.next() image_test, target_test = concat_examples(test_batch) image_test = F.resize_images(image_test, (32, 32)).data adv = fgsm(models, forward, image_test, eps=eps) plt.subplot(1, 4, i) plt.imshow(to_cpu(adv.squeeze())) plt.title("Label:{}".format(to_cpu(F.argmax(forward(adv, models)).data))) i = i + 1 plt.savefig("demo.jpg")
def save_results(dataset, saveflag, filename, lam, w, z): for n in range(len(dataset.lens)): in_vars = tuple( Variable(x, volatile='on') for x in dataset_module.concat_examples( dataset[sum(dataset.lens[0:n]):sum(dataset.lens[0:n + 1])])) g0, g1, h0, h1 = loss.net(*in_vars, phi=loss.phi, train=False) # prepare outputs y0 = np.transpose(in_vars[0].data, axes=(1, 0, 2))[-1] y1 = np.transpose(in_vars[1].data, axes=(1, 0, 2))[-1] g0 = cuda.to_cpu(g0.data) g1 = cuda.to_cpu(g1.data) h0 = cuda.to_cpu(h0.data) h1 = cuda.to_cpu(h1.data) # do prediction in g-space g_preds = [] # starts from prediction corresponding to g1 for j in range(args.horizon): tmp = np.zeros(g0.shape) + 1j * np.zeros(g0.shape) if j == 0: g_prev = g0 else: g_prev = g_preds[j - 1] for i in range(g1.shape[0]): tmp[i] = np.sum(np.dot( w, np.diag(np.dot(np.diag(lam), np.dot(z.conj().T, g_prev[i])))), axis=1) g_preds.append(tmp.copy()) # back-project the prediction to y-space y_preds = [] rmses = [0 for i in range(args.horizon)] for j in range(args.horizon): g_pred_var = Variable(np.real(g_preds[j]).astype(dtype=np.float32), volatile='on') tmp = loss.net.h(g_pred_var, train=False).data y_preds.append(tmp.copy()) if j == 0: tmp_end = tmp.shape[0] else: tmp_end = -j rmses[j] = np.sqrt(np.mean(np.square(tmp[:tmp_end] - y1[j:]))) # save if saveflag: savemat( os.path.join(args.outputdir, '{0}_{1:d}'.format(filename, n)), { 'lam': lam, 'w': w, 'z': z, 'prediction_rmses': rmses, 'phi_W': loss.phi.l1.W.data, 'phi_b': loss.phi.l1.b.data, 'g_preds': g_preds, 'y_preds': y_preds, 'y0': y0, 'g0': g0, 'h0': h0, 'y1': y1, 'g1': g1, 'h1': h1 }) return rmses
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=50, help='Number of units') parser.add_argument('--example', '-ex', type=int, default=3, help='Example mode') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Load the MNIST dataset train, test = chainer.datasets.get_mnist() model = SklearnWrapperClassifier(MLP(args.unit, 10), device=args.gpu) if args.example == 1: print("Example 1. fit with x, y numpy array (same with sklearn's fit)") x, y = concat_examples(train) model.fit(x, y) elif args.example == 2: print("Example 2. Train with Chainer's dataset") # `train` is TupleDataset in this example # Even this one line work! (but no validation) model.fit(train) else: print("Example 3. Train with configuration") model.fit( train, test=test, batchsize=args.batchsize, #iterator_class=chainer.iterators.SerialIterator, optimizer=chainer.optimizers.Adam(), epoch=args.epoch, out=args.out, snapshot_frequency=1, #dump_graph=False #log_report=True, plot_report=False, #print_report=True, progress_report=False, resume=args.resume) # Save trained model serializers.save_npz('{}/mlp.model'.format(args.out), model)
def evaluate(self, snapshot_name=''): reporter = reporter_module.Reporter() current_device = chainer.get_device(self.args.gpu) summary = reporter_module.DictSummary() with chainer.using_device(current_device), reporter, configuration.using_config('train', False): for i, batch in enumerate(tqdm(self.data_iterator, total=len(self.data_loader) // self.args.batchsize)): observation = {} batch = concat_examples(batch, self.args.gpu) image_size = Size._make(batch['image'].shape[-2:]) with reporter_module.report_scope(observation): rois, bboxes, text_predictions, best_indices, chosen_prediction, scores = self.evaluator(return_predictions=True, **batch) summary.add(observation) if self.args.save_predictions: assert self.args.batchsize == 1, "if you want to save predictions, batchsize must be 1!" batch_size, num_predictions, num_bboxes, num_channels, height, width = rois.shape base_image = self.bbox_plotter.array_to_image(batch['image'][0, 0]) chosen_word = self.data_loader.decode_chars(cuda.to_cpu(text_predictions[0, chosen_prediction[0]].squeeze())) base_image = self.bbox_plotter.render_text(base_image, base_image, chosen_word, 0, bottom=True) rendered_images = [base_image] iterator = zip( F.separate(self.localizer.xp.stack( [batch['image'][i, best_indices[i]] for i in range(self.args.batchsize)]), axis=1), F.separate(rois, axis=1), F.separate(bboxes, axis=1), F.separate(text_predictions, axis=1), F.separate(scores, axis=1) ) for image, roi, bbox, prediction, score in iterator: image = image.array roi = roi.array bbox = bbox.array prediction = prediction.array score = score.array bbox = self.localizer.xp.reshape(bbox, (-1, 2, height, width)) predicted_words = self.data_loader.decode_chars(cuda.to_cpu(prediction.squeeze())) predicted_words = f"{predicted_words} {format(float(score[0]), '.4f')}" if args.cut_bboxes: cut_length = batch['num_words'][0] if 'num_words' in batch else len(predicted_words) bbox = bbox[:cut_length, ...] roi = roi[:cut_length, ...] if args.render_no_boxes: bbox = bbox[:1] roi = roi[:1] rendered_images.append( self.render_roi( [], bbox, None, i, image, roi, predicted_words ) ) self.save_rois(rendered_images, i) self.save_eval_results(snapshot_name, summary)
def grad_cam(model, val, gpu, save_dir, backward_label='true_label'): xp = cp if gpu >= 0 else np # batchサイズ(今回は1枚ずつ行うので1) cam_iter = chainer.iterators.SerialIterator(val, 1, repeat=False, shuffle=False) count = 0 while True: # brakeするまで続ける X_test_batch = cam_iter.next() # テストデータをデータとラベルに分ける X_test, y_test = concat_examples(X_test_batch, gpu) with chainer.using_config("train", False): pred = model.predictor(X_test) probs = F.softmax(pred).data[0] if gpu >= 0: probs = chainer.cuda.to_cpu(probs) top1 = np.argsort(probs)[::-1][0] pred.zerograd() pred.grad = xp.zero([1, 8], dtype=np.float32) if backward_label == 'true_label': backward_label = y_test[0] else: backward_label = top1 pred.grad[0, backward_label] = 1 pred.backward(True) feature = model.predictor.cam.data[0] grad = model.predictor.cam.grad[0] cam = xp.ones(feature.shape[1:], dtype=xp.float32) weights = grad.mean((1, 2)) * 1000 for i, w in enumerate(weights): cam += feature[i] * w if gpu >= 0: cam = chainer.cuda.to_cpu(cam) X_test = chainer.cuda.to_cpu(X_test) cam = cv2.resize(cam, (224, 224)) cam = np.maximum(cam, 0) heatmap = cam / np.mat(cam) heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET) ########################################################################### image = X_test[0, ::-1, :, :].transpose(1, 2, 0) image -= np.min(image) image = np.minimum(image, 255) cam_img = np.float32(image) cam_img = np.float32(heatmap) + np.float32(image) cam_img = 255 * cam_img / np.max(cam_img) save_img_dir = save_dir + '/visualize' if not os.exists(save_img_dir): os.mkdir(save_img_dir) save_name = save_img_dir + '/cam_img_{}.png'.format(count) cv2.imwrite(save_name, cam_img) count += 1 if cam_iter.is_new_epoch: cam_iter.epoch = 0 cam_iter.current_position = 0 cam_iter.is_new_epoch = False cam_iter._pushed_position = None break
def training_loop(model=None, optimizer=None, stats=None, epochs=None, train_iter=None, test_iter=None, lf=None, models_folder=None, epochs_so_far=0, args=None): train_losses = [] train_accs = [] while train_iter.epoch < epochs: # ------------ One epoch of the training loop ------------ # ---------- One iteration of the training loop ---------- train_batch = train_iter.next() image_train = concat_examples(train_batch, 0) # Calculate the loss with softmax_cross_entropy train_loss, train_rec_loss, train_label_loss, acc, _ = model.get_loss_func( )(image_train) train_losses.append(train_loss.array) train_accs.append(acc.array) # Calculate the gradients in the network model.cleargrads() train_loss.backward() # Update all the trainable paremters optimizer.update() if train_iter.epoch % int(args.freq) == 0: serializers.save_npz( os.path.join(models_folder, str(train_iter.epoch + epochs_so_far) + '.model'), model) # --------------------- iteration until here --------------------- if train_iter.is_new_epoch: test_losses = [] test_accs = [] test_rec_losses = [] test_label_losses = [] test_kl = [] while True: test_batch = test_iter.next() image_test = concat_examples(test_batch, 0) loss, rec_loss, label_loss, label_acc, kl = model.get_loss_func( )(image_test) test_losses.append(loss.array) test_rec_losses.append(rec_loss.array) test_label_losses.append(label_loss.array) test_accs.append(label_acc.array) test_kl.append(kl.array) if test_iter.is_new_epoch: test_iter.epoch = 0 test_iter.current_position = 0 test_iter.is_new_epoch = False test_iter._pushed_position = None break stats['train_loss'].append(np.mean(to_cpu(train_losses))) stats['train_accs'].append(np.mean(to_cpu(train_accs))) stats['valid_loss'].append(np.mean(to_cpu(test_losses))) stats['valid_rec_loss'].append(np.mean(to_cpu(test_rec_losses))) stats['valid_label_loss'].append(np.mean( to_cpu(test_label_losses))) stats['valid_label_acc'].append(np.mean(to_cpu(test_accs))) stats['valid_kl'].append(np.mean(to_cpu(test_kl))) print(("Epoch: {0} \t T_Loss: {1} \t V_Loss: {2} \t V_Rec_Loss: {3} \t V_Label_Loss: {4} \t " + \ "V_KL: {6} \t T_Acc: {7} \t V_Acc: {5}").format(train_iter.epoch, round(stats['train_loss'][-1], 2), round(stats['valid_loss'][-1], 2), round(stats['valid_rec_loss'][-1], 2), round(stats['valid_label_loss'][-1], 2), round(stats['valid_label_acc'][-1], 2), round(stats['valid_kl'][-1], 2), round(stats['train_accs'][-1], 2))) train_losses = [] train_accs = [] # --------------------- epoch until here --------------------- return stats, model, optimizer, epochs