def test_savefun_and_writer_exclusive(self): # savefun and writer arguments cannot be specified together. def savefun(*args, **kwargs): assert False writer = extensions.snapshot_writers.SimpleWriter() with pytest.raises(TypeError): extensions.snapshot(savefun=savefun, writer=writer)
def train(): model = SuperResolution() if DEVICE >= 0: chainer.cuda.get_device_from_id(DEVICE).use() chainer.cuda.check_cuda_available() print("USEDEVICE", DEVICE) model.to_gpu() images = collect_train_patch('train') train_iter = iterators.SerialIterator(images, BATCH_SIZE, shuffle=True) optimizer = optimizers.Adam() optimizer.setup(model) updater = SRUpdater(train_iter, optimizer, device=DEVICE) snapshot_interval=(500, 'epoch') trainer = training.Trainer(updater, (10000, 'epoch'), out='result') trainer.extend(extensions.snapshot( filename='snapshot_epoch_{.updater.epoch}.npz'), trigger=snapshot_interval) trainer.extend(extensions.ProgressBar()) trainer.extend(extensions.snapshot_object(model,'model_epoch_{.updater.epoch}.npz'), trigger=snapshot_interval) if RESUME: # Resume from a snapshot chainer.serializers.load_npz('result/snapshot_epoch_25.npz', trainer) trainer.run() chainer.serializers.save_hdf5('model.hdf5', model)
def main(): parser = argparse.ArgumentParser(dscsription='AutoEncoder') parser.add_argument('--model', '-m', default='AutoEncoder', help='AutoEncoder or ConvolutionalAutoencoder') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--unit', '-u', type=int, default=1000, help='Number of units') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') train_, test_ = chainer.datasets.get_mnist(withlabel=False) if args.model=='CovolutionalAutoencoder': model = ConvolutionalAutoencoder(1, 20, 5, pad=2) train_ = train_.reshape(train_.shape[0], 1, 28, 28) test_ = test_.reshape(test_.shape[0], 1, 28, 28) else: model = AutoEncoder(784, args.unit) model.compute_accuracy = False optimizer = chainer.optimizers.Adam() optimizer.setup(model) train = tuple_dataset.TupleDataset(train_, train_) test = tuple_dataset.TupleDataset(test_, test_) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) trainer.extend(extensions.snapshot()) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss'])) trainer.extend(extensions.ProgressBar()) trainer.run() for i in xrange(args.epoch): draw_digit(pred_list[i*700], 5, 4, i+1, 'decode') plt.show()
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', default=20, type=int, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu0', '-g', default=0, type=int, help='First GPU ID') parser.add_argument('--gpu1', '-G', default=1, type=int, help='Second GPU ID') parser.add_argument('--out', '-o', default='result_parallel', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', default=1000, type=int, help='Number of units') args = parser.parse_args() print('GPU: {}, {}'.format(args.gpu0, args.gpu1)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # See train_mnist.py for the meaning of these lines model = L.Classifier(ParallelMLP(args.unit, 10, args.gpu0, args.gpu1)) chainer.backends.cuda.get_device_from_id(args.gpu0).use() optimizer = chainer.optimizers.Adam() optimizer.setup(model) train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu0) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu0)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def test_call(self): t = mock.MagicMock() c = mock.MagicMock(side_effect=[True, False]) w = mock.MagicMock() snapshot = extensions.snapshot(target=t, condition=c, writer=w) trainer = mock.MagicMock() snapshot(trainer) snapshot(trainer) assert c.call_count == 2 assert w.call_count == 1
def main(): # 辞書の読み込み dictionary = corpus.get_dictionary(create_flg=False) # 記事の読み込み contents = corpus.get_contents() # 特徴抽出 data_train = [] label_train = [] for file_name, content in contents.items(): data_train.append(corpus.get_vector(dictionary, content)) label_train.append(corpus.get_class_id(file_name)) data_train_s, data_test_s, label_train_s, label_test_s = train_test_split(data_train, label_train, test_size=0.5) N_test = len(data_test_s) # test data size N = len(data_train_s) # train data size in_units = len(data_train_s[0]) # 入力層のユニット数 (語彙数) n_units = 1000 # 隠れ層のユニット数 n_label = 9 # 出力層のユニット数 #モデルの定義 model = L.Classifier(MLP(in_units, n_units, n_label)) # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) np_data_train_s = np.array(data_train_s, dtype=np.float32) np_label_train_s = np.array(label_train_s, dtype=np.int32) np_data_test_s = np.array(data_test_s, dtype=np.float32) np_label_test_s = np.array(label_test_s, dtype=np.int32) train_iter = chainer.iterators.SerialIterator(tuple_dataset.TupleDataset(np_data_train_s, np_label_train_s), 100) test_iter = chainer.iterators.SerialIterator(tuple_dataset.TupleDataset(np_data_test_s, np_label_test_s), 100, repeat=False, shuffle=False) updater = training.StandardUpdater(train_iter, optimizer, device=-1) trainer = training.Trainer(updater, (20, 'epoch'), out='result') trainer.extend(extensions.Evaluator(test_iter, model, device=-1)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot()) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy'])) trainer.extend(extensions.ProgressBar()) trainer.run()
def main(): parser = argparse.ArgumentParser(description="Chainer example: MNIST") parser.add_argument("--batchsize", "-b", type=int, default=100, help="Number of images in each mini batch") parser.add_argument("--epoch", "-e", default=20, type=int, help="Number of sweeps over the dataset to train") parser.add_argument("--gpu0", "-g", default=0, type=int, help="First GPU ID") parser.add_argument("--gpu1", "-G", default=1, type=int, help="Second GPU ID") parser.add_argument("--out", "-o", default="result_parallel", help="Directory to output the result") parser.add_argument("--resume", "-r", default="", help="Resume the training from snapshot") parser.add_argument("--unit", "-u", default=1000, type=int, help="Number of units") args = parser.parse_args() print("GPU: {}, {}".format(args.gpu0, args.gpu1)) print("# unit: {}".format(args.unit)) print("# Minibatch-size: {}".format(args.batchsize)) print("# epoch: {}".format(args.epoch)) print("") # See train_mnist.py for the meaning of these lines model = L.Classifier(ParallelMLP(784, args.unit, 10, args.gpu0, args.gpu1)) chainer.cuda.get_device(args.gpu0).use() optimizer = chainer.optimizers.Adam() optimizer.setup(model) train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu0) trainer = training.Trainer(updater, (args.epoch, "epoch"), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu0)) trainer.extend(extensions.dump_graph("main/loss")) trainer.extend(extensions.snapshot()) trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport( ["epoch", "main/loss", "validation/main/loss", "main/accuracy", "validation/main/accuracy"] ) ) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def train(): model_gen = Generator() model_dis = Discriminator() if DEVICE >= 0: chainer.cuda.get_device_from_id(DEVICE).use() chainer.cuda.check_cuda_available() model_gen.to_gpu(DEVICE) model_dis.to_gpu(DEVICE) images = [] fs = os.listdir('train') for f in fs: img = Image.open( 'train/'+f).convert('RGB').resize((IMAGE_SIZE, IMAGE_SIZE)) hpix = np.array(img, dtype=np.float32)/255.0 hpix = hpix.transpose(2, 0, 1) images.append(hpix) train_iter = iterators.SerialIterator(images, BATCH_SIZE, shuffle=True) optimizer_gen = optimizers.Adam(alpha=0.0002, beta1=0.5) optimizer_gen.setup(model_gen) optimizers_dis = optimizers.Adam(alpha=0.0002, beta1=0.5) optimizers_dis.setup(model_dis) updater = Updater( train_iter, {'opt_gen': optimizer_gen, 'opt_dis': optimizers_dis}, device=DEVICE) trainer = training.Trainer(updater, (100000, 'epoch'), out='result') trainer.extend(extensions.ProgressBar()) snapshot_interval = (5000, 'epoch') trainer.extend(extensions.snapshot( filename='snapshot_epoch_{.updater.epoch}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( model_gen, 'model_gen_epoch_{.updater.epoch}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( model_dis, 'model_dis_epoch_{.updater.epoch}.npz'), trigger=snapshot_interval) if RESUME: chainer.serializers.load_npz('result/snapshot_epoch_26797.npz',trainer) trainer.run()
def main(): train,test=chainer.datasets.get_mnist() train_iter=chainer.iterators.SerialIterator(train,100) test_iter=chainer.iterators.SerialIterator(test,100,repeat=False,shuffle=False) model=L.Classifier(MLP(784,10)) optimizer=chainer.optimizers.SGD() optimizer.setup(model) updater=training.StandardUpdater(train_iter,optimizer,device=-1) trainer=training.Trainer(updater,(500,'epoch'),out='result_') trainer.extend(extensions.Evaluator(test_iter,model,device=-1)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(),trigger=(100,'iteration')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch','main/loss','validation/main/loss','main/accuracy','validation/main/acuracy'])) trainer.extend(extensions.ProgressBar()) trainer.run()
def run(datasetPath, resultPath, modelPath="", resumePath=""): # set dataset if isinstance(datasetPath, str): ds = datasetVOC(datasetPath, 32) elif isinstance(datasetPath, list): ds = datasetVOCs(datasetPath, 32) else: raise Exception("データセットパスの型が不正です。") train, test = ds.getDataset() # set model model = chainer.links.Classifier(Alex()) if os.path.isfile(modelPath): chainer.serializers.load_npz(modelPath, model) optimizer = chainer.optimizers.Adam() optimizer.setup(model) # set evaluation model eval_model = model.copy() eval_model.train = False # train and test train_iter = chainer.iterators.SerialIterator(train, BATCH_SIZE) test_iter = chainer.iterators.SerialIterator(test, BATCH_SIZE, repeat=False, shuffle=False) updater = chainer.training.StandardUpdater(train_iter, optimizer, device=-1) trainer = chainer.training.Trainer(updater, (EPOCH, "epoch"), out=resultPath) trainer.extend(extensions.Evaluator(test_iter, eval_model, device=-1)) trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport( ["epoch", "main/loss", "validation/main/loss", "main/accuracy", "validation/main/accuracy"] ) ) trainer.extend(extensions.ProgressBar(update_interval=5)) trainer.extend(extensions.snapshot(filename="snapshot_epoch_{.updater.epoch}")) trainer.extend(extensions.snapshot_object(model, filename="model_epoch_{.updater.epoch}")) trainer.extend(extensions.dump_graph("main/loss")) if os.path.isfile(resumePath): chainer.serializers.load_npz(resumePath, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser(description='MNIST tutorial') parser.add_argument('--model', default='SLP', help='Model to use') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU to use') args = parser.parse_args() model = L.Classifier(models[args.model]()) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() model.to_gpu() optimizer = chainer.optimizers.Adam() optimizer.setup(model) train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=32, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=30, help='Number of sweeps over the dataset to train') parser.add_argument('--centerloss', '-c', action='store_true', default=False, help='Use center loss') parser.add_argument('--alpha_ratio', '-a', type=float, default=0.5, help='alpha ratio') parser.add_argument('--lambda_ratio', '-l', type=float, default=0.1, help='lambda ratio') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') args = parser.parse_args() logger = setup_logger(__name__) logger.info("GPU: {}".format(args.gpu)) logger.info("# Minibatch-size: {}".format(args.batchsize)) logger.info("# epoch: {}".format(args.epoch)) logger.info("Calculate center loss: {}".format(args.centerloss)) if args.centerloss: logger.info('# alpha: {}'.format(args.alpha_ratio)) logger.info('# lambda: {}'.format(args.lambda_ratio)) NUM_CLASSES = 10 model = LeNets( out_dim=NUM_CLASSES, alpha_ratio=args.alpha_ratio, lambda_ratio=args.lambda_ratio, is_center_loss=args.centerloss, ) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist(ndim=3) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize, n_processes=4) test_iter = chainer.iterators.MultiprocessIterator(test, args.batchsize, n_processes=4, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend( extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ])) # Visualize Deep Features trainer.extend(VisualizeDeepFeature(train[:10000], NUM_CLASSES, args.centerloss), trigger=(1, 'epoch')) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=1, help='Number of images in each mini batch') parser.add_argument('--epoch', '-e', type=int, default=40, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=10, help='Number of units') args = parser.parse_args() # load csv n_in = 32*32 print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') model = L.Classifier(MLP(n_in, args.unit, 3)) # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load dataset from CSV csv = pd.read_csv('csv/images-data.csv') dd = [] for file, label in zip(csv['file'], csv['label']): print file, label # load a color image img = cv2.imread(file, cv2.IMREAD_COLOR) # color -> grayscale imggray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) # image -> array gray = [] for y in range(len(imggray)): for x in range(len(imggray[y])): gray.append(imggray[y][x]) imgdata = np.array(gray, dtype='f') imgdata = imgdata.reshape(1, 1, 32, 32) imgdata = imgdata / 255.0 # set dataset x = imgdata y = np.array(label, dtype=np.int32) dataset = (x, y) dd.append(dataset) train, test = dd, dd train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot()) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) # Resume from a snapshot #chainer.serializers.load_npz(resume, trainer) # Run the training trainer.run() # Save Model serializers.save_npz('model/simple-3layer-perceptron.model', model) serializers.save_npz('model/simple-3layer-perceptron.state', optimizer) # Predictor xx = Variable(np.array([dd[1][0],]), volatile=True) y = model.predictor(xx) print y.data print np.argmax(y.data)
def main(): generators = { 'fcn32s': (FCN32s, VGG16, 1e-10), # (model, initmodel, learning_rate) 'fcn16s': (FCN16s, FCN32s, 1e-12), 'fcn8s': (FCN8s, FCN16s, 1e-14), } discriminators = { 'largefov': (LargeFOV, LargeFOV, 0.1, 1.0), # (model, initmodel, learning_rate, L_bce_weight) 'largefov-light': (LargeFOVLight, LargeFOVLight, 0.1, 1.0), 'smallfov': (SmallFOV, SmallFOV, 0.1, 0.1), 'smallfov-light': (SmallFOVLight, SmallFOVLight, 0.2, 1.0), 'sppdis': (SPPDiscriminator, SPPDiscriminator, 0.1, 1.0), } updaters = {'gan': GANUpdater, 'standard': NonAdversarialUpdater} args = parse_args(generators, discriminators, updaters) print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# iteration: {}'.format(args.iteration)) # dataset train = dataset.PascalVOC2012Dataset('train') val = dataset.PascalVOC2012Dataset('val') n_class = len(train.label_names) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) val_iter = chainer.iterators.SerialIterator(val, args.batchsize, repeat=False, shuffle=False) # Set up a neural network to train and an optimizer if args.updater == 'gan': gen_cls, initgen_cls, gen_lr = generators[args.generator] dis_cls, initdis_cls, dis_lr, L_bce_weight = discriminators[ args.discriminator] print('# generator: {}'.format(gen_cls.__name__)) print('# discriminator: {}'.format(dis_cls.__name__)) print('') # Initialize generator if args.initgen_path: gen, initgen = gen_cls(n_class), initgen_cls(n_class) gen = load_pretrained_model(args.initgen_path, initgen, gen, n_class, args.gpu) else: gen = gen_cls(n_class) # Initialize discriminator if args.initdis_path: dis, initdis = dis_cls(n_class), initdis_cls(n_class) dis = load_pretrained_model(args.initdis_path, initdis, dis, n_class, args.gpu) else: dis = dis_cls(n_class) if args.gpu >= 0: chainer.cuda.get_device( args.gpu).use() # Make a specified GPU current gen.to_gpu() # Copy the model to the GPU dis.to_gpu() opt_gen = make_optimizer(gen, gen_lr) opt_dis = make_optimizer(dis, dis_lr) model = {'gen': gen, 'dis': dis} optimizer = {'gen': opt_gen, 'dis': opt_dis} elif args.updater == 'standard': model_cls, initmodel_cls, lr = generators[args.generator] L_bce_weight = None print('# model: {}'.format(model_cls.__name__)) print('') if args.initgen_path: model, initmodel = model_cls(n_class), initmodel_cls(n_class) model = load_pretrained_model(args.initgen_path, initmodel, model, n_class, args.gpu) else: model = model_cls(n_class) if args.gpu >= 0: chainer.cuda.get_device( args.gpu).use() # Make a specified GPU current model.to_gpu() # Copy the model to the GPU optimizer = make_optimizer(model, lr) # Set up a trainer updater = updaters[args.updater]( model=model, iterator=train_iter, optimizer=optimizer, device=args.gpu, L_bce_weight=L_bce_weight, n_class=n_class, ) trainer = training.Trainer(updater, (args.iteration, 'iteration'), out=args.out) evaluate_interval = (args.evaluate_interval, 'iteration') snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend(TestModeEvaluator(val_iter, updater, device=args.gpu), trigger=snapshot_interval, invoke_before_training=True) trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) if args.updater == 'gan': trainer.extend(extensions.snapshot_object( gen, 'gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'iteration', 'gen/loss', 'validation/gen/loss', 'dis/loss', 'gen/accuracy', 'validation/gen/accuracy', 'gen/iu', 'validation/gen/iu', 'elapsed_time', ]), trigger=display_interval) elif args.updater == 'standard': trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'main/iu', 'validation/main/iu', 'elapsed_time', ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=1)) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) print('\nRun the training') trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--dataset', '-d', default='cifar10', help='The dataset to use: cifar10 or cifar100') parser.add_argument('--batchsize', '-b', type=int, default=64, help='Number of images in each mini-batch') parser.add_argument('--learnrate', '-l', type=float, default=0.05, help='Learning rate for SGD') parser.add_argument('--epoch', '-e', type=int, default=300, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train. # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. if args.dataset == 'cifar10': print('Using CIFAR10 dataset.') class_labels = 10 train, test = get_cifar10() elif args.dataset == 'cifar100': print('Using CIFAR100 dataset.') class_labels = 100 train, test = get_cifar100() else: raise RuntimeError('Invalid dataset choice.') model = L.Classifier(models.VGG.VGG(class_labels)) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU optimizer = chainer.optimizers.MomentumSGD(args.learnrate) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4)) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Reduce the learning rate by half every 25 epochs. trainer.extend(extensions.ExponentialShift('lr', 0.5), trigger=(25, 'epoch')) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def train(): parser = argparse.ArgumentParser(description='DAGMM') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--batchsize', '-b', type=int, default=128, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=10000, help='Number of sweeps over the dataset to train') parser.add_argument('--cn_h_unit', type=int, default=10, help='Number of Compression Network hidden units') parser.add_argument('--cn_z_unit', type=int, default=2, help='Number of Compression Network z units') parser.add_argument('--en_h_unit', type=int, default=10, help='Number of Estimation Network hidden units') parser.add_argument('--en_o_unit', type=int, default=2, help='Number of Estimation Network output units') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--frequency', '-f', type=int, default=20, help='Frequency of taking a snapshot') parser.add_argument( '--resume', '-r', type=int, help='Resume the training from snapshot that is designated epoch number' ) args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('# Compression Network: Dim - {0} - {1} - {0} - Dim'.format( args.cn_h_unit, args.cn_z_unit)) print('# Estimation Network: {} - {} - {}'.format(args.cn_z_unit + 2, args.en_h_unit, args.en_o_unit)) print('# Output-directory: {}'.format(args.out)) print('# Frequency-snapshot: {}'.format(args.frequency)) if args.resume: print('# Resume-epochNumber: {}'.format(args.resume)) print('') # データセット読み込み x_data = np.loadtxt('./dataset_arrhythmia/ExplanatoryVariables.csv', delimiter=',') y_label = np.loadtxt('./dataset_arrhythmia/CriterionVariables.csv', delimiter=',') # 正常データのみを抽出 HealthData = x_data[y_label[:] == 1] # 正常データを学習用と検証用に分割 NumOfHealthData = len(HealthData) trainData = HealthData[:math.floor(NumOfHealthData * 0.9)] validData = HealthData[len(trainData):] # 型変換 trainData = trainData.astype(np.float32) validData = validData.astype(np.float32) train_iter = chainer.iterators.SerialIterator(trainData, batch_size=args.batchsize, repeat=True, shuffle=True) valid_iter = chainer.iterators.SerialIterator(validData, batch_size=len(validData), repeat=False, shuffle=False) model = DAGMM(args.cn_h_unit, args.cn_z_unit, len(trainData[0]), args.en_h_unit, args.en_o_unit) if args.gpu >= 0: # Make a specified GPU current chainer.backends.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU optimizer = optimizers.Adam(alpha=0.0001) optimizer.setup(model) if args.resume: serializers.load_npz( args.out + '/model_snapshot_epoch_' + str(args.resume), model) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu, loss_func=model.lossFunc(gpu=args.gpu)) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend( extensions.Evaluator(valid_iter, model, device=args.gpu, eval_func=model.lossFunc(gpu=args.gpu))) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend( extensions.snapshot(filename='snapshot_epoch-{.updater.epoch}'), trigger=(args.frequency, 'epoch')) trainer.extend(extensions.snapshot_object( model, filename='model_snapshot_epoch_{.updater.epoch}'), trigger=(args.frequency, 'epoch')) trainer.extend(extensions.snapshot_object( optimizer, filename='optimizer_snapshot_epoch_{.updater.epoch}'), trigger=(args.frequency, 'epoch')) trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], x_key='epoch', file_name='loss1.png')) trainer.extend( extensions.PlotReport(['main/loss'], x_key='epoch', file_name='loss2.png')) trainer.extend(extensions.LogReport(log_name="log", trigger=(1, 'epoch'))) trainer.extend( extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time'])) trainer.extend(extensions.ProgressBar()) if args.resume: serializers.load_npz(args.out + '/snapshot_epoch-' + str(args.resume), trainer) trainer.run()
def main(): parser = argparse.ArgumentParser( description='chainer line drawing colorization') parser.add_argument('--batchsize', '-b', type=int, default=2, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=6, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--dataset', '-i', default='./images/', help='Directory of image files.') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--seed', type=int, default=0, help='Random seed') parser.add_argument('--snapshot_interval', type=int, default=10000, help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=3, help='Interval of displaying log to console') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') root = args.dataset #model = "./model_paint" cnn = unet.UNET() #serializers.load_npz("result/model_iter_10000", cnn) dis = unet.DIS() #serializers.load_npz("result/model_dis_iter_20000", dis) l = lnet.LNET() #serializers.load_npz("models/liner_f", l) dataset = Image2ImageDataset("dat/images_color_train.dat", root + "line/", root + "color/", train=True) # dataset.set_img_dict(img_dict) train_iter = chainer.iterators.SerialIterator(dataset, args.batchsize) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current cnn.to_gpu() # Copy the model to the GPU dis.to_gpu() # Copy the model to the GPU l.to_gpu() # Setup optimizer parameters. opt = optimizers.Adam(alpha=0.0001) opt.setup(cnn) opt.add_hook(chainer.optimizer.WeightDecay(1e-5), 'hook_cnn') opt_d = chainer.optimizers.Adam(alpha=0.0001) opt_d.setup(dis) opt_d.add_hook(chainer.optimizer.WeightDecay(1e-5), 'hook_dec') # Set up a trainer updater = ganUpdater( models=(cnn, dis, l), iterator={ 'main': train_iter, #'test': test_iter }, optimizer={ 'cnn': opt, 'dis': opt_d }, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) snapshot_interval = (args.snapshot_interval, 'iteration') snapshot_interval2 = (args.snapshot_interval * 2, 'iteration') trainer.extend(extensions.dump_graph('cnn/loss')) trainer.extend(extensions.snapshot(), trigger=snapshot_interval2) trainer.extend(extensions.snapshot_object( cnn, 'cnn_128_iter_{.updater.iteration}'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'cnn_128_dis_iter_{.updater.iteration}'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object(opt, 'optimizer_'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=(10, 'iteration'), )) trainer.extend( extensions.PrintReport([ 'epoch', 'cnn/loss', 'cnn/loss_rec', 'cnn/loss_adv', 'cnn/loss_tag', 'cnn/loss_l', 'dis/loss' ])) trainer.extend(extensions.ProgressBar(update_interval=20)) trainer.run() if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Save the trained model out_dir = args.out chainer.serializers.save_npz(os.path.join(out_dir, 'model_final'), cnn) chainer.serializers.save_npz(os.path.join(out_dir, 'optimizer_final'), opt)
def main(): parser = argparse.ArgumentParser(description='Chainer example: VAE') parser.add_argument('--initmodel', '-m', default='', help='Initialize the model from given file') parser.add_argument('--resume', '-r', default='', help='Resume the optimization from snapshot') parser.add_argument('--gpu', '-g', default=-1, type=int, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='results', help='Directory to output the result') parser.add_argument('--epoch', '-e', default=100, type=int, help='number of epochs to learn') parser.add_argument('--dim-z', '-z', default=20, type=int, help='dimention of encoded vector') parser.add_argument('--dim-h', default=500, type=int, help='dimention of hidden layer') parser.add_argument('--beta', default=1.0, type=float, help='Regularization coefficient for ' 'the second term of ELBO bound') parser.add_argument('--k', '-k', default=1, type=int, help='Number of Monte Carlo samples used in ' 'encoded vector') parser.add_argument('--binary', action='store_true', help='Use binarized MNIST') parser.add_argument('--batch-size', '-b', type=int, default=100, help='learning minibatch size') parser.add_argument('--test', action='store_true', help='Use tiny datasets for quick tests') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# dim z: {}'.format(args.dim_z)) print('# Minibatch-size: {}'.format(args.batch_size)) print('# epoch: {}'.format(args.epoch)) print('') # Prepare VAE model, defined in net.py encoder = net.make_encoder(784, args.dim_z, args.dim_h) decoder = net.make_decoder(784, args.dim_z, args.dim_h, binary_check=args.binary) prior = net.make_prior(args.dim_z) avg_elbo_loss = net.AvgELBOLoss(encoder, decoder, prior, beta=args.beta, k=args.k) if args.gpu >= 0: avg_elbo_loss.to_gpu(args.gpu) # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(avg_elbo_loss) # Initialize if args.initmodel: chainer.serializers.load_npz(args.initmodel, avg_elbo_loss) # Load the MNIST dataset train, test = chainer.datasets.get_mnist(withlabel=False) if args.binary: # Binarize dataset train = (train >= 0.5).astype(np.float32) test = (test >= 0.5).astype(np.float32) if args.test: train, _ = chainer.datasets.split_dataset(train, 100) test, _ = chainer.datasets.split_dataset(test, 100) train_iter = chainer.iterators.SerialIterator(train, args.batch_size) test_iter = chainer.iterators.SerialIterator(test, args.batch_size, repeat=False, shuffle=False) # Set up an updater. StandardUpdater can explicitly specify a loss function # used in the training with 'loss_func' option updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu, loss_func=avg_elbo_loss) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator( test_iter, avg_elbo_loss, device=args.gpu)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/reconstr', 'main/kl_penalty', 'elapsed_time'])) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run() # Visualize the results def save_images(x, filename): import matplotlib.pyplot as plt fig, ax = plt.subplots(3, 3, figsize=(9, 9), dpi=100) for ai, xi in zip(ax.flatten(), x): ai.imshow(xi.reshape(28, 28)) fig.savefig(filename) avg_elbo_loss.to_cpu() train_ind = [1, 3, 5, 10, 2, 0, 13, 15, 17] x = chainer.Variable(np.asarray(train[train_ind])) with chainer.using_config('train', False), chainer.no_backprop_mode(): x1 = decoder(encoder(x).mean, inference=True).mean save_images(x.array, os.path.join(args.out, 'train')) save_images(x1.array, os.path.join(args.out, 'train_reconstructed')) test_ind = [3, 2, 1, 18, 4, 8, 11, 17, 61] x = chainer.Variable(np.asarray(test[test_ind])) with chainer.using_config('train', False), chainer.no_backprop_mode(): x1 = decoder(encoder(x).mean, inference=True).mean save_images(x.array, os.path.join(args.out, 'test')) save_images(x1.array, os.path.join(args.out, 'test_reconstructed')) # draw images from randomly sampled z z = prior().sample(9) x = decoder(z, inference=True).mean save_images(x.array, os.path.join(args.out, 'sampled'))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--batchsize', '-b', type=int, default=20, help='Number of examples in each mini batch') parser.add_argument('--bproplen', '-l', type=int, default=35, help='Number of words in each mini batch ' '(= length of truncated BPTT)') parser.add_argument('--epoch', '-e', type=int, default=39, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--gradclip', '-c', type=float, default=5, help='Gradient norm threshold to clip') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--test', action='store_true', help='Use tiny datasets for quick tests') parser.set_defaults(test=False) parser.add_argument('--unit', '-u', type=int, default=650, help='Number of LSTM units in each layer') args = parser.parse_args() # Load the Penn Tree Bank long word sequence dataset train, val, test = chainer.datasets.get_ptb_words() n_vocab = max(train) + 1 # train is just an array of integers print('#vocab =', n_vocab) if args.test: train = train[:100] val = val[:100] test = test[:100] train_iter = ParallelSequentialIterator(train, args.batchsize) val_iter = ParallelSequentialIterator(val, 1, repeat=False) test_iter = ParallelSequentialIterator(test, 1, repeat=False) # Prepare an RNNLM model rnn = RNNForLM(n_vocab, args.unit) model = L.Classifier(rnn) model.compute_accuracy = False # we only want the perplexity if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # make the GPU current model.to_gpu() # Set up an optimizer optimizer = chainer.optimizers.SGD(lr=1.0) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.gradclip)) # Set up a trainer updater = BPTTUpdater(train_iter, optimizer, args.bproplen, args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) eval_model = model.copy() # Model with shared params and distinct states eval_rnn = eval_model.predictor eval_rnn.train = False trainer.extend(extensions.Evaluator( val_iter, eval_model, device=args.gpu, # Reset the RNN state at the beginning of each evaluation eval_hook=lambda _: eval_rnn.reset_state())) interval = 10 if args.test else 500 trainer.extend(extensions.LogReport(postprocess=compute_perplexity, trigger=(interval, 'iteration'))) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'perplexity', 'val_perplexity'] ), trigger=(interval, 'iteration')) trainer.extend(extensions.ProgressBar( update_interval=1 if args.test else 10)) trainer.extend(extensions.snapshot()) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}')) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run() # Evaluate the final model print('test') eval_rnn.reset_state() evaluator = extensions.Evaluator(test_iter, eval_model, device=args.gpu) result = evaluator() print('test perplexity:', np.exp(float(result['main/loss'])))
def main(): parser = argparse.ArgumentParser(description='Chainer example: VAE') parser.add_argument('--initmodel', '-m', default='', help='Initialize the model from given file') parser.add_argument('--resume', '-r', default='', help='Resume the optimization from snapshot') parser.add_argument('--gpu', '-g', default=-1, type=int, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--epoch', '-e', default=100, type=int, help='number of epochs to learn') parser.add_argument('--dim-hidden', '-u', default=500, type=int, help='dimention of hidden layers') parser.add_argument('--dimz', '-z', default=20, type=int, help='dimention of encoded vector') parser.add_argument('--batchsize', '-b', type=int, default=100, help='learning minibatch size') parser.add_argument('--test', action='store_true', help='Use tiny datasets for quick tests') parser.add_argument('--vqvae', action='store_true', help='Use VQVAE') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# dim z: {}'.format(args.dimz)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Prepare VAE model, defined in net.py if args.vqvae: model = net.VQVAE(784, args.dimz, args.dim_hidden) else: model = net.VAE(784, args.dimz, args.dim_hidden) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Setup an optimizer optimizer = chainer.optimizers.Adam(1e-4) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(5.)) # Initialize if args.initmodel: chainer.serializers.load_npz(args.initmodel, model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist(withlabel=False) if args.test: train, _ = chainer.datasets.split_dataset(train, 100) test, _ = chainer.datasets.split_dataset(test, 100) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up an updater. StandardUpdater can explicitly specify a loss function # used in the training with 'loss_func' option updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu, loss_func=model.get_loss_func()) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend( extensions.Evaluator(test_iter, model, device=args.gpu, eval_func=model.get_loss_func(k=10))) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'main/rec_loss', 'validation/main/rec_loss', 'main/other_loss', 'validation/main/other_loss', 'elapsed_time' ])) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) @chainer.training.make_extension() def confirm_images(trainer): # Visualize the results def save_images(x, filename): import matplotlib.pyplot as plt fig, ax = plt.subplots(3, 3, figsize=(9, 9), dpi=100) for ai, xi in zip(ax.flatten(), x): ai.imshow(xi.reshape(28, 28)) fig.savefig(filename) plt.close() model.to_cpu() train_ind = [1, 3, 5, 10, 2, 0, 13, 15, 17] x = chainer.Variable(np.asarray(train[train_ind])) with chainer.using_config('train', False), chainer.no_backprop_mode(): x1 = model(x) save_images( x.data, os.path.join(args.out, '{.updater.iteration}_train'.format(trainer))) save_images( x1.data, os.path.join( args.out, '{.updater.iteration}_train_reconstructed'.format(trainer))) test_ind = [3, 2, 1, 18, 4, 8, 11, 17, 61] x = chainer.Variable(np.asarray(test[test_ind])) with chainer.using_config('train', False), chainer.no_backprop_mode(): x1 = model(x) save_images( x.data, os.path.join(args.out, '{.updater.iteration}_test'.format(trainer))) save_images( x1.data, os.path.join( args.out, '{.updater.iteration}_test_reconstructed'.format(trainer))) # draw images from randomly sampled z if args.vqvae: z = model.sample(size=9) else: z = chainer.Variable( np.random.normal(0, 1, (9, args.dimz)).astype(np.float32)) x = model.decode(z) save_images( x.data, os.path.join(args.out, '{.updater.iteration}_sampled'.format(trainer))) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() trainer.extend(confirm_images, trigger=(args.epoch // 10, 'epoch')) # Run the training trainer.run()
def gan_training(args, train, test, model_path): # These iterators load the images with subprocesses running in parallel to # the training/validation. if args.loaderjob: train_iter = chainer.iterators.MultiprocessIterator( train, args.batchsize, n_processes=args.loaderjob) else: train_iter = chainer.iterators.SerialIterator(train, args.batchsize) # Prepare Flow and Texture GAN model, defined in net.py gen_flow = net.FlowGenerator() dis_flow = net.FlowDiscriminator() gen_tex = net.Generator(args.dimz) dis_tex = net.Discriminator() serializers.load_npz(model_path["gen_flow"], gen_flow) serializers.load_npz(model_path["dis_flow"], dis_flow) serializers.load_npz(model_path["gen_tex"], gen_tex) serializers.load_npz(model_path["dis_tex"], dis_tex) if args.gpu >= 0: cuda.get_device(args.gpu).use() gen_flow.to_gpu() dis_flow.to_gpu() gen_tex.to_gpu() dis_tex.to_gpu() xp = np if args.gpu < 0 else cuda.cupy opt_flow_gen = make_optimizer(gen_flow, args, alpha=1e-7) opt_flow_dis = make_optimizer(dis_flow, args, alpha=1e-7) opt_tex_gen = make_optimizer(gen_tex, args, alpha=1e-6) opt_tex_dis = make_optimizer(dis_tex, args, alpha=1e-6) # Updater updater = GAN_Updater(models=(gen_flow, dis_flow, gen_tex, dis_tex), iterator=train_iter, optimizer={ 'gen_flow': opt_flow_gen, 'dis_flow': opt_flow_dis, 'gen_tex': opt_tex_gen, 'dis_tex': opt_tex_dis }, device=args.gpu, C=args.C) trainer = training.Trainer(updater, (args.iteration, 'iteration'), out=args.out) snapshot_interval = (args.snapshot_interval), 'iteration' visualize_interval = (args.visualize_interval), 'iteration' log_interval = (args.log_interval), 'iteration' # Be careful to pass the interval directly to LogReport # (it determines when to emit log rather than when to read observations) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend( extensions.PlotReport( ['gen_flow/loss', 'dis_flow/loss', 'gen_tex/loss', 'dis_tex/loss'], trigger=log_interval, file_name='plot.png')) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'gen_flow/loss', 'dis_flow/loss', 'gen_tex/loss', 'dis_tex/loss' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(extensions.snapshot(), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen_flow, 'gen_flow_iteration_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis_flow, 'dis_flow_iteration_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen_tex, 'gen_tex_iteration_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis_tex, 'dis_tex_iteration_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(visualizer.extension(test, (gen_flow, gen_tex), args, rows=args.rows, cols=args.cols), trigger=visualize_interval) if args.adam_decay_iteration: trainer.extend(extensions.ExponentialShift("alpha", 0.5, optimizer=opt_flow_gen), trigger=(args.adam_decay_iteration, 'iteration')) trainer.extend(extensions.ExponentialShift("alpha", 0.5, optimizer=opt_flow_dis), trigger=(args.adam_decay_iteration, 'iteration')) trainer.extend(extensions.ExponentialShift("alpha", 0.5, optimizer=opt_tex_gen), trigger=(args.adam_decay_iteration, 'iteration')) trainer.extend(extensions.ExponentialShift("alpha", 0.5, optimizer=opt_tex_dis), trigger=(args.adam_decay_iteration, 'iteration')) if args.resume: chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser( description='chainer implementation of pix2pix') parser.add_argument('--batchsize', '-b', type=int, default=1, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=200, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--snapshot_interval', type=int, default=100, help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=100, help='Interval of displaying log to console') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train enc = Encoder(in_ch=3) dec = Decoder(out_ch=1) dis = Discriminator(in_ch=3, out_ch=1) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current enc.to_gpu() # Copy the model to the GPU dec.to_gpu() dis.to_gpu() # Setup an optimizer def make_optimizer(model, alpha=0.0002, beta1=0.5): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.00001), 'hook_dec') return optimizer opt_enc = make_optimizer(enc) opt_dec = make_optimizer(dec) opt_dis = make_optimizer(dis) train_d = P2BDataset(data_range=(1, 11)) test_d = P2BDataset(data_range=(11, 12)) train_iter = chainer.iterators.SerialIterator(train_d, args.batchsize) test_iter = chainer.iterators.SerialIterator(test_d, args.batchsize) # Set up a trainer updater = P2BUpdater(models=(enc, dec, dis), iterator={ 'main': train_iter, 'test': test_iter }, optimizer={ 'enc': opt_enc, 'dec': opt_dec, 'dis': opt_dis }, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=str(RESULT_PATH)) snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( enc, 'enc_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dec, 'dec_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'enc/loss', 'dec/loss', 'dis/loss', ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run() chainer.serializers.save_npz(MODEL_PATH.joinpath('dec_model.npz'), dec) chainer.serializers.save_npz(MODEL_PATH.joinpath('enc_model.npz'), enc)
def get_trainer( dataset_class, gpu, max_iter, out=None, resume=None, interval_log=10, interval_eval=1000, optimizer=None, batch_size=1, ): if isinstance(gpu, list): gpus = gpu else: gpus = [gpu] if out is None: out = tempfile.mktemp() if optimizer is None: optimizer = chainer.optimizers.MomentumSGD(lr=1e-10, momentum=0.99) if not resume and osp.exists(out): print('Result dir already exists: {}'.format(osp.abspath(out)), file=sys.stderr) quit(1) # dump params params = { 'dataset_class': dataset_class.__name__, 'gpu': {'ids': gpu}, 'batch_size': batch_size, 'resume': resume, 'optimizer': { 'name': optimizer.__class__.__name__, 'params': optimizer.__dict__, } } print('>' * 20 + ' Parameters ' + '>' * 20) yaml.safe_dump(params, sys.stderr, default_flow_style=False) print('<' * 20 + ' Parameters ' + '<' * 20) if not osp.exists(out): os.makedirs(out) yaml.safe_dump(params, open(osp.join(out, 'param.yaml'), 'w'), default_flow_style=False) # 1. dataset dataset_train = dataset_class('train') dataset_val = dataset_class('val') if len(gpus) > 1: iter_train = chainer.iterators.MultiprocessIterator( dataset_train, batch_size=batch_size*len(gpus), shared_mem=10000000) else: iter_train = chainer.iterators.SerialIterator( dataset_train, batch_size=batch_size) iter_val = chainer.iterators.SerialIterator( dataset_val, batch_size=batch_size, repeat=False, shuffle=False) # 2. model vgg_path = data.download_vgg16_chainermodel() vgg = models.VGG16() chainer.serializers.load_hdf5(vgg_path, vgg) n_class = len(dataset_train.label_names) model = models.FCN32s(n_class=n_class) model.train = True utils.copy_chainermodel(vgg, model) if len(gpus) > 1 or gpus[0] >= 0: chainer.cuda.get_device(gpus[0]).use() if len(gpus) == 1 and gpus[0] >= 0: model.to_gpu() # 3. optimizer optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005)) # 4. trainer if len(gpus) > 1: devices = {'main': gpus[0]} for gpu in gpus[1:]: devices['gpu{}'.format(gpu)] = gpu updater = chainer.training.ParallelUpdater( iter_train, optimizer, devices=devices) else: updater = chainer.training.StandardUpdater( iter_train, optimizer, device=gpus[0]) trainer = chainer.training.Trainer( updater, (max_iter, 'iteration'), out=out) trainer.extend( training.extensions.TestModeEvaluator( iter_val, model, device=gpus[0]), trigger=(interval_eval, 'iteration'), invoke_before_training=False, ) def visualize_segmentation(target): datum = chainer.cuda.to_cpu(target.x.data[0]) img = dataset_val.datum_to_img(datum) label_true = chainer.cuda.to_cpu(target.t.data[0]) label_pred = chainer.cuda.to_cpu(target.score.data[0]).argmax(axis=0) label_pred[label_true == -1] = 0 cmap = utils.labelcolormap(len(dataset_val.label_names)) label_viz0 = skimage.color.label2rgb( label_pred, colors=cmap[1:], bg_label=0) label_viz0[label_true == -1] = (0, 0, 0) label_viz0 = (label_viz0 * 255).astype(np.uint8) label_viz1 = skimage.color.label2rgb( label_pred, img, colors=cmap[1:], bg_label=0) label_viz1[label_true == -1] = (0, 0, 0) label_viz1 = (label_viz1 * 255).astype(np.uint8) return utils.get_tile_image([img, label_viz0, label_viz1]) trainer.extend( training.extensions.ImageVisualizer( iter_val, model, viz_func=visualize_segmentation, device=gpus[0]), trigger=(interval_eval, 'iteration'), invoke_before_training=True, ) model_name = model.__class__.__name__ trainer.extend(extensions.snapshot( savefun=chainer.serializers.hdf5.save_hdf5, filename='%s_trainer_iter_{.updater.iteration}.h5' % model_name, trigger=(interval_eval, 'iteration'))) trainer.extend(extensions.snapshot_object( model, savefun=chainer.serializers.hdf5.save_hdf5, filename='%s_model_iter_{.updater.iteration}.h5' % model_name, trigger=(interval_eval, 'iteration'))) trainer.extend(extensions.LogReport( trigger=(interval_log, 'iteration'), log_name='log.json')) trainer.extend(extensions.PrintReport([ 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'main/iu', 'validation/main/iu', 'elapsed_time', ])) trainer.extend(extensions.ProgressBar(update_interval=1)) if resume: if resume.endswith('npz'): chainer.serializers.load_npz(resume, trainer) else: chainer.serializers.load_hdf5(resume, trainer) return trainer
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=5, help='Number of sweeps over the dataset to train') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='output', help='Directory to output the graph descriptor and sample test data') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=100, help='Number of units') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') os.makedirs(args.out, exist_ok=True) # Set up a neural network to train # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model = L.Classifier(MLP(args.unit, 10)) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=os.path.join(args.out, 'chainer_model')) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run() # conversion print('Transpiling model to WebDNN graph descriptor') example_input = numpy.expand_dims(train[0][0], axis=0) # example input (anything ok, (batch_size, 784)) x = chainer.Variable(example_input) y = model.predictor(x) # run model (without softmax) graph = ChainerGraphConverter().convert_from_inout_vars([x], [y]) # convert graph to intermediate representation for backend in ["webgpu", "webassembly", "fallback"]: try: exec_info = generate_descriptor(backend, graph) exec_info.save(args.out) except Exception as ex: print(f"Failed generating descriptor for backend {backend}: {str(ex)}\n") else: print(f"Backend {backend} ok\n") print('Exporting test samples (for demo purpose)') test_samples_json = [] for i in range(10): image, label = test[i] test_samples_json.append({'x': image.tolist(), 'y': int(label)}) with open(os.path.join(args.out, 'test_samples.json'), 'w') as f: json.dump(test_samples_json, f)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', default=-1, type=int, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', type=str, help='Directory to ouput the result') parser.add_argument('--resume', '-r', type=str, help='Resume the training from snapshot') parser.add_argument('--epoch', '-e', default=400, type=int, help='number of epochs to learn') parser.add_argument('--unit', '-u', default=30, type=int, help='number of units') parser.add_argument('--batchsize', '-b', type=int, default=25, help='learning minibatch size') parser.add_argument('--label', '-l', type=int, default=5, help='number of labels') parser.add_argument('--epocheval', '-p', type=int, default=5, help='number of epochs per evaluation') parser.add_argument('--test', dest='test', action='store_true') parser.set_defaults(test=False) args = parser.parse_args() n_epoch = args.epoch # number of epochs n_units = args.unit # number of units per layer batchsize = args.batchsize # minibatch size n_label = args.label # number of labels epoch_per_eval = args.epocheval # number of epochs per evaluation if args.test: max_size = 10 else: max_size = None vocab = {} train_data = [convert_tree(vocab, tree) for tree in data.read_corpus('trees/train.txt', max_size)] train_iter = chainer.iterators.SerialIterator(train_data, batchsize) validation_data = [convert_tree(vocab, tree) for tree in data.read_corpus('trees/dev.txt', max_size)] validation_iter = chainer.iterators.SerialIterator( validation_data, batchsize, repeat=False, shuffle=False) test_data = [convert_tree(vocab, tree) for tree in data.read_corpus('trees/test.txt', max_size)] model = RecursiveNet(len(vocab), n_units, n_label) if args.gpu >= 0: cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Setup optimizer optimizer = optimizers.AdaGrad(lr=0.1) optimizer.setup(model) optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(0.0001)) def _convert(batch, _): return batch # Setup updater updater = chainer.training.StandardUpdater( train_iter, optimizer, device=args.gpu, converter=_convert) # Setup trainer and run trainer = chainer.training.Trainer(updater, (n_epoch, 'epoch'), args.out) trainer.extend( extensions.Evaluator(validation_iter, model, device=args.gpu, converter=_convert), trigger=(epoch_per_eval, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.MicroAverage( 'main/correct', 'main/total', 'main/accuracy')) trainer.extend(extensions.MicroAverage( 'validation/main/correct', 'validation/main/total', 'validation/main/accuracy')) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) trainer.extend( extensions.snapshot(filename='snapshot_epoch_{.updater.epoch}'), trigger=(epoch_per_eval, 'epoch')) if args.resume is not None: chainer.serializers.load_npz(args.resume, trainer) trainer.run() print('Test evaluation') evaluate(model, test_data)
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result_u100', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=100, help='Number of units') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model = mlp.MLP(args.unit, 10) classifier_model = L.Classifier(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() # Make a specified GPU current classifier_model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(classifier_model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, classifier_model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch #trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.snapshot(), trigger=(1, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) if extensions.PlotReport.available(): # Plot graph for loss for each epoch trainer.extend(extensions.PlotReport( ['main/loss', 'validation/main/loss'], x_key='epoch', file_name='loss.png')) trainer.extend(extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], x_key='epoch', file_name='accuracy.png')) # Print a progress bar to stdout #trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot serializers.load_npz(args.resume, trainer) # Run the training trainer.run() serializers.save_npz('{}/mlp.model'.format(args.out), model) serializers.save_npz('{}/clf.model'.format(args.out), classifier_model)
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--dataset', '-d', default='cifar100', help='The dataset to use: cifar10 or cifar100') parser.add_argument('--model', '-m', default='VGG16', help='The model to use: VGG16 or PreResNet110' ' or WideResNet28x10') parser.add_argument('--batchsize', '-b', type=int, default=128, help='Number of images in each mini-batch') parser.add_argument('--lr_init', '-l', type=float, default=0.05, help='Learning rate for SGD') parser.add_argument('--epoch', '-e', type=int, default=200, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--wd', type=float, default=1e-4, help='weight decay') parser.add_argument('--swa', action='store_true', help='swa usage flag') parser.add_argument('--swa_start', type=float, default=161, help='SWA start epoch number') parser.add_argument('--swa_lr', type=float, default=0.05, help='SWA LR') parser.add_argument('--swa_c_epochs', type=int, default=1, help='SWA model collection frequency length in epochs') args = parser.parse_args() if args.dataset.lower() == 'cifar10': print('Using CIFAR10 dataset') class_labels = 10 train, test = get_cifar10() elif args.dataset.lower() == 'cifar100': print('Using CIFAR100 dataset') class_labels = 100 train, test = get_cifar100() else: raise RuntimeError('Invalid dataset choice.') print('Using %s model' % args.model) if args.model == 'VGG16': model_cls = VGG16 elif args.model == 'PreResNet110': model_cls = PreResNet110 elif args.model == 'WideResNet28x10': model_cls = WideResNet28x10 else: raise RuntimeError('Invalid model choice.') model = L.Classifier(model_cls(class_labels)) if args.swa: swa_model = L.Classifier(model_cls(class_labels)) swa_n = 0 if args.gpu >= 0: chainer.backends.cuda.get_device_from_id(args.gpu).use() model.to_gpu() if args.swa: swa_model.to_gpu() # Data augmentation / preprocess train = TransformDataset(train, partial(transform, train=True)) test = TransformDataset(test, partial(transform, train=False)) optimizer = chainer.optimizers.MomentumSGD(args.lr_init, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(args.wd)) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) swa_train_iter = chainer.iterators.SerialIterator( train, args.batchsize, repeat=False, shuffle=False) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) stop_trigger = (args.epoch, 'epoch') # Set up a trainer updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, stop_trigger, out=args.out) # Learning rate adjustment (this function is called every epoch) def lr_schedule(trainer): epoch = trainer.updater.epoch t = epoch / (args.swa_start if args.swa else args.epoch) lr_ratio = args.swa_lr / args.lr_init if args.swa else 0.01 if t <= 0.5: factor = 1.0 elif t <= 0.9: factor = 1.0 - (1.0 - lr_ratio) * (t - 0.5) / 0.4 else: factor = lr_ratio trainer.updater.get_optimizer('main').lr = factor * args.lr_init # The main function for SWA (this function is called every epoch) def avg_weight(trainer): epoch = trainer.updater.epoch if args.swa and (epoch + 1) >= args.swa_start and \ (epoch + 1 - args.swa_start) % args.swa_c_epochs == 0: nonlocal swa_n # moving average alpha = 1.0 / (swa_n + 1) for param1, param2 in zip(swa_model.params(), model.params()): param1.data *= (1.0 - alpha) param1.data += param2.data * alpha swa_n += 1 # This funtion is called before evaluating SWA model # for fixing batchnorm's running mean and variance def fix_swa_batchnorm(evaluator): # Check batchnorm layer bn_flg = False for l in swa_model.links(): if type(l) == L.normalization.batch_normalization.BatchNormalization: bn_flg = True break # Fix batchnorm's running mean and variance if bn_flg: swa_train_iter.reset() with chainer.using_config('train', True): for batch in swa_train_iter: in_arrays = evaluator.converter(batch, evaluator.device) with function.no_backprop_mode(): swa_model(*in_arrays) # Set up extentions trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu), trigger=(5, 'epoch')) if args.swa: eval_points = [x for x in range(args.epoch + 1) if x > args.swa_start and x % 5 == 0] trainer.extend(SwaEvaluator(test_iter, swa_model, device=args.gpu, eval_hook=fix_swa_batchnorm), trigger=triggers.ManualScheduleTrigger(eval_points, 'epoch')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(lr_schedule, trigger=triggers.IntervalTrigger(1, 'epoch')) trainer.extend(avg_weight, trigger=triggers.IntervalTrigger(1, 'epoch')) trainer.extend(extensions.observe_lr()) trainer.extend(extensions.LogReport()) cols = ['epoch', 'lr', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time'] if args.swa: cols = cols[:-1] + ['swa/main/loss', 'swa/main/accuracy'] + cols[-1:] trainer.extend(extensions.PrintReport(cols)) trainer.extend(extensions.ProgressBar()) trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--train', default='train.txt', type=str, help='File name of train data') parser.add_argument('--test', default='validation.txt', type=str, help='File name of validation data') parser.add_argument('--root', '-R', default='.', help='Root directory path of image files') parser.add_argument('--batchsize', '-b', type=int, default=128, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--mean', default=None, help='mean file (computed by compute_mean.py)') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train. # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. train = image_dataset.ImageDataset(args.train, args.root, max_size=128, mean=args.mean) test = image_dataset.ImageDataset(args.test, args.root, max_size=128, mean=args.mean) model = L.Classifier(alexnet.FromCaffeAlexnet(1), lossfun=F.mean_squared_error) original_model = pickle.load(open('alexnet.pkl', 'rb')) copy_model(original_model, model.predictor) model.compute_accuracy = False if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current model.to_gpu() # Copy the model to the GPU optimizer = chainer.optimizers.Adam() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4)) optimizer.add_hook( DelGradient(["conv1", "conv2", "conv3", "conv4", "conv5"])) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(TestModeEvaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=(args.epoch, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend( extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: DCGAN') parser.add_argument('--batchsize', '-b', type=int, default=50, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=1000, help='Number of sweeps over the dataset to train') parser.add_argument('--device', '-d', type=str, default='-1', help='Device specifier. Either ChainerX device ' 'specifier or an integer. If non-negative integer, ' 'CuPy arrays with specified device id are used. If ' 'negative integer, NumPy arrays are used') parser.add_argument('--dataset', '-i', default='', help='Directory of image files. Default is cifar-10.') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', type=str, help='Resume the training from snapshot') parser.add_argument('--n_hidden', '-n', type=int, default=100, help='Number of hidden units (z)') parser.add_argument('--seed', type=int, default=0, help='Random seed of z at visualization stage') parser.add_argument('--snapshot_interval', type=int, default=1000, help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=100, help='Interval of displaying log to console') group = parser.add_argument_group('deprecated arguments') group.add_argument('--gpu', '-g', dest='device', type=int, nargs='?', const=0, help='GPU ID (negative value indicates CPU)') args = parser.parse_args() device = chainer.get_device(args.device) device.use() print('Device: {}'.format(device)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# n_hidden: {}'.format(args.n_hidden)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train gen = Generator(n_hidden=args.n_hidden) dis = Discriminator() gen.to_device(device) # Copy the model to the device dis.to_device(device) # Setup an optimizer def make_optimizer(model, alpha=0.0002, beta1=0.5): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1) optimizer.setup(model) optimizer.add_hook( chainer.optimizer_hooks.WeightDecay(0.0001), 'hook_dec') return optimizer opt_gen = make_optimizer(gen) opt_dis = make_optimizer(dis) if args.dataset == '': # Load the CIFAR10 dataset if args.dataset is not specified train, _ = chainer.datasets.get_cifar10(withlabel=False, scale=255.) else: all_files = os.listdir(args.dataset) image_files = [f for f in all_files if ('png' in f or 'jpg' in f)] print('{} contains {} image files' .format(args.dataset, len(image_files))) train = chainer.datasets\ .ImageDataset(paths=image_files, root=args.dataset) # Setup an iterator train_iter = chainer.iterators.SerialIterator(train, args.batchsize) # Setup an updater updater = DCGANUpdater( models=(gen, dis), iterator=train_iter, optimizer={ 'gen': opt_gen, 'dis': opt_dis}, device=device) # Setup a trainer trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen, 'gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'gen/loss', 'dis/loss', ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend( out_generated_image( gen, dis, 10, 10, args.seed, args.out), trigger=snapshot_interval) if args.resume is not None: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def train(args): '''RUN TRAINING''' # seed setting torch.manual_seed(args.seed) # use determinisitic computation or not if args.debugmode < 1: torch.backends.cudnn.deterministic = False logging.info('torch cudnn deterministic is disabled') else: torch.backends.cudnn.deterministic = True # check cuda availability if not torch.cuda.is_available(): logging.warning('cuda is not available') # get input and output dimension info with open(args.valid_json, 'rb') as f: valid_json = json.load(f)['utts'] utts = list(valid_json.keys()) # reverse input and output dimension idim = int(valid_json[utts[0]]['output'][0]['shape'][1]) odim = int(valid_json[utts[0]]['input'][0]['shape'][1]) if args.use_speaker_embedding: args.spk_embed_dim = int(valid_json[utts[0]]['input'][1]['shape'][0]) else: args.spk_embed_dim = None logging.info('#input dims : ' + str(idim)) logging.info('#output dims: ' + str(odim)) # write model config if not os.path.exists(args.outdir): os.makedirs(args.outdir) model_conf = args.outdir + '/model.conf' with open(model_conf, 'wb') as f: logging.info('writing a model config file to' + model_conf) pickle.dump((idim, odim, args), f) for key in sorted(vars(args).keys()): logging.info('ARGS: ' + key + ': ' + str(vars(args)[key])) # define output activation function if args.output_activation is None: output_activation_fn = None elif hasattr(torch.nn.functional, args.output_activation): output_activation_fn = getattr(torch.nn.functional, args.output_activation) else: raise ValueError('there is no such an activation function. (%s)' % args.output_activation) # specify model architecture tacotron2 = Tacotron2(idim=idim, odim=odim, spk_embed_dim=args.spk_embed_dim, embed_dim=args.embed_dim, elayers=args.elayers, eunits=args.eunits, econv_layers=args.econv_layers, econv_chans=args.econv_chans, econv_filts=args.econv_filts, dlayers=args.dlayers, dunits=args.dunits, prenet_layers=args.prenet_layers, prenet_units=args.prenet_units, postnet_layers=args.postnet_layers, postnet_chans=args.postnet_chans, postnet_filts=args.postnet_filts, output_activation_fn=output_activation_fn, adim=args.adim, aconv_chans=args.aconv_chans, aconv_filts=args.aconv_filts, cumulate_att_w=args.cumulate_att_w, use_batch_norm=args.use_batch_norm, use_concate=args.use_concate, dropout=args.dropout_rate, zoneout=args.zoneout_rate) logging.info(tacotron2) # Set gpu ngpu = args.ngpu if ngpu == 1: gpu_id = range(ngpu) logging.info('gpu id: ' + str(gpu_id)) tacotron2.cuda() elif ngpu > 1: gpu_id = range(ngpu) logging.info('gpu id: ' + str(gpu_id)) tacotron2 = torch.nn.DataParallel(tacotron2, device_ids=gpu_id) tacotron2.cuda() logging.info('batch size is automatically increased (%d -> %d)' % (args.batch_size, args.batch_size * ngpu)) args.batch_size *= ngpu else: gpu_id = [-1] # define loss model = Tacotron2Loss(model=tacotron2, use_masking=args.use_masking, bce_pos_weight=args.bce_pos_weight) reporter = model.reporter # Setup an optimizer optimizer = torch.optim.Adam(model.parameters(), args.lr, eps=args.eps, weight_decay=args.weight_decay) # FIXME: TOO DIRTY HACK setattr(optimizer, 'target', reporter) setattr(optimizer, 'serialize', lambda s: reporter.serialize(s)) # read json data with open(args.train_json, 'rb') as f: train_json = json.load(f)['utts'] with open(args.valid_json, 'rb') as f: valid_json = json.load(f)['utts'] # make minibatch list (variable length) train_batchset = make_batchset(train_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches, args.batch_sort_key) valid_batchset = make_batchset(valid_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches, args.batch_sort_key) # hack to make batchsze argument as 1 # actual bathsize is included in a list train_iter = chainer.iterators.SerialIterator(train_batchset, 1) valid_iter = chainer.iterators.SerialIterator(valid_batchset, 1, repeat=False, shuffle=False) # Set up a trainer converter = CustomConverter(gpu_id, True, args.use_speaker_embedding) updater = CustomUpdater(model, args.grad_clip, train_iter, optimizer, converter) trainer = training.Trainer(updater, (args.epochs, 'epoch'), out=args.outdir) # Resume from a snapshot if args.resume: logging.info('restored from %s' % args.resume) chainer.serializers.load_npz(args.resume, trainer) torch_load(args.outdir + '/model.ep.%d' % trainer.updater.epoch, tacotron2) model = trainer.updater.model # Evaluate the model with the test dataset for each epoch trainer.extend(CustomEvaluator(model, valid_iter, reporter, converter)) # Take a snapshot for each specified epoch trainer.extend( extensions.snapshot(filename='snapshot.ep.{.updater.epoch}'), trigger=(1, 'epoch')) # Save attention figure for each epoch if args.num_save_attention > 0: data = sorted(valid_json.items()[:args.num_save_attention], key=lambda x: int(x[1]['input'][0]['shape'][1]), reverse=True) trainer.extend(PlotAttentionReport( tacotron2, data, args.outdir + '/att_ws', CustomConverter(gpu_id, False, args.use_speaker_embedding), True), trigger=(1, 'epoch')) # Make a plot for training and validation values trainer.extend( extensions.PlotReport([ 'main/loss', 'validation/main/loss', 'main/l1_loss', 'validation/main/l1_loss', 'main/mse_loss', 'validation/main/mse_loss', 'main/bce_loss', 'validation/main/bce_loss' ], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(['main/l1_loss', 'validation/main/l1_loss'], 'epoch', file_name='l1_loss.png')) trainer.extend( extensions.PlotReport(['main/mse_loss', 'validation/main/mse_loss'], 'epoch', file_name='mse_loss.png')) trainer.extend( extensions.PlotReport(['main/bce_loss', 'validation/main/bce_loss'], 'epoch', file_name='bce_loss.png')) # Save model for each epoch trainer.extend(extensions.snapshot_object(tacotron2, 'model.ep.{.updater.epoch}', savefun=torch_save), trigger=(1, 'epoch')) # Save best models trainer.extend( extensions.snapshot_object(tacotron2, 'model.loss.best', savefun=torch_save), trigger=training.triggers.MinValueTrigger('validation/main/loss')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport(trigger=(100, 'iteration'))) report_keys = [ 'epoch', 'iteration', 'elapsed_time', 'main/loss', 'main/l1_loss', 'main/mse_loss', 'main/bce_loss', 'validation/main/loss', 'validation/main/l1_loss', 'validation/main/mse_loss', 'validation/main/bce_loss' ] trainer.extend(extensions.PrintReport(report_keys), trigger=(100, 'iteration')) trainer.extend(extensions.ProgressBar()) # Run the training trainer.run()
def main(): # This script is almost identical to train_mnist.py. The only difference is # that this script uses data-parallel computation on two GPUs. # See train_mnist.py for more details. parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=400, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu0', '-g', type=int, default=0, help='First GPU ID') parser.add_argument('--gpu1', '-G', type=int, default=1, help='Second GPU ID') parser.add_argument('--out', '-o', default='result_parallel', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=1000, help='Number of units') parser.add_argument('--train_imgs', default='data/kmnist-train-imgs.npz', help='Path to kmnist training images') parser.add_argument('--train_label', default='data/kmnist-train-labels.npz', help='Path to kmnist training labels') parser.add_argument('--test_imgs', default='data/kmnist-test-imgs.npz', help='Path to kmnist test images') parser.add_argument('--test_label', default='data/kmnist-test-labels.npz', help='Path to kmnist test labels') args = parser.parse_args() print('GPU: {}, {}'.format(args.gpu0, args.gpu1)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') chainer.backends.cuda.get_device_from_id(args.gpu0).use() model = L.Classifier(train_kmnist.MLP(args.unit, 10)) optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load and prepare the KMNIST dataset train_data = np.load(args.train_imgs)['arr_0'].\ reshape((60000, 784)).astype(np.float32)/255. train_labels = [int(n) for n in np.load(args.train_label)['arr_0']] train = TupleDataset(train_data, train_labels) test_data = np.load(args.test_imgs)['arr_0'].\ reshape((10000, 784)).astype(np.float32)/255. test_labels = [int(n) for n in np.load(args.test_label)['arr_0']] test = TupleDataset(test_data, test_labels) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # ParallelUpdater implements the data-parallel gradient computation on # multiple GPUs. It accepts "devices" argument that specifies which GPU to # use. updater = training.updaters.ParallelUpdater( train_iter, optimizer, # The device of the name 'main' is used as a "master", while others are # used as slaves. Names other than 'main' are arbitrary. devices={ 'main': args.gpu0, 'second': args.gpu1 }, ) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu0)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ])) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def main(): # Supported preprocessing/network list method_list = [ 'nfp', 'ggnn', 'schnet', 'weavenet', 'rsgcn', 'relgcn', 'relgat' ] label_names = D.get_tox21_label_names() iterator_type = ['serial', 'balanced'] parser = argparse.ArgumentParser( description='Multitask Learning with Tox21.') parser.add_argument('--method', '-m', type=str, choices=method_list, default='nfp', help='graph convolution model to use ' 'as a predictor.') parser.add_argument('--label', '-l', type=str, choices=label_names, default='', help='target label for logistic ' 'regression. Use all labels if this option ' 'is not specified.') parser.add_argument('--iterator-type', type=str, choices=iterator_type, default='serial', help='iterator type. If `balanced` ' 'is specified, data is sampled to take same number of' 'positive/negative labels during training.') parser.add_argument('--eval-mode', type=int, default=1, help='Evaluation mode.' '0: only binary_accuracy is calculated.' '1: binary_accuracy and ROC-AUC score is calculated') parser.add_argument('--conv-layers', '-c', type=int, default=4, help='number of convolution layers') parser.add_argument('--batchsize', '-b', type=int, default=32, help='batch size') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID to use. Negative value indicates ' 'not to use GPU and to run the code in CPU.') parser.add_argument('--out', '-o', type=str, default='result', help='path to output directory') parser.add_argument('--epoch', '-e', type=int, default=10, help='number of epochs') parser.add_argument('--unit-num', '-u', type=int, default=16, help='number of units in one layer of the model') parser.add_argument('--resume', '-r', type=str, default='', help='path to a trainer snapshot') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--protocol', type=int, default=2, help='protocol version for pickle') parser.add_argument('--model-filename', type=str, default='classifier.pkl', help='file name for pickled model') parser.add_argument('--num-data', type=int, default=-1, help='Number of data to be parsed from parser.' '-1 indicates to parse all data.') args = parser.parse_args() method = args.method if args.label: labels = args.label class_num = len(labels) if isinstance(labels, list) else 1 else: labels = None class_num = len(label_names) # Dataset preparation train, val, _ = data.load_dataset(method, labels, num_data=args.num_data) # Network predictor_ = set_up_predictor(method, args.unit_num, args.conv_layers, class_num) iterator_type = args.iterator_type if iterator_type == 'serial': train_iter = I.SerialIterator(train, args.batchsize) elif iterator_type == 'balanced': if class_num > 1: raise ValueError('BalancedSerialIterator can be used with only one' 'label classification, please specify label to' 'be predicted by --label option.') train_iter = BalancedSerialIterator(train, args.batchsize, train.features[:, -1], ignore_labels=-1) train_iter.show_label_stats() else: raise ValueError('Invalid iterator type {}'.format(iterator_type)) val_iter = I.SerialIterator(val, args.batchsize, repeat=False, shuffle=False) classifier = Classifier(predictor_, lossfun=F.sigmoid_cross_entropy, metrics_fun=F.binary_accuracy, device=args.gpu) optimizer = O.Adam() optimizer.setup(classifier) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu, converter=concat_mols) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend( E.Evaluator(val_iter, classifier, device=args.gpu, converter=concat_mols)) trainer.extend(E.LogReport()) eval_mode = args.eval_mode if eval_mode == 0: trainer.extend( E.PrintReport([ 'epoch', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time' ])) elif eval_mode == 1: train_eval_iter = I.SerialIterator(train, args.batchsize, repeat=False, shuffle=False) trainer.extend( ROCAUCEvaluator(train_eval_iter, classifier, eval_func=predictor_, device=args.gpu, converter=concat_mols, name='train', pos_labels=1, ignore_labels=-1, raise_value_error=False)) # extension name='validation' is already used by `Evaluator`, # instead extension name `val` is used. trainer.extend( ROCAUCEvaluator(val_iter, classifier, eval_func=predictor_, device=args.gpu, converter=concat_mols, name='val', pos_labels=1, ignore_labels=-1)) trainer.extend( E.PrintReport([ 'epoch', 'main/loss', 'main/accuracy', 'train/main/roc_auc', 'validation/main/loss', 'validation/main/accuracy', 'val/main/roc_auc', 'elapsed_time' ])) else: raise ValueError('Invalid accfun_mode {}'.format(eval_mode)) trainer.extend(E.ProgressBar(update_interval=10)) frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(E.snapshot(), trigger=(frequency, 'epoch')) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run() config = { 'method': args.method, 'conv_layers': args.conv_layers, 'unit_num': args.unit_num, 'labels': args.label } with open(os.path.join(args.out, 'config.json'), 'w') as o: o.write(json.dumps(config)) classifier.save_pickle(os.path.join(args.out, args.model_filename), protocol=args.protocol)
def main(): # Parse the arguments. args = parse_arguments() # Set up some useful variables that will be used later on. method = args.method if args.label != 'all': labels = args.label cache_dir = os.path.join('input', '{}_{}'.format(method, labels)) class_num = len(labels) if isinstance(labels, list) else 1 else: labels = None cache_dir = os.path.join('input', '{}_all'.format(method)) class_num = len(D.get_qm9_label_names()) # Get the filename corresponding to the cached dataset, based on the amount # of data samples that need to be parsed from the original dataset. num_data = args.num_data if num_data >= 0: dataset_filename = 'data_{}.npz'.format(num_data) else: dataset_filename = 'data.npz' # Load the cached dataset. dataset_cache_path = os.path.join(cache_dir, dataset_filename) dataset = None if os.path.exists(dataset_cache_path): print('Loading cached dataset from {}.'.format(dataset_cache_path)) dataset = NumpyTupleDataset.load(dataset_cache_path) if dataset is None: print('Preprocessing dataset...') preprocessor = preprocess_method_dict[method]() if num_data >= 0: # Select the first `num_data` samples from the dataset. target_index = numpy.arange(num_data) dataset = D.get_qm9(preprocessor, labels=labels, target_index=target_index) else: # Load the entire dataset. dataset = D.get_qm9(preprocessor, labels=labels) # Cache the laded dataset. if not os.path.exists(cache_dir): os.makedirs(cache_dir) NumpyTupleDataset.save(dataset_cache_path, dataset) # Scale the label values, if necessary. if args.scale == 'standardize': print('Applying standard scaling to the labels.') scaler = StandardScaler() scaled_t = scaler.fit_transform(dataset.get_datasets()[-1]) dataset = NumpyTupleDataset(*(dataset.get_datasets()[:-1] + (scaled_t, ))) else: print('No standard scaling was selected.') scaler = None # Split the dataset into training and validation. train_data_size = int(len(dataset) * args.train_data_ratio) train, valid = split_dataset_random(dataset, train_data_size, args.seed) # Set up the predictor. predictor = set_up_predictor(method, args.unit_num, args.conv_layers, class_num, scaler) # Set up the iterators. train_iter = iterators.SerialIterator(train, args.batchsize) valid_iter = iterators.SerialIterator(valid, args.batchsize, repeat=False, shuffle=False) # Set up the regressor. device = args.gpu metrics_fun = { 'mae': MeanAbsError(scaler=scaler), 'rmse': RootMeanSqrError(scaler=scaler) } regressor = Regressor(predictor, lossfun=F.mean_squared_error, metrics_fun=metrics_fun, device=device) # Set up the optimizer. optimizer = optimizers.Adam() optimizer.setup(regressor) # Set up the updater. updater = training.StandardUpdater(train_iter, optimizer, device=device, converter=concat_mols) # Set up the trainer. trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend( E.Evaluator(valid_iter, regressor, device=device, converter=concat_mols)) trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(E.LogReport()) trainer.extend( E.PrintReport([ 'epoch', 'main/loss', 'main/mae', 'main/rmse', 'validation/main/loss', 'validation/main/mae', 'validation/main/rmse', 'elapsed_time' ])) trainer.extend(E.ProgressBar()) trainer.run() # Save the regressor's parameters. model_path = os.path.join(args.out, args.model_filename) print('Saving the trained model to {}...'.format(model_path)) regressor.save_pickle(model_path, protocol=args.protocol)
def set_event_handler(self): self.set_target() # (Not Implemented)Evaluator(train) self.trainer.extend(extensions.Evaluator( self.valid_loader, self.target, converter=self.converter, device=self.device, ), trigger=(self.eval_interval, 'epoch'), call_before_training=self.call_before_training) self.trainer.extend(extensions.ProgressBar()) self.trainer.extend(extensions.observe_lr()) # self.trainer.extend(extensions.MicroAverage('loss', 'lr', 'mav')) self.trainer.extend(extensions.LogReport(trigger=(self.log_interval, 'epoch')), call_before_training=self.call_before_training) self.trainer.extend(extensions.FailOnNonNumber()) # self.trainer.extend(extensions.ExponentialShift('lr', rate=0.9)) self.trainer.extend( extensions.ExponentialShift('lr', rate=0.99, init=self.lr * 10.0)) # (Not Implemented)InverseShift # (Not Implemented)LinearShift # (Not Implemented)MultistepShift # (Not Implemented)PolynomialShift # (Not Implemented)StepShift # (Not Implemented)WarmupShift self.trainer.extend( extensions.ParameterStatistics(self.model, trigger=(self.eval_interval, 'epoch'))) self.trainer.extend(extensions.VariableStatisticsPlot(self.model)) self.trainer.extend(extensions.PrintReport([ 'epoch', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time' ]), call_before_training=self.call_before_training) self.trainer.extend(extensions.PlotReport( ['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png'), call_before_training=self.call_before_training) self.trainer.extend(extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png'), call_before_training=self.call_before_training) self.trainer.extend(extensions.snapshot(n_retains=self.retain_num), trigger=(self.log_interval, 'epoch')) self.set_additonal_event_handler()
) # set dataset, model and optimizer train, test = chainer.datasets.get_mnist() model = chainer.links.Classifier(MnistMLP()) if os.path.isfile(MODEL_PATH): chainer.serializers.load_npz(MODEL_PATH, model) optimizer = chainer.optimizers.Adam() optimizer.setup(model) # set evaluation model eval_model = model.copy() eval_model.train = False # train and test train_iter = chainer.iterators.SerialIterator(train, 100) test_iter = chainer.iterators.SerialIterator(test, 100,repeat=False, shuffle=False) updater = chainer.training.StandardUpdater(train_iter, optimizer, device=-1) trainer = chainer.training.Trainer(updater, (10, 'epoch'), out=DESKTOP_PATH + "/result") trainer.extend(extensions.Evaluator(test_iter, eval_model, device=-1)) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy'])) trainer.extend(extensions.ProgressBar(update_interval=5)) trainer.extend(extensions.snapshot()) trainer.extend(extensions.snapshot_object(model, 'model_iter_{.updater.iteration}')) trainer.extend(extensions.dump_graph('main/loss')) if os.path.isfile(RESUME_PATH): chainer.serializers.load_npz(RESUME_PATH, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: DCGAN') parser.add_argument( '--batchsize', '-b', type=int, default=50, # default=50 help='Number of images in each mini-batch') parser.add_argument( '--epoch', '-e', type=int, default=10000, # defalt=500 help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument( '--dataset', '-i', default='train_illya.txt', # defalt='' help='Directory of image files. Default is cifar-10.') parser.add_argument( '--out', '-o', default='result_anime_illya4', # defalt='result' help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--n_hidden', '-n', type=int, default=100, help='Number of hidden units (z)') parser.add_argument('--seed', type=int, default=0, help='Random seed of z at visualization stage') parser.add_argument( '--snapshot_interval', type=int, default=1000, # defalt=1000 help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=100, help='Interval of displaying log to console') parser.add_argument( '--noise_sigma', type=float, default=0.2, # best: 0.2 help='Std of noise added the descriminator') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# n_hidden: {}'.format(args.n_hidden)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train gen = Generator(n_hidden=args.n_hidden, wscale=0.02) dis = Discriminator(init_sigma=args.noise_sigma, wscale=0.02) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() # Copy the model to the GPU dis.to_gpu() # Setup an optimizer # https://elix-tech.github.io/ja/2017/02/06/gan.html def make_optimizer(model, alpha=0.0002, beta1=0.5): # 元論文 optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.00001), 'hook_dec') return optimizer # # For WGAN # # Not good # def make_optimizer(model, alpha=0.0001, beta1=0.0, beta2=0.9): # optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1, beta2=beta2) # optimizer.setup(model) # return optimizer opt_gen = make_optimizer(gen) opt_dis = make_optimizer(dis) if args.dataset == '': # Load the CIFAR10 dataset if args.dataset is not specified train, _ = cifar.get_cifar10(withlabel=False, scale=255.) else: resized_paths = resize_data(args.dataset, insize + 16) train = PreprocessedDataset(resized_paths, crop_size=insize) print('{} contains {} image files'.format(args.dataset, train.__len__())) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) # Set up a trainer updater = DCGANUpdater(models=(gen, dis), iterator=train_iter, optimizer={ 'gen': opt_gen, 'dis': opt_dis }, device=args.gpu) # updater = WGANGPUpdater( # models=(gen, dis), # iterator=train_iter, # optimizer={ # 'gen': opt_gen, 'dis': opt_dis}, # device=args.gpu, # l=10, # n_c=5 # ) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen, 'gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend( extensions.PrintReport([ 'epoch', 'iteration', 'gen/loss', 'dis/loss', #'epoch', 'iteration', 'gen/loss', 'dis/loss/gan', 'dis/loss/grad', # For WGAN ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend(out_generated_image(gen, dis, 5, 5, args.seed, args.out), trigger=snapshot_interval) # 次第にDescriminatorのノイズを低減させる @training.make_extension(trigger=snapshot_interval) def shift_sigma(trainer): s = dis.shift_sigma(alpha_sigma=0.9) print('sigma={}'.format(s)) print('') trainer.extend(shift_sigma) # 通常のDCGANではコメントイン if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser( description= 'Fully Convolutional Dual Center Pose Proposal Network for Pose Estimation' ) parser.add_argument('--batchsize', '-b', type=int, default=1, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=200, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='results/dual_cp', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--snapshot_interval', type=int, default=1000, help='Interval of snapshot') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--train_resnet', type=bool, default=False, help='train resnet') parser.add_argument('--train-resnet', dest='train_resnet', action='store_true') parser.set_defaults(train_resnet=False) parser.add_argument('--no-accuracy', dest='compute_acc', action='store_false') parser.set_defaults(compute_acc=True) parser.add_argument('--no-pose-accuracy', dest='compute_pose_acc', action='store_false') parser.set_defaults(compute_pose_acc=True) args = parser.parse_args() compute_class_accuracy = args.compute_acc compute_pose_accuracy = args.compute_pose_acc and args.compute_acc print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('# compute class accuracy: {}'.format(compute_class_accuracy)) print('# compute pose accuracy: {}'.format(compute_pose_accuracy)) print('') im_size = (640, 480) objs = np.arange(3) + 1 n_class = len(objs) + 1 train_path = os.path.join(os.getcwd(), root, 'train_data/JSK_Objects/train') bg_path = os.path.join(os.getcwd(), root, 'train_data/MS_COCO/train2017') # bg_path = os.path.join(os.getcwd(), root, 'train_data/VOCdevkit/VOC2012/JPEGImages') caffe_model = 'ResNet-50-model.caffemodel' distance_sanity = 0.05 output_scale = 0.6 eps = 0.05 interval = 15 chainer.using_config('cudnn_deterministic', True) model = DualCPNetClassifier(DualCenterProposalNetworkRes50_predict7( n_class=n_class, pretrained_model=not args.train_resnet), basepath=train_path, im_size=im_size, distance_sanity=distance_sanity, compute_class_accuracy=compute_class_accuracy, compute_pose_accuracy=compute_pose_accuracy, output_scale=output_scale) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.0001)) # load train data train = JSKPoseEstimationAutoContextDataset(train_path, objs, bg_path, interval=interval, iteration_per_epoch=1000, mode='test', resize_rate=0.5, metric_filter=output_scale + eps) # load test data # test = JSKPoseEstimationAutoContextDataset(train_path, objs, bg_path, # interval=interval, # mode='train', # resize_rate=0.5, # metric_filter=output_scale + eps) test = JSKPoseEstimationDataset(train_path, objs, mode='train', interval=interval, resize_rate=0.5, metric_filter=output_scale + eps) print "number of train data : ", train.__len__() print "number of test data : ", test.__len__() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize) # test_iter = chainer.iterators.MultiprocessIterator(test, args.batchsize, # repeat=False, shuffle=False) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch evaluator = extensions.Evaluator(test_iter, model, device=args.gpu) evaluator.default_name = 'val' trainer.extend(evaluator) # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot and snapshot object for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) trainer.extend(extensions.snapshot_object( model.predictor, filename='model_iteration-{.updater.iteration}'), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport([ 'epoch', 'main/l_cls', 'main/l_cp', 'main/l_ocp', 'main/cls_acc', 'main/ocp_acc', 'main/rot_acc', 'val/main/l_cls', 'val/main/l_cp', 'val/main/l_ocp', 'val/main/cls_acc', 'val/main/ocp_acc', 'val/main/rot_acc', 'elapsed_time' ])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) else: npz_name = 'DualCenterProposalNetworkRes50_jsk_class{}.npz' caffemodel_name = 'ResNet-50-model.caffemodel' path = os.path.join(root, 'trained_data/', npz_name.format(n_class)) path_caffemodel = os.path.join(root, 'trained_data/', caffemodel_name) print 'npz model path : ' + path print 'caffe model path : ' + path_caffemodel download.cache_or_load_file( path, lambda path: _make_chainermodel_npz(path, path_caffemodel, model, n_class), lambda path: serializers.load_npz(path, model)) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: seq2seq') parser.add_argument('SOURCE', help='source sentence list') parser.add_argument('TARGET', help='target sentence list') parser.add_argument('SOURCE_VOCAB', help='source vocabulary file') parser.add_argument('TARGET_VOCAB', help='target vocabulary file') parser.add_argument('--validation-source', help='source sentence list for validation') parser.add_argument('--validation-target', help='target sentence list for validation') parser.add_argument('--batchsize', '-b', type=int, default=64, help='number of sentence pairs in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='number of sweeps over the dataset to train') parser.add_argument('--resume', '-r', type=str, help='resume the training from snapshot') parser.add_argument('--save', '-s', type=str, help='save a snapshot of the training') parser.add_argument('--unit', '-u', type=int, default=1024, help='number of units') parser.add_argument('--layer', '-l', type=int, default=3, help='number of layers') parser.add_argument('--use-dataset-api', default=False, action='store_true', help='use TextDataset API to reduce CPU memory usage') parser.add_argument('--min-source-sentence', type=int, default=1, help='minimium length of source sentence') parser.add_argument('--max-source-sentence', type=int, default=50, help='maximum length of source sentence') parser.add_argument('--min-target-sentence', type=int, default=1, help='minimium length of target sentence') parser.add_argument('--max-target-sentence', type=int, default=50, help='maximum length of target sentence') parser.add_argument('--log-interval', type=int, default=200, help='number of iteration to show log') parser.add_argument('--validation-interval', type=int, default=4000, help='number of iteration to evlauate the model ' 'with validation dataset') parser.add_argument('--device', '-d', type=str, default='-1', help='Device specifier. Either ChainerX device ' 'specifier or an integer. If non-negative integer, ' 'CuPy arrays with specified device id are used. If ' 'negative integer, NumPy arrays are used') parser.add_argument('--out', '-o', default='result', help='directory to output the result') group = parser.add_argument_group('deprecated arguments') group.add_argument('--gpu', '-g', dest='device', type=int, nargs='?', const=0, help='GPU ID (negative value indicates CPU)') args = parser.parse_args() device = chainer.get_device(args.device) print('Device: {}'.format(device)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # If the device is a ChainerX CUDA device, use the shared device memory # pool between ChainerX and CuPy. if device.xp is chainerx and device.device.backend.name == 'cuda': # TODO(niboshi): The API is provisional. chainerx._cuda.cupy_share_allocator() # Load pre-processed dataset print('[{}] Loading dataset... (this may take several minutes)'.format( datetime.datetime.now())) source_ids = load_vocabulary(args.SOURCE_VOCAB) target_ids = load_vocabulary(args.TARGET_VOCAB) if args.use_dataset_api: # By using TextDataset, you can avoid loading whole dataset on memory. # This significantly reduces the host memory usage. def _filter_func(s, t): sl = len(s.strip().split()) # number of words in source line tl = len(t.strip().split()) # number of words in target line return ( args.min_source_sentence <= sl <= args.max_source_sentence and args.min_target_sentence <= tl <= args.max_target_sentence) train_data = load_data_using_dataset_api( source_ids, args.SOURCE, target_ids, args.TARGET, _filter_func, ) else: # Load all records on memory. train_source = load_data(source_ids, args.SOURCE) train_target = load_data(target_ids, args.TARGET) assert len(train_source) == len(train_target) train_data = [ (s, t) for s, t in six.moves.zip(train_source, train_target) if (args.min_source_sentence <= len(s) <= args.max_source_sentence and args.min_target_sentence <= len(t) <= args.max_target_sentence) ] print('[{}] Dataset loaded.'.format(datetime.datetime.now())) if not args.use_dataset_api: # Skip printing statistics when using TextDataset API, as it is slow. train_source_unknown = calculate_unknown_ratio( [s for s, _ in train_data]) train_target_unknown = calculate_unknown_ratio( [t for _, t in train_data]) print('Source vocabulary size: %d' % len(source_ids)) print('Target vocabulary size: %d' % len(target_ids)) print('Train data size: %d' % len(train_data)) print('Train source unknown ratio: %.2f%%' % ( train_source_unknown * 100)) print('Train target unknown ratio: %.2f%%' % ( train_target_unknown * 100)) target_words = {i: w for w, i in target_ids.items()} source_words = {i: w for w, i in source_ids.items()} # Set the current device device.use() # Setup model model = Seq2seq(args.layer, len(source_ids), len(target_ids), args.unit) model.to_device(device) # Setup optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Setup iterator train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize) # Setup updater and trainer updater = training.updaters.StandardUpdater( train_iter, optimizer, converter=convert, device=device) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.LogReport( trigger=(args.log_interval, 'iteration'))) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'main/loss', 'main/perp', 'validation/main/bleu', 'elapsed_time']), trigger=(args.log_interval, 'iteration')) trainer.extend( extensions.snapshot(filename='snapshot_epoch_{.updater.iteration}'), trigger=(args.validation_interval, 'iteration')) if args.validation_source and args.validation_target: test_source = load_data(source_ids, args.validation_source) test_target = load_data(target_ids, args.validation_target) assert len(test_source) == len(test_target) test_data = list(six.moves.zip(test_source, test_target)) test_data = [(s, t) for s, t in test_data if 0 < len(s) and 0 < len(t)] test_source_unknown = calculate_unknown_ratio( [s for s, _ in test_data]) test_target_unknown = calculate_unknown_ratio( [t for _, t in test_data]) print('Validation data: %d' % len(test_data)) print('Validation source unknown ratio: %.2f%%' % (test_source_unknown * 100)) print('Validation target unknown ratio: %.2f%%' % (test_target_unknown * 100)) @chainer.training.make_extension() def translate(trainer): source, target = test_data[numpy.random.choice(len(test_data))] result = model.translate([model.xp.array(source)])[0] source_sentence = ' '.join([source_words[x] for x in source]) target_sentence = ' '.join([target_words[y] for y in target]) result_sentence = ' '.join([target_words[y] for y in result]) print('# source : ' + source_sentence) print('# result : ' + result_sentence) print('# expect : ' + target_sentence) trainer.extend( translate, trigger=(args.validation_interval, 'iteration')) trainer.extend( CalculateBleu( model, test_data, 'validation/main/bleu', device), trigger=(args.validation_interval, 'iteration')) if args.resume is not None: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) print('start training') trainer.run() if args.save is not None: # Save a snapshot chainer.serializers.save_npz(args.save, trainer)
# Iterator train_iter = iterators.MultiprocessIterator(train, args.batchsize, shared_mem=10000000) valid_iter = iterators.SerialIterator(valid, args.valid_batchsize, repeat=False, shuffle=False) # Updater updater = ParallelUpdater(train_iter, optimizer, devices=devices) trainer = training.Trainer(updater, (args.epoch, "epoch"), out=result_dir) # Extentions trainer.extend(extensions.Evaluator(valid_iter, model, device=devices["main"]), trigger=(args.valid_freq, "epoch")) trainer.extend(extensions.dump_graph("main/rpn_loss_cls", out_name="rpn_loss_cls.dot")) trainer.extend(extensions.dump_graph("main/rpn_loss_bbox", out_name="rpn_loss_bbox.dot")) trainer.extend(extensions.dump_graph("main/loss_cls", out_name="loss_cls.dot")) trainer.extend(extensions.dump_graph("main/loss_bbox", out_name="loss_bbox.dot")) trainer.extend(extensions.snapshot(trigger=(args.snapshot_iter, "iteration"))) trainer.extend(extensions.LogReport(trigger=(args.show_log_iter, "iteration"))) trainer.extend( extensions.PrintReport( [ "epoch", "iteration", "main/rpn_loss_cls", "main/rpn_loss_bbox", "main/loss_cls", "main/loss_bbox", "validation/main/rpn_loss_cls", "validation/main/rpn_loss_bbox", "validation/main/loss_cls", "validation/main/loss_bbox", ]
save_interval = (5, 'epoch') log_interval = (max(n_iteration // 1, 1), 'iteration') progressbar_interval = 3 imgview_face_interval = (5, 'iteration') imgview_weight_interval = (1, 'epoch') logger.info('Test interval : {}'.format(test_interval)) logger.info('Save interval : {}'.format(save_interval)) logger.info('Log interval : {}'.format(log_interval)) logger.info('ProgressBar interval : {}'.format(progressbar_interval)) logger.info('ImgView face interval : {}'.format(imgview_face_interval)) logger.info('ImgView weight interval : {}'.format(imgview_weight_interval)) # Extensions trainer.extend(extensions.dump_graph('main/loss'), trigger=save_interval) trainer.extend( extensions.snapshot(filename='snapshot_epoch_{.updater.epoch}'), trigger=save_interval) trainer.extend(extensions.snapshot_object(model, 'model_epoch_{.updater.epoch}'), trigger=save_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'main/loss', 'validation/main/loss']), trigger=log_interval) trainer.extend( extensions.ProgressBar(update_interval=progressbar_interval)) # My extensions # Sequential Evaluator trainer.extend( SequentialEvaluator(test_iter, eval_model, device=config.gpu),
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=1, help='Number of images in each mini batch') parser.add_argument('--epoch', '-e', type=int, default=100, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=10, help='Number of units') args = parser.parse_args() # load a color image img1 = cv2.imread('images/zero.jpg', cv2.IMREAD_COLOR) img2 = cv2.imread('images/black.jpg', cv2.IMREAD_COLOR) img3 = cv2.imread('images/white.jpg', cv2.IMREAD_COLOR) # color -> grayscale imggray1 = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY) imggray2 = cv2.cvtColor(img2, cv2.COLOR_RGB2GRAY) imggray3 = cv2.cvtColor(img3, cv2.COLOR_RGB2GRAY) # image -> array gray = [] for y in range(len(imggray1)): for x in range(len(imggray1[y])): gray.append(imggray1[y][x]) imgdata1 = np.array(gray, dtype='f') imgdata1 = imgdata1.reshape(1, 1, 32, 32) imgdata1 = imgdata1 / 255.0 gray = [] for y in range(len(imggray2)): for x in range(len(imggray2[y])): gray.append(imggray2[y][x]) imgdata2 = np.array(gray, dtype='f') imgdata2 = imgdata2.reshape(1, 1, 32, 32) imgdata2 = imgdata2 / 255.0 gray = [] for y in range(len(imggray3)): for x in range(len(imggray3[y])): gray.append(imggray3[y][x]) imgdata3 = np.array(gray, dtype='f') imgdata3 = imgdata3.reshape(1, 1, 32, 32) imgdata3 = imgdata3 / 255.0 n_in = 32*32 print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') model = L.Classifier(MLP(n_in, args.unit, 3)) # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load dataset x1 = imgdata1 x2 = imgdata2 x3 = imgdata3 y1 = np.array(0, dtype=np.int32) y2 = np.array(1, dtype=np.int32) y3 = np.array(2, dtype=np.int32) dd = [(x1, y1), (x2, y2), (x3, y3)] train, test = dd, dd train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot()) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) # Resume from a snapshot #chainer.serializers.load_npz(resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: DCGAN') parser.add_argument('--batchsize', '-b', type=int, default=50, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=1000, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--dataset', '-i', default='', help='Directory of image files. Default is cifar-10.') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--n_hidden', '-n', type=int, default=100, help='Number of hidden units (z)') parser.add_argument('--seed', type=int, default=0, help='Random seed of z at visualization stage') parser.add_argument('--snapshot_interval', type=int, default=1000, help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=100, help='Interval of displaying log to console') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# n_hidden: {}'.format(args.n_hidden)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train gen = Generator(n_hidden=args.n_hidden) dis = Discriminator() if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() # Copy the model to the GPU dis.to_gpu() # Setup an optimizer def make_optimizer(model, alpha=0.0002, beta1=0.5): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.0001), 'hook_dec') return optimizer opt_gen = make_optimizer(gen) opt_dis = make_optimizer(dis) if args.dataset == '': # Load the CIFAR10 dataset if args.dataset is not specified train, _ = chainer.datasets.get_cifar10(withlabel=False, scale=255.) else: all_files = os.listdir(args.dataset) image_files = [f for f in all_files if ('png' in f or 'jpg' in f)] print('{} contains {} image files'.format(args.dataset, len(image_files))) train = chainer.datasets\ .ImageDataset(paths=image_files, root=args.dataset) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) # Set up a trainer updater = DCGANUpdater(models=(gen, dis), iterator=train_iter, optimizer={ 'gen': opt_gen, 'dis': opt_dis }, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen, 'gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'gen/loss', 'dis/loss', ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) visualizer = out_generated_image(gen, dis, 10, 10, args.seed) trainer.extend( ImageReport(trigger=(1, 'epoch'), image_generator=visualizer)) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--device', '-d', type=str, default='-1', help='Device specifier. Either ChainerX device ' 'specifier or an integer. If non-negative integer, ' 'CuPy arrays with specified device id are used. If ' 'negative integer, NumPy arrays are used') parser.add_argument('--unit', '-u', default=100, type=int, help='number of units') parser.add_argument('--window', '-w', default=5, type=int, help='window size') parser.add_argument('--batchsize', '-b', type=int, default=1000, help='learning minibatch size') parser.add_argument('--epoch', '-e', default=20, type=int, help='number of epochs to learn') parser.add_argument('--model', '-m', choices=['skipgram', 'cbow'], default='skipgram', help='model type ("skipgram", "cbow")') parser.add_argument('--negative-size', default=5, type=int, help='number of negative samples') parser.add_argument('--out-type', '-o', choices=['hsm', 'ns', 'original'], default='hsm', help='output model type ("hsm": hierarchical softmax, ' '"ns": negative sampling, "original": ' 'no approximation)') parser.add_argument('--out', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', type=str, help='Resume the training from snapshot') parser.add_argument('--snapshot-interval', type=int, help='Interval of snapshots') parser.add_argument('--test', dest='test', action='store_true') parser.set_defaults(test=False) group = parser.add_argument_group('deprecated arguments') group.add_argument('--gpu', '-g', dest='device', type=int, nargs='?', const=0, help='GPU ID (negative value indicates CPU)') args = parser.parse_args() device = chainer.get_device(args.device) device.use() if args.snapshot_interval is None: args.snapshot_interval = args.epoch args.snapshot_interval = min(args.snapshot_interval, args.epoch) print('Device: {}'.format(device)) print('# unit: {}'.format(args.unit)) print('Window: {}'.format(args.window)) print('Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('Training model: {}'.format(args.model)) print('Output type: {}'.format(args.out_type)) print('') # Load the dataset train, val, _ = chainer.datasets.get_ptb_words() counts = collections.Counter(train) counts.update(collections.Counter(val)) n_vocab = max(train) + 1 if args.test: train = train[:100] val = val[:100] vocab = chainer.datasets.get_ptb_words_vocabulary() index2word = {wid: word for word, wid in six.iteritems(vocab)} print('n_vocab: %d' % n_vocab) print('data length: %d' % len(train)) if args.out_type == 'hsm': HSM = L.BinaryHierarchicalSoftmax tree = HSM.create_huffman_tree(counts) loss_func = HSM(args.unit, tree) loss_func.W.array[...] = 0 elif args.out_type == 'ns': cs = [counts[w] for w in range(len(counts))] loss_func = L.NegativeSampling(args.unit, cs, args.negative_size) loss_func.W.array[...] = 0 elif args.out_type == 'original': loss_func = SoftmaxCrossEntropyLoss(args.unit, n_vocab) else: raise Exception('Unknown output type: {}'.format(args.out_type)) # Choose the model if args.model == 'skipgram': model = SkipGram(n_vocab, args.unit, loss_func) elif args.model == 'cbow': model = ContinuousBoW(n_vocab, args.unit, loss_func) else: raise Exception('Unknown model type: {}'.format(args.model)) model.to_device(device) # Set up an optimizer optimizer = O.Adam() optimizer.setup(model) # Set up an iterator train_iter = WindowIterator(train, args.window, args.batchsize) val_iter = WindowIterator(val, args.window, args.batchsize, repeat=False) # Set up an updater updater = training.updaters.StandardUpdater( train_iter, optimizer, converter=convert, device=device) # Set up a trainer trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator( val_iter, model, converter=convert, device=device)) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss'])) trainer.extend(extensions.ProgressBar()) trainer.extend( extensions.snapshot(filename='snapshot_epoch_{.updater.epoch}'), trigger=(args.snapshot_interval, 'epoch')) if args.resume is not None: chainer.serializers.load_npz(args.resume, trainer) trainer.run() # Save the word2vec model with open(os.path.join(args.out, 'word2vec.model'), 'w') as f: f.write('%d %d\n' % (len(index2word), args.unit)) w = cuda.to_cpu(model.embed.W.array) for i, wi in enumerate(w): v = ' '.join(map(str, wi)) f.write('%s %s\n' % (index2word[i], v))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--model', choices=('ssd300', 'ssd512'), default='ssd300') parser.add_argument('--batchsize', type=int, default=1) parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--out', default='result') parser.add_argument('--resume') args = parser.parse_args() model = SSD300( n_fg_class=len(inria_bbox_label_names), pretrained_model='./ssd_vgg16_imagenet_2017_06_09.npz') print("###n_fg_class= ", len(inria_bbox_label_names)) model.use_preset('evaluate') train_chain = MultiboxTrainChain(model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() train = TransformDataset( ConcatenatedDataset( INRIABboxDataset(data_dir='../INRIAPerson', split='Train') ), Transform(model.coder, model.insize, model.mean)) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test = INRIABboxDataset(data_dir='../INRIAPerson', split='Test') test_iter = chainer.iterators.SerialIterator( test, args.batchsize, repeat=False, shuffle=False) # initial lr is set to 1e-3 by ExponentialShift optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(train_chain) for param in train_chain.params(): if param.name == 'b': param.update_rule.add_hook(GradientScaling(2)) else: param.update_rule.add_hook(WeightDecay(0.0005)) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (120000, 'iteration'), args.out) trainer.extend( extensions.ExponentialShift('lr', 0.1, init=1e-4), trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration')) trainer.extend( DetectionINRIAEvaluator( test_iter, model, use_07_metric=False, label_names=inria_bbox_label_names), trigger=(1, 'iteration')) log_interval = 1, 'iteration' trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport( ['epoch', 'iteration', 'lr', 'main/loss', 'main/loss/loc', 'main/loss/conf', 'validation/main/map']), trigger=log_interval) #trainer.extend(extensions.ProgressBar(update_interval=1)) trainer.extend(extensions.snapshot(), trigger=(10000, 'iteration')) trainer.extend( extensions.snapshot_object(model, 'model_iter_{.updater.iteration}'), trigger=(120000, 'iteration')) if args.resume: serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=1000, help='Number of units') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model = L.Classifier(MLP(args.unit, 10)) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def main(arg_list=None): parser = argparse.ArgumentParser(description='Chainer LSTM') parser.add_argument('--epoch', '-e', type=int, nargs='+', default=[20], help='Number of sweeps over the dataset to train') parser.add_argument('--optimizer', '-o', nargs='+', default=['momentumsgd'], help='Optimizer (sgd, momentumsgd, adam)') parser.add_argument('--batch-size', '-b', type=int, nargs='+', default=[128], help='Number of training points in each mini-batch') parser.add_argument('--lr', type=float, nargs='+', default=[1e-2, 1e-3, 1e-4, 1e-5], help='Learning rate') parser.add_argument('--early-stopping', type=str2bool, nargs='+', default=[True], help="True if early stopping should be enabled") parser.add_argument( '--network', '-n', default='ff', help= 'Neural network type, either "ff", "tdnn", "lstm", "zoneoutlstm", "peepholelstm" or "gru". Setting any recurrent network implies "--shuffle-sequences"' ) parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--units', '-u', type=int, nargs='+', default=[1024], help='Number of units') parser.add_argument('--layers', '-l', type=int, default=2, help='Number of hidden layers') parser.add_argument('--activation', '-a', default='relu', help='FF activation function (sigmoid, tanh or relu)') parser.add_argument('--tdnn-ksize', type=int, nargs='+', default=[5], help='TDNN kernel size') parser.add_argument('--bproplen', type=int, default=20, help='Backpropagation length') parser.add_argument('--timedelay', type=int, default=0, help='Delay target values by this many time steps') parser.add_argument('--noplot', dest='plot', action='store_false', help='Disable PlotReport extension') parser.add_argument('--splice', type=int, default=0, help='Splicing size') parser.add_argument( '--dropout', '-d', type=float, nargs='+', default=[0], help= 'Dropout rate (0 to disable). In case of Zoneout LSTM, this parameter has 2 arguments: c_ratio h_ratio' ) parser.add_argument('--ft', default='final.feature_transform', help='Kaldi feature transform file') parser.add_argument('--tri', action='store_true', help='Use triphones') parser.add_argument( '--shuffle-sequences', action='store_true', help= 'True if sequences should be shuffled as a whole, otherwise all frames will be shuffled independent of each other' ) parser.add_argument( '--data-dir', default='data/fmllr', help= 'Data directory, this will be prepended to data files and feature transform' ) parser.add_argument( '--offset-dir', default='data', help='Data directory, this will be prepended to offset files') parser.add_argument( '--target-dir', default='data/targets', help='Data directory, this will be prepended to target files') parser.add_argument( '--ivector-dir', help='Data directory, this will be prepended to ivector files') parser.add_argument('--data', default='data_{}.npy', help='Training data') parser.add_argument('--offsets', default='offsets_{}.npy', help='Training offsets') parser.add_argument('--targets', default='targets_{}.npy', help='Training targets') parser.add_argument('--ivectors', default='ivectors_{}.npy', help='Training ivectors') parser.add_argument('--no-validation', dest='use_validation', action='store_false', help='Do not evaluate validation data while training') parser.add_argument('--train-fold', type=int, help='Train fold network with this ID') parser.add_argument('--train-rpl', action='store_true', help='Train RPL layer') parser.add_argument('--rpl-model', default="result_rpl/model", help='RPL layer model') parser.add_argument('--fold-data-dir', default="fold_data", help='Directory with fold input data') parser.add_argument('--fold-output-dir', default="fold_data_out", help='Directory with predicted fold output') parser.add_argument('--fold-model-dir', default="fold_models", help='Directory with output fold model') parser.add_argument( '--fold-data-pattern', default='data_{0}.npy', help= 'Filename pattern of each fold data, {0} will be replaced by fold ID') parser.add_argument('--fold-offset-pattern', default='offsets_{0}.npy', help='Filename pattern of each fold offset') parser.add_argument('--fold-target-pattern', default='targets_{0}.npy', help='Filename pattern of each fold targets') parser.add_argument( '--fold-ivector-pattern', default='ivectors_{}.npy', help= 'Filename pattern of each fold i-vectors file, {} will be replaced by fold ID' ) parser.add_argument('--fold-output-pattern', default='data_{0}.npy', help='Filename pattern of each fold network output') parser.add_argument('--fold-network-pattern', default='fold_{0}.npz', help='Filename pattern of each fold network') parser.add_argument('--no-progress', action='store_true', help='Disable progress bar') if arg_list is not None: args = parser.parse_args(list(map(str, arg_list))) else: args = parser.parse_args() # set options implied by other options if is_nn_recurrent(args.network): args.shuffle_sequences = True # create output directories Path(args.out).mkdir(exist_ok=True, parents=True) if args.train_fold is not None: file_out = Path(args.fold_model_dir, args.fold_network_pattern.format(args.train_fold)) Path(file_out.parent).mkdir(exist_ok=True, parents=True) # print arguments to the file with open(args.out + "/args.txt", "w") as f: for attr in dir(args): if not attr.startswith('_'): f.write('# {}: {}\n'.format(attr, getattr(args, attr))) f.write(' '.join( map(lambda x: "'" + x + "'" if ' ' in x else x, sys.argv)) + '\n') # print arguments to stdout for attr in dir(args): if not attr.startswith('_'): print('# {}: {}'.format(attr, getattr(args, attr))) print('') # input feature vector length num_classes = 1909 if args.tri else 39 # create model if args.train_rpl: model = RPL4(num_classes) model_cls = L.Classifier(model) else: if args.activation == "sigmoid": activation = F.sigmoid elif args.activation == "tanh": activation = F.tanh elif args.activation == "relu": activation = F.relu else: print("Wrong activation function specified") return model = get_nn(args.network, args.layers, args.units, num_classes, activation, args.tdnn_ksize, args.dropout) # classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model_cls = L.Classifier(model) if args.gpu >= 0: # make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model_cls.to_gpu() # copy the model to the GPU offsets = offsets_dev = None if args.train_rpl: # load training data fold = 0 x = [] y = [] while True: x_file = Path(args.fold_output_dir, args.fold_output_pattern.format(fold)) y_file = Path(args.fold_data_dir, args.fold_target_pattern.format(fold)) if not x_file.is_file() or not y_file.is_file(): break print("Loading fold {} data".format(fold)) x_ = np.load(str(x_file)) y_ = np.load(str(y_file)) x.append(x_) y.append(y_) fold += 1 if fold == 0: print("Error: No fold data found") return x = np.concatenate(x, axis=0) y = np.concatenate(y, axis=0) if args.use_validation: #TODO: use args.data instead of args.dev_data x_dev = np.load(str(Path(args.data_dir, args.data.format("dev")))) # offsets_dev = loadBin(str(Path(args.datadir, args.dev_offsets)), np.int32) y_dev = np.load( str(Path(args.target_dir, args.targets.format("dev")))) else: # load training data ivectors = None ivectors_dev = None if args.train_fold is not None: x = [] offsets = [0] y = [] if args.ivector_dir is not None: ivectors = [] num = 0 fold = 0 while True: if fold != args.train_fold: x_file = Path(args.fold_data_dir, args.fold_data_pattern.format(fold)) if not x_file.is_file(): break offsets_file = Path(args.fold_data_dir, args.fold_offset_pattern.format(fold)) y_file = Path(args.fold_data_dir, args.fold_target_pattern.format(fold)) if args.ivector_dir is not None: ivectors_file = Path( args.fold_data_dir, args.fold_ivector_pattern.format(fold)) if not ivectors_file.is_file(): print("Error: missing ivectors for fold data {}". format(fold)) return print("Loading fold {} data".format(fold)) x_fold = np.load(str(x_file)) x.append(x_fold) if is_nn_recurrent(args.network): offsets_fold = np.load(str(offsets_file)) offsets.extend(offsets_fold[1:] + num) y_fold = np.load(str(y_file)) y.append(y_fold) if args.ivector_dir is not None: ivectors_fold = np.load(str(ivectors_file)) ivectors.append(ivectors_fold) num += x_fold.shape[0] fold += 1 if len(x) == 0: print("Error: No fold data found") return x = np.concatenate(x, axis=0) if is_nn_recurrent(args.network): offsets = np.array(offsets, dtype=np.int32) y = np.concatenate(y, axis=0) if args.ivector_dir is not None: ivectors = np.concatenate(ivectors, axis=0) else: x = np.load(str(Path(args.data_dir, args.data.format("train")))) if is_nn_recurrent(args.network): offsets = np.load( str(Path(args.offset_dir, args.offsets.format("train")))) y = np.load( str(Path(args.target_dir, args.targets.format("train")))) if args.ivector_dir is not None: ivectors = np.load( str(Path(args.ivector_dir, args.ivectors.format("train")))) if args.use_validation: x_dev = np.load(str(Path(args.data_dir, args.data.format("dev")))) if is_nn_recurrent(args.network): offsets_dev = np.load( str(Path(args.offset_dir, args.offsets.format("dev")))) y_dev = np.load( str(Path(args.target_dir, args.targets.format("dev")))) if args.ivector_dir is not None: ivectors_dev = np.load( str(Path(args.ivector_dir, args.ivectors.format("dev")))) # apply splicing if args.network == "tdnn": splice = (sum(args.tdnn_ksize) - len(args.tdnn_ksize)) // 2 else: splice = args.splice if splice > 0: x = splicing(x, range(-splice, splice + 1)) x_dev = splicing(x_dev, range(-splice, splice + 1)) # load feature transform if args.ft is not None and args.ft != '-': ft = loadKaldiFeatureTransform(str(Path(args.data_dir, args.ft))) if is_nn_recurrent( args.network ): # select transform middle frame if the network is recurrent dim = ft["shape"][1] zi = ft["shifts"].index(0) ft["rescale"] = ft["rescale"][zi * dim:(zi + 1) * dim] ft["addShift"] = ft["addShift"][zi * dim:(zi + 1) * dim] ft["shape"][0] = dim ft["shifts"] = [0] elif args.network == "tdnn": dim = ft["shape"][1] zi = ft["shifts"].index(0) winlen = 2 * splice + 1 ft["rescale"] = np.tile(ft["rescale"][zi * dim:(zi + 1) * dim], winlen) ft["addShift"] = np.tile( ft["addShift"][zi * dim:(zi + 1) * dim], winlen) ft["shape"][0] = dim * winlen ft["shifts"] = list(range(-splice, splice + 1)) # apply feature transform x = applyKaldiFeatureTransform(x, ft) if args.use_validation: x_dev = applyKaldiFeatureTransform(x_dev, ft) if ivectors is not None: x = np.concatenate((x, ivectors), axis=1) if ivectors_dev is not None: x_dev = np.concatenate((x_dev, ivectors_dev), axis=1) # shift the input dataset according to time delay if is_nn_recurrent(args.network) and args.timedelay != 0: x, y, offsets = apply_time_delay(x, y, offsets, args.timedelay) if args.use_validation: x_dev, y_dev, offsets_dev = apply_time_delay( x_dev, y_dev, offsets_dev, args.timedelay) # create chainer datasets train_dataset = chainer.datasets.TupleDataset(x, y) if args.use_validation: dev_dataset = chainer.datasets.TupleDataset(x_dev, y_dev) # prepare train stages train_stages_len = max([ len(a) for a in [ args.epoch, args.optimizer, args.batch_size, args.lr, args.early_stopping ] ]) train_stages = [{ 'epoch': index_padded(args.epoch, i), 'opt': index_padded(args.optimizer, i), 'bs': index_padded(args.batch_size, i), 'lr': index_padded(args.lr, i), 'es': index_padded(args.early_stopping, i) } for i in range(train_stages_len)] for i, ts in enumerate(train_stages): if ts['opt'] == 'adam': # learning rate not used, don't print it print( "=== Training stage {}: epoch = {}, batch size = {}, optimizer = {}, early stopping = {}" .format(i, ts['epoch'], ts['bs'], ts['opt'], ts['es'])) else: print( "=== Training stage {}: epoch = {}, batch size = {}, optimizer = {}, learning rate = {}, early stopping = {}" .format(i, ts['epoch'], ts['bs'], ts['opt'], ts['lr'], ts['es'])) # reset state to allow training with different batch size in each stage if not args.train_rpl and is_nn_recurrent(args.network): model.reset_state() # setup an optimizer if ts['opt'] == "sgd": optimizer = chainer.optimizers.SGD(lr=ts['lr']) elif ts['opt'] == "momentumsgd": optimizer = chainer.optimizers.MomentumSGD(lr=ts['lr']) elif ts['opt'] == "adam": optimizer = chainer.optimizers.Adam() else: print("Wrong optimizer specified: {}".format(ts['opt'])) exit(1) optimizer.setup(model_cls) if args.shuffle_sequences: train_iter = SequenceShuffleIterator(train_dataset, offsets, ts['bs']) if args.use_validation: dev_iter = SequenceShuffleIterator(dev_dataset, None, ts['bs'], repeat=False, shuffle=False) else: train_iter = SerialIterator(train_dataset, ts['bs']) if args.use_validation: dev_iter = SerialIterator(dev_dataset, ts['bs'], repeat=False, shuffle=False) # set up a trainer if is_nn_recurrent(args.network): updater = BPTTUpdater(train_iter, optimizer, args.bproplen, device=args.gpu) else: updater = StandardUpdater(train_iter, optimizer, device=args.gpu) if ts['es'] and args.use_validation: stop_trigger = EarlyStoppingTrigger(ts['epoch'], key='validation/main/loss', eps=-0.001) else: stop_trigger = (ts['epoch'], 'epoch') trainer = training.Trainer(updater, stop_trigger, out="{}/{}".format(args.out, i)) if ts['es']: trainer.extend(model_saver) else: trainer.extend(BestModelSaver(key="validation/main/loss")) # evaluate the model with the development dataset for each epoch if args.use_validation: trainer.extend( extensions.Evaluator(dev_iter, model_cls, device=args.gpu)) # dump a computational graph from 'loss' variable at the first iteration # the "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # take a snapshot for each specified epoch frequency = ts['epoch'] if args.frequency == -1 else max( 1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # save two plot images to the result dir if args.plot and extensions.PlotReport.available(): plot_vars_loss = ['main/loss'] plot_vars_acc = ['main/accuracy'] if args.use_validation: plot_vars_loss.append('validation/main/loss') plot_vars_acc.append('validation/main/accuracy') trainer.extend( extensions.PlotReport(plot_vars_loss, 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(plot_vars_acc, 'epoch', file_name='accuracy.png')) # print selected entries of the log to stdout # here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. if args.use_validation: print_report_vars = [ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ] else: print_report_vars = [ 'epoch', 'main/loss', 'main/accuracy', 'elapsed_time' ] trainer.extend(extensions.PrintReport(print_report_vars)) # print a progress bar to stdout # trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run() if ts['es']: # load the last model if the max epoch was not reached (that means early stopping trigger stopped training # because the validation loss increased) if updater.epoch_detail < ts['epoch']: chainer.serializers.load_npz( "{}/{}/model_tmp".format(args.out, i), model_cls) # remove temporary model from this training stage os.remove("{}/{}/model_tmp".format(args.out, i)) else: # load the best model from this training stage chainer.serializers.load_npz( "{}/{}/model_best".format(args.out, i), model_cls) # remove the best model from this training stage os.remove("{}/{}/model_best".format(args.out, i)) # save the final model chainer.serializers.save_npz("{}/model".format(args.out), model_cls) if args.train_fold is not None: chainer.serializers.save_npz( str( Path(args.fold_model_dir, args.fold_network_pattern.format(args.train_fold))), model_cls)
def main(): # This script is almost identical to train_mnist.py. The only difference is # that this script uses data-parallel computation on two GPUs. # See train_mnist.py for more details. parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=400, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu0', '-g', type=int, default=0, help='First GPU ID') parser.add_argument('--gpu1', '-G', type=int, default=1, help='Second GPU ID') parser.add_argument('--out', '-o', default='result_parallel', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--unit', '-u', type=int, default=1000, help='Number of units') args = parser.parse_args() print('GPU: {}, {}'.format(args.gpu0, args.gpu1)) print('# unit: {}'.format(args.unit)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') chainer.cuda.get_device(args.gpu0).use() model = L.Classifier(train_mnist.MLP(args.unit, 10)) optimizer = chainer.optimizers.Adam() optimizer.setup(model) train, test = chainer.datasets.get_mnist() train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # ParallelUpdater implements the data-parallel gradient computation on # multiple GPUs. It accepts "devices" argument that specifies which GPU to # use. updater = training.ParallelUpdater( train_iter, optimizer, # The device of the name 'main' is used as a "master", while others are # used as slaves. Names other than 'main' are arbitrary. devices={'main': args.gpu0, 'second': args.gpu1}, ) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu0)) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy'])) trainer.extend(extensions.ProgressBar()) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser(description="Learning from flowers data") parser.add_argument("--gpu", "-g", type=int, default=-1, help="GPU ID (negative value indicates CPU") parser.add_argument("--init", help="Initialize the model from given file") parser.add_argument('--job', '-j', type=int, help='Number of parallel data loading processes') parser.add_argument("--resume", '-r', default='', help="Initialize the trainer from given file") args = parser.parse_args() batch = 32 epoch = 50 val_batch = 200 model = models.ResNet50V1(data.ClassNumber) if args.init: print('Load model from', args.initmodel) chainer.serializers.load_npz(args.init, model) if args.gpu >= 0: chainer.backends.cuda.get_device_from_id(args.gpu).use() model.to_gpu() if data.fetch_flowers() and data.fetch_labels(): print("Flower images and labels have been fetched.") else: print("Failed to fetch flower images and labels") return data.pre_process_data(224) output_name = output.init_train(model.__class__) output_path = path.join(output.OutPath, output_name) train, validate = data.get_datasets() train_iter = chainer.iterators.MultiprocessIterator(train, batch, n_processes=args.job) val_iter = chainer.iterators.MultiprocessIterator(validate, val_batch, repeat=False, n_processes=args.job) classifier = chainer.links.Classifier(model) optimizer = chainer.optimizers.Adam() optimizer.setup(classifier) model.base.disable_update() updater = training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (epoch, 'epoch'), output_path) val_interval = 500, 'iteration' log_interval = 250, 'iteration' snapshot_interval = 5000, 'iteration' trainer.extend(extensions.Evaluator(val_iter, classifier, device=args.gpu), trigger=val_interval) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'lr' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if args.resume: chainer.serializers.load_npz(args.resume, trainer) print("Start training") trainer.run() model.to_cpu() chainer.serializers.save_npz(path.join(output_path, "model.npz"), model) print("Uploading files") output.upload_result(output_name) print("Finish training")
def test_trigger(self): snapshot = extensions.snapshot(trigger=self.trigger) self.assertEqual(snapshot.trigger, self.trigger)
if 'Imdb' in args.dataset: train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, 1, repeat=False, shuffle=False) elif 'nli' in args.dataset: train_iter = train test_iter = test updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) trainer.extend(TestModeEvaluator(test_iter, model, device=args.gpu)) #trainer.extend(extensions.ExponentialShift('lr', 0.5), trigger=(25, 'epoch')) trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(extensions.LogReport()) trainer.extend( extensions.LogReport(postprocess=store_model, trigger=(1, 'epoch'))) trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy' ])) trainer.extend(extensions.ProgressBar(update_interval=8)) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def main(): parser = argparse.ArgumentParser(description='chainer implementation of pix2pix') parser.add_argument('--batchsize', '-b', type=int, default=1, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=200, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--dataset', '-i', default='./facade/base', help='Directory of image files.') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--seed', type=int, default=0, help='Random seed') parser.add_argument('--snapshot_interval', type=int, default=1000, help='Interval of snapshot') parser.add_argument('--display_interval', type=int, default=100, help='Interval of displaying log to console') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train enc = Encoder(in_ch=12) dec = Decoder(out_ch=3) dis = Discriminator(in_ch=12, out_ch=3) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current enc.to_gpu() # Copy the model to the GPU dec.to_gpu() dis.to_gpu() # Setup an optimizer def make_optimizer(model, alpha=0.0002, beta1=0.5): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.00001), 'hook_dec') return optimizer opt_enc = make_optimizer(enc) opt_dec = make_optimizer(dec) opt_dis = make_optimizer(dis) train_d = FacadeDataset(args.dataset, data_range=(1,300)) test_d = FacadeDataset(args.dataset, data_range=(300,379)) #train_iter = chainer.iterators.MultiprocessIterator(train_d, args.batchsize, n_processes=4) #test_iter = chainer.iterators.MultiprocessIterator(test_d, args.batchsize, n_processes=4) train_iter = chainer.iterators.SerialIterator(train_d, args.batchsize) test_iter = chainer.iterators.SerialIterator(test_d, args.batchsize) # Set up a trainer updater = FacadeUpdater( models=(enc, dec, dis), iterator={ 'main': train_iter, 'test': test_iter}, optimizer={ 'enc': opt_enc, 'dec': opt_dec, 'dis': opt_dis}, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) snapshot_interval = (args.snapshot_interval, 'iteration') display_interval = (args.display_interval, 'iteration') trainer.extend(extensions.snapshot( filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( enc, 'enc_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dec, 'dec_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'enc/loss', 'dec/loss', 'dis/loss', ]), trigger=display_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.extend( out_image( updater, enc, dec, 5, 5, args.seed, args.out), trigger=snapshot_interval) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def train(args): '''Run training''' # display chainer version logging.info('chainer version = ' + chainer.__version__) # seed setting (chainer seed may not need it) os.environ['CHAINER_SEED'] = str(args.seed) logging.info('chainer seed = ' + os.environ['CHAINER_SEED']) # debug mode setting # 0 would be fastest, but 1 seems to be reasonable # by considering reproducability # revmoe type check if args.debugmode < 2: chainer.config.type_check = False logging.info('chainer type check is disabled') # use determinisitic computation or not if args.debugmode < 1: chainer.config.cudnn_deterministic = False logging.info('chainer cudnn deterministic is disabled') else: chainer.config.cudnn_deterministic = True # check cuda and cudnn availability if not chainer.cuda.available: logging.warning('cuda is not available') if not chainer.cuda.cudnn_enabled: logging.warning('cudnn is not available') # get input and output dimension info with open(args.valid_label, 'rb') as f: valid_json = json.load(f)['utts'] utts = list(valid_json.keys()) idim = int(valid_json[utts[0]]['idim']) odim = int(valid_json[utts[0]]['odim']) logging.info('#input dims : ' + str(idim)) logging.info('#output dims: ' + str(odim)) # check attention type if args.atype not in ['noatt', 'dot', 'location']: raise NotImplementedError( 'chainer supports only noatt, dot, and location attention.') # specify attention, CTC, hybrid mode if args.mtlalpha == 1.0: mtl_mode = 'ctc' logging.info('Pure CTC mode') elif args.mtlalpha == 0.0: mtl_mode = 'att' logging.info('Pure attention mode') else: mtl_mode = 'mtl' logging.info('Multitask learning mode') # specify model architecture e2e = E2E(idim, odim, args) model = Loss(e2e, args.mtlalpha) # write model config if not os.path.exists(args.outdir): os.makedirs(args.outdir) model_conf = args.outdir + '/model.conf' with open(model_conf, 'wb') as f: logging.info('writing a model config file to' + model_conf) # TODO(watanabe) use others than pickle, possibly json, and save as a text pickle.dump((idim, odim, args), f) for key in sorted(vars(args).keys()): logging.info('ARGS: ' + key + ': ' + str(vars(args)[key])) # Set gpu ngpu = args.ngpu if ngpu == 1: gpu_id = 0 # Make a specified GPU current chainer.cuda.get_device_from_id(gpu_id).use() model.to_gpu() # Copy the model to the GPU logging.info('single gpu calculation.') elif ngpu > 1: gpu_id = 0 devices = {'main': gpu_id} for gid in six.moves.xrange(1, ngpu): devices['sub_%d' % gid] = gid logging.info('multi gpu calculation (#gpus = %d).' % ngpu) logging.info('batch size is automatically increased (%d -> %d)' % (args.batch_size, args.batch_size * args.ngpu)) else: gpu_id = -1 logging.info('cpu calculation') # Setup an optimizer if args.opt == 'adadelta': optimizer = chainer.optimizers.AdaDelta(eps=args.eps) elif args.opt == 'adam': optimizer = chainer.optimizers.Adam() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip)) # read json data with open(args.train_label, 'rb') as f: train_json = json.load(f)['utts'] with open(args.valid_label, 'rb') as f: valid_json = json.load(f)['utts'] # prepare Kaldi reader train_reader = lazy_io.read_dict_scp(args.train_feat) valid_reader = lazy_io.read_dict_scp(args.valid_feat) # set up training iterator and updater if ngpu <= 1: # make minibatch list (variable length) train = make_batchset(train_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches) # hack to make batchsize argument as 1 # actual batchsize is included in a list train_iter = chainer.iterators.SerialIterator(train, 1) # set up updater updater = ChainerSeqUpdaterKaldi(train_iter, optimizer, train_reader, gpu_id) else: # set up minibatches train_subsets = [] for gid in six.moves.xrange(ngpu): # make subset train_json_subset = { k: v for i, (k, v) in enumerate(train_json.viewitems()) if i % ngpu == gid } # make minibatch list (variable length) train_subsets += [ make_batchset(train_json_subset, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches) ] # each subset must have same length for MultiprocessParallelUpdater maxlen = max([len(train_subset) for train_subset in train_subsets]) for train_subset in train_subsets: if maxlen != len(train_subset): for i in six.moves.xrange(maxlen - len(train_subset)): train_subset += [train_subset[i]] # hack to make batchsize argument as 1 # actual batchsize is included in a list train_iters = [ chainer.iterators.MultiprocessIterator(train_subsets[gid], 1, n_processes=1) for gid in six.moves.xrange(ngpu) ] # set up updater updater = ChainerMultiProcessParallelUpdaterKaldi( train_iters, optimizer, train_reader, devices) # Set up a trainer trainer = training.Trainer(updater, (args.epochs, 'epoch'), out=args.outdir) # Resume from a snapshot if args.resume: chainer.serializers.load_npz(args.resume, trainer) # set up validation iterator valid = make_batchset(valid_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches) valid_iter = chainer.iterators.SerialIterator(valid, 1, repeat=False, shuffle=False) # Evaluate the model with the test dataset for each epoch trainer.extend( ChainerSeqEvaluaterKaldi(valid_iter, model, valid_reader, device=gpu_id)) # Save attention weight each epoch if args.num_save_attention > 0 and args.mtlalpha != 1.0: data = sorted(valid_json.items()[:args.num_save_attention], key=lambda x: int(x[1]['ilen']), reverse=True) data = converter_kaldi(data, valid_reader) trainer.extend(PlotAttentionReport(model, data, args.outdir + "/att_ws"), trigger=(1, 'epoch')) # Take a snapshot for each specified epoch trainer.extend(extensions.snapshot(), trigger=(1, 'epoch')) # Make a plot for training and validation values trainer.extend( extensions.PlotReport([ 'main/loss', 'validation/main/loss', 'main/loss_ctc', 'validation/main/loss_ctc', 'main/loss_att', 'validation/main/loss_att' ], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(['main/acc', 'validation/main/acc'], 'epoch', file_name='acc.png')) # Save best models trainer.extend( extensions.snapshot_object(model, 'model.loss.best'), trigger=training.triggers.MinValueTrigger('validation/main/loss')) if mtl_mode is not 'ctc': trainer.extend( extensions.snapshot_object(model, 'model.acc.best'), trigger=training.triggers.MaxValueTrigger('validation/main/acc')) # epsilon decay in the optimizer if args.opt == 'adadelta': if args.criterion == 'acc' and mtl_mode is not 'ctc': trainer.extend(restore_snapshot(model, args.outdir + '/model.acc.best'), trigger=CompareValueTrigger( 'validation/main/acc', lambda best_value, current_value: best_value > current_value)) trainer.extend(adadelta_eps_decay(args.eps_decay), trigger=CompareValueTrigger( 'validation/main/acc', lambda best_value, current_value: best_value > current_value)) elif args.criterion == 'loss': trainer.extend(restore_snapshot(model, args.outdir + '/model.loss.best'), trigger=CompareValueTrigger( 'validation/main/loss', lambda best_value, current_value: best_value < current_value)) trainer.extend(adadelta_eps_decay(args.eps_decay), trigger=CompareValueTrigger( 'validation/main/loss', lambda best_value, current_value: best_value < current_value)) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport(trigger=(100, 'iteration'))) report_keys = [ 'epoch', 'iteration', 'main/loss', 'main/loss_ctc', 'main/loss_att', 'validation/main/loss', 'validation/main/loss_ctc', 'validation/main/loss_att', 'main/acc', 'validation/main/acc', 'elapsed_time' ] if args.opt == 'adadelta': trainer.extend(extensions.observe_value( 'eps', lambda trainer: trainer.updater.get_optimizer('main').eps), trigger=(100, 'iteration')) report_keys.append('eps') trainer.extend(extensions.PrintReport(report_keys), trigger=(100, 'iteration')) trainer.extend(extensions.ProgressBar()) # Run the training trainer.run()
def main(): archs = { 'alex': alex.Alex, 'alex_fp16': alex.AlexFp16, 'googlenet': googlenet.GoogLeNet, 'googlenetbn': googlenetbn.GoogLeNetBN, 'googlenetbn_fp16': googlenetbn.GoogLeNetBNFp16, 'nin': nin.NIN, 'resnet50': resnet50.ResNet50, 'resnext50': resnet50.ResNeXt50, } parser = argparse.ArgumentParser( description='Learning convnet from ILSVRC2012 dataset') parser.add_argument('train', help='Path to training image-label list file') parser.add_argument('val', help='Path to validation image-label list file') parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin', help='Convnet architecture') parser.add_argument('--batchsize', '-B', type=int, default=32, help='Learning minibatch size') parser.add_argument('--epoch', '-E', type=int, default=10, help='Number of epochs to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU') parser.add_argument('--initmodel', help='Initialize the model from given file') parser.add_argument('--loaderjob', '-j', type=int, help='Number of parallel data loading processes') parser.add_argument('--mean', '-m', default='mean.npy', help='Mean file (computed by compute_mean.py)') parser.add_argument('--resume', '-r', default='', help='Initialize the trainer from given file') parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--root', '-R', default='.', help='Root directory path of image files') parser.add_argument('--val_batchsize', '-b', type=int, default=250, help='Validation minibatch size') parser.add_argument('--test', action='store_true') parser.set_defaults(test=False) args = parser.parse_args() # Initialize the model to train model = archs[args.arch]() if args.initmodel: print('Load model from {}'.format(args.initmodel)) chainer.serializers.load_npz(args.initmodel, model) if args.gpu >= 0: chainer.backends.cuda.get_device_from_id( args.gpu).use() # Make the GPU current model.to_gpu() # Load the datasets and mean file mean = np.load(args.mean) train = PreprocessedDataset(args.train, args.root, mean, model.insize) val = PreprocessedDataset(args.val, args.root, mean, model.insize, False) # These iterators load the images with subprocesses running in parallel to # the training/validation. train_iter = chainer.iterators.MultiprocessIterator( train, args.batchsize, n_processes=args.loaderjob) val_iter = chainer.iterators.MultiprocessIterator( val, args.val_batchsize, repeat=False, n_processes=args.loaderjob) # Set up an optimizer optimizer = chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) # Set up a trainer updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out) val_interval = (1 if args.test else 100000), 'iteration' log_interval = (1 if args.test else 1000), 'iteration' trainer.extend(extensions.Evaluator(val_iter, model, device=args.gpu), trigger=val_interval) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=val_interval) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=val_interval) # Be careful to pass the interval directly to LogReport # (it determines when to emit log rather than when to read observations) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'lr' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def train_model(self, datasets): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--dataset', '-d', default='cifar10', help='The dataset to use: cifar10 or cifar100') parser.add_argument('--batchsize', '-b', type=int, default=10, help='Number of images in each mini-batch') parser.add_argument('--learnrate', '-l', type=float, default=0.05, help='Learning rate for SGD') parser.add_argument('--epoch', '-e', type=int, default=300, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--early-stopping', type=str, help='Metric to watch for early stopping') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) if args.gpu >= 0: chainer.backends.cuda.get_device_from_id(args.gpu).use() self.model.to_gpu() optimizer = chainer.optimizers.Adam(args.learnrate) optimizer.setup(self.model) optimizer.add_hook(chainer.optimizer_hooks.WeightDecay(5e-4)) train, test = split_dataset(datasets, 80) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) stop_trigger = (args.epoch, 'epoch') # Early stopping option if args.early_stopping: stop_trigger = triggers.EarlyStoppingTrigger( monitor=args.early_stopping, verbose=True, max_trigger=(args.epoch, 'epoch')) # Set up a trainer updater = training.updaters.StandardUpdater( train_iter, optimizer, device=args.gpu, loss_func=mean_squared_error) trainer = training.Trainer(updater, stop_trigger, out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend( extensions.Evaluator(test_iter, self.model, device=args.gpu)) # Reduce the learning rate by half every 25 epochs. trainer.extend(extensions.ExponentialShift('lr', 0.5), trigger=(25, 'epoch')) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend( extensions.snapshot(filename='snaphot_epoch_{.updater.epoch}')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) print(train[:1]) # Run the training trainer.run() return self.model
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--dataset', '-d', default='cifar10', help='The dataset to use: cifar10 or cifar100') parser.add_argument('--batchsize', '-b', type=int, default=64, help='Number of images in each mini-batch') parser.add_argument('--learnrate', '-l', type=float, default=0.05, help='Learning rate for SGD') parser.add_argument('--epoch', '-e', type=int, default=300, help='Number of sweeps over the dataset to train') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') args = parser.parse_args() print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train. # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. if args.dataset == 'cifar10': print('Using CIFAR10 dataset.') class_labels = 10 train, test = get_cifar10() elif args.dataset == 'cifar100': print('Using CIFAR100 dataset.') class_labels = 100 train, test = get_cifar100() else: raise RuntimeError('Invalid dataset choice.') model = L.Classifier(models.VGG.VGG(class_labels)) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current model.to_gpu() # Copy the model to the GPU optimizer = chainer.optimizers.MomentumSGD(args.learnrate) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4)) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(TestModeEvaluator(test_iter, model, device=args.gpu)) # Reduce the learning rate by half every 25 epochs. trainer.extend(extensions.ExponentialShift('lr', 0.5), trigger=(25, 'epoch')) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot at each epoch trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
def train(args): """Train with the given args. Args: args (namespace): The program arguments. """ # display chainer version logging.info('chainer version = ' + chainer.__version__) set_deterministic_chainer(args) # check cuda and cudnn availability if not chainer.cuda.available: logging.warning('cuda is not available') if not chainer.cuda.cudnn_enabled: logging.warning('cudnn is not available') # get input and output dimension info with open(args.valid_json, 'rb') as f: valid_json = json.load(f)['utts'] utts = list(valid_json.keys()) idim = int(valid_json[utts[0]]['input'][0]['shape'][1]) odim = int(valid_json[utts[0]]['output'][0]['shape'][1]) logging.info('#input dims : ' + str(idim)) logging.info('#output dims: ' + str(odim)) # specify attention, CTC, hybrid mode if args.mtlalpha == 1.0: mtl_mode = 'ctc' logging.info('Pure CTC mode') elif args.mtlalpha == 0.0: mtl_mode = 'att' logging.info('Pure attention mode') else: mtl_mode = 'mtl' logging.info('Multitask learning mode') # specify model architecture logging.info('import model module: ' + args.model_module) model_class = dynamic_import(args.model_module) model = model_class(idim, odim, args, flag_return=False) assert isinstance(model, ASRInterface) # write model config if not os.path.exists(args.outdir): os.makedirs(args.outdir) model_conf = args.outdir + '/model.json' with open(model_conf, 'wb') as f: logging.info('writing a model config file to ' + model_conf) f.write( json.dumps((idim, odim, vars(args)), indent=4, ensure_ascii=False, sort_keys=True).encode('utf_8')) for key in sorted(vars(args).keys()): logging.info('ARGS: ' + key + ': ' + str(vars(args)[key])) # Set gpu ngpu = args.ngpu if ngpu == 1: gpu_id = 0 # Make a specified GPU current chainer.cuda.get_device_from_id(gpu_id).use() model.to_gpu() # Copy the model to the GPU logging.info('single gpu calculation.') elif ngpu > 1: gpu_id = 0 devices = {'main': gpu_id} for gid in six.moves.xrange(1, ngpu): devices['sub_%d' % gid] = gid logging.info('multi gpu calculation (#gpus = %d).' % ngpu) logging.warning('batch size is automatically increased (%d -> %d)' % (args.batch_size, args.batch_size * args.ngpu)) else: gpu_id = -1 logging.info('cpu calculation') # Setup an optimizer if args.opt == 'adadelta': optimizer = chainer.optimizers.AdaDelta(eps=args.eps) elif args.opt == 'adam': optimizer = chainer.optimizers.Adam() elif args.opt == 'noam': optimizer = chainer.optimizers.Adam(alpha=0, beta1=0.9, beta2=0.98, eps=1e-9) else: raise NotImplementedError('args.opt={}'.format(args.opt)) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip)) # Setup Training Extensions if 'transformer' in args.model_module: from espnet.nets.chainer_backend.transformer.training import CustomConverter from espnet.nets.chainer_backend.transformer.training import CustomParallelUpdater from espnet.nets.chainer_backend.transformer.training import CustomUpdater else: from espnet.nets.chainer_backend.rnn.training import CustomConverter from espnet.nets.chainer_backend.rnn.training import CustomParallelUpdater from espnet.nets.chainer_backend.rnn.training import CustomUpdater # Setup a converter converter = CustomConverter(subsampling_factor=model.subsample[0]) # read json data with open(args.train_json, 'rb') as f: train_json = json.load(f)['utts'] with open(args.valid_json, 'rb') as f: valid_json = json.load(f)['utts'] # set up training iterator and updater load_tr = LoadInputsAndTargets( mode='asr', load_output=True, preprocess_conf=args.preprocess_conf, preprocess_args={'train': True} # Switch the mode of preprocessing ) load_cv = LoadInputsAndTargets( mode='asr', load_output=True, preprocess_conf=args.preprocess_conf, preprocess_args={'train': False} # Switch the mode of preprocessing ) use_sortagrad = args.sortagrad == -1 or args.sortagrad > 0 accum_grad = args.accum_grad if ngpu <= 1: # make minibatch list (variable length) train = make_batchset(train_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches, min_batch_size=args.ngpu if args.ngpu > 1 else 1, shortest_first=use_sortagrad, count=args.batch_count, batch_bins=args.batch_bins, batch_frames_in=args.batch_frames_in, batch_frames_out=args.batch_frames_out, batch_frames_inout=args.batch_frames_inout, iaxis=0, oaxis=0) # hack to make batchsize argument as 1 # actual batchsize is included in a list if args.n_iter_processes > 0: train_iters = [ ToggleableShufflingMultiprocessIterator( TransformDataset(train, load_tr), batch_size=1, n_processes=args.n_iter_processes, n_prefetch=8, maxtasksperchild=20, shuffle=not use_sortagrad) ] else: train_iters = [ ToggleableShufflingSerialIterator(TransformDataset( train, load_tr), batch_size=1, shuffle=not use_sortagrad) ] # set up updater updater = CustomUpdater(train_iters[0], optimizer, converter=converter, device=gpu_id, accum_grad=accum_grad) else: if args.batch_count not in ("auto", "seq") and args.batch_size == 0: raise NotImplementedError( "--batch-count 'bin' and 'frame' are not implemented in chainer multi gpu" ) # set up minibatches train_subsets = [] for gid in six.moves.xrange(ngpu): # make subset train_json_subset = { k: v for i, (k, v) in enumerate(train_json.items()) if i % ngpu == gid } # make minibatch list (variable length) train_subsets += [ make_batchset(train_json_subset, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches) ] # each subset must have same length for MultiprocessParallelUpdater maxlen = max([len(train_subset) for train_subset in train_subsets]) for train_subset in train_subsets: if maxlen != len(train_subset): for i in six.moves.xrange(maxlen - len(train_subset)): train_subset += [train_subset[i]] # hack to make batchsize argument as 1 # actual batchsize is included in a list if args.n_iter_processes > 0: train_iters = [ ToggleableShufflingMultiprocessIterator( TransformDataset(train_subsets[gid], load_tr), batch_size=1, n_processes=args.n_iter_processes, n_prefetch=8, maxtasksperchild=20, shuffle=not use_sortagrad) for gid in six.moves.xrange(ngpu) ] else: train_iters = [ ToggleableShufflingSerialIterator(TransformDataset( train_subsets[gid], load_tr), batch_size=1, shuffle=not use_sortagrad) for gid in six.moves.xrange(ngpu) ] # set up updater updater = CustomParallelUpdater(train_iters, optimizer, converter=converter, devices=devices) # Set up a trainer trainer = training.Trainer(updater, (args.epochs, 'epoch'), out=args.outdir) if use_sortagrad: trainer.extend( ShufflingEnabler(train_iters), trigger=(args.sortagrad if args.sortagrad != -1 else args.epochs, 'epoch')) if args.opt == 'noam': from espnet.nets.chainer_backend.transformer.training import VaswaniRule trainer.extend(VaswaniRule('alpha', d=args.adim, warmup_steps=args.transformer_warmup_steps, scale=args.transformer_lr), trigger=(1, 'iteration')) # Resume from a snapshot if args.resume: chainer.serializers.load_npz(args.resume, trainer) # set up validation iterator valid = make_batchset(valid_json, args.batch_size, args.maxlen_in, args.maxlen_out, args.minibatches, min_batch_size=args.ngpu if args.ngpu > 1 else 1, count=args.batch_count, batch_bins=args.batch_bins, batch_frames_in=args.batch_frames_in, batch_frames_out=args.batch_frames_out, batch_frames_inout=args.batch_frames_inout, iaxis=0, oaxis=0) if args.n_iter_processes > 0: valid_iter = chainer.iterators.MultiprocessIterator( TransformDataset(valid, load_cv), batch_size=1, repeat=False, shuffle=False, n_processes=args.n_iter_processes, n_prefetch=8, maxtasksperchild=20) else: valid_iter = chainer.iterators.SerialIterator(TransformDataset( valid, load_cv), batch_size=1, repeat=False, shuffle=False) # Evaluate the model with the test dataset for each epoch trainer.extend( BaseEvaluator(valid_iter, model, converter=converter, device=gpu_id)) # Save attention weight each epoch if args.num_save_attention > 0 and args.mtlalpha != 1.0: data = sorted(list(valid_json.items())[:args.num_save_attention], key=lambda x: int(x[1]['input'][0]['shape'][1]), reverse=True) if hasattr(model, "module"): att_vis_fn = model.module.calculate_all_attentions plot_class = model.module.attention_plot_class else: att_vis_fn = model.calculate_all_attentions plot_class = model.attention_plot_class logging.info('Using custom PlotAttentionReport') att_reporter = plot_class(att_vis_fn, data, args.outdir + "/att_ws", converter=converter, transform=load_cv, device=gpu_id) trainer.extend(att_reporter, trigger=(1, 'epoch')) else: att_reporter = None # Take a snapshot for each specified epoch trainer.extend( extensions.snapshot(filename='snapshot.ep.{.updater.epoch}'), trigger=(1, 'epoch')) # Make a plot for training and validation values trainer.extend( extensions.PlotReport([ 'main/loss', 'validation/main/loss', 'main/loss_ctc', 'validation/main/loss_ctc', 'main/loss_att', 'validation/main/loss_att' ], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(['main/acc', 'validation/main/acc'], 'epoch', file_name='acc.png')) # Save best models trainer.extend( extensions.snapshot_object(model, 'model.loss.best'), trigger=training.triggers.MinValueTrigger('validation/main/loss')) if mtl_mode != 'ctc': trainer.extend( extensions.snapshot_object(model, 'model.acc.best'), trigger=training.triggers.MaxValueTrigger('validation/main/acc')) # epsilon decay in the optimizer if args.opt == 'adadelta': if args.criterion == 'acc' and mtl_mode != 'ctc': trainer.extend(restore_snapshot(model, args.outdir + '/model.acc.best'), trigger=CompareValueTrigger( 'validation/main/acc', lambda best_value, current_value: best_value > current_value)) trainer.extend(adadelta_eps_decay(args.eps_decay), trigger=CompareValueTrigger( 'validation/main/acc', lambda best_value, current_value: best_value > current_value)) elif args.criterion == 'loss': trainer.extend(restore_snapshot(model, args.outdir + '/model.loss.best'), trigger=CompareValueTrigger( 'validation/main/loss', lambda best_value, current_value: best_value < current_value)) trainer.extend(adadelta_eps_decay(args.eps_decay), trigger=CompareValueTrigger( 'validation/main/loss', lambda best_value, current_value: best_value < current_value)) # Write a log of evaluation statistics for each epoch trainer.extend( extensions.LogReport(trigger=(args.report_interval_iters, 'iteration'))) report_keys = [ 'epoch', 'iteration', 'main/loss', 'main/loss_ctc', 'main/loss_att', 'validation/main/loss', 'validation/main/loss_ctc', 'validation/main/loss_att', 'main/acc', 'validation/main/acc', 'elapsed_time' ] if args.opt == 'adadelta': trainer.extend(extensions.observe_value( 'eps', lambda trainer: trainer.updater.get_optimizer('main').eps), trigger=(args.report_interval_iters, 'iteration')) report_keys.append('eps') trainer.extend(extensions.PrintReport(report_keys), trigger=(args.report_interval_iters, 'iteration')) trainer.extend( extensions.ProgressBar(update_interval=args.report_interval_iters)) set_early_stop(trainer, args) if args.tensorboard_dir is not None and args.tensorboard_dir != "": writer = SummaryWriter(args.tensorboard_dir) trainer.extend(TensorboardLogger(writer, att_reporter), trigger=(args.report_interval_iters, 'iteration')) # Run the training trainer.run() check_early_stop(trainer, args.epochs)
def main(): archs = { 'alex': alex.Alex, 'alex_fp16': alex.AlexFp16, 'googlenet': googlenet.GoogLeNet, 'googlenetbn': googlenetbn.GoogLeNetBN, 'googlenetbn_fp16': googlenetbn.GoogLeNetBNFp16, 'nin': nin.NIN, 'resnet50': resnet50.ResNet50, 'resnext50': resnext50.ResNeXt50, } parser = argparse.ArgumentParser( description='Learning convnet from ILSVRC2012 dataset') parser.add_argument('train', help='Path to training image-label list file') parser.add_argument('val', help='Path to validation image-label list file') parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin', help='Convnet architecture') parser.add_argument('--batchsize', '-B', type=int, default=32, help='Learning minibatch size') parser.add_argument('--epoch', '-E', type=int, default=10, help='Number of epochs to train') parser.add_argument('--device', '-d', type=str, default='-1', help='Device specifier. Either ChainerX device ' 'specifier or an integer. If non-negative integer, ' 'CuPy arrays with specified device id are used. If ' 'negative integer, NumPy arrays are used') parser.add_argument('--initmodel', help='Initialize the model from given file') parser.add_argument('--loaderjob', '-j', type=int, help='Number of parallel data loading processes') parser.add_argument('--mean', '-m', default='mean.npy', help='Mean file (computed by compute_mean.py)') parser.add_argument('--resume', '-r', default='', help='Initialize the trainer from given file') parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--root', '-R', default='.', help='Root directory path of image files') parser.add_argument('--val_batchsize', '-b', type=int, default=250, help='Validation minibatch size') parser.add_argument('--test', action='store_true') parser.set_defaults(test=False) parser.add_argument('--dali', action='store_true') parser.set_defaults(dali=False) group = parser.add_argument_group('deprecated arguments') group.add_argument('--gpu', '-g', type=int, nargs='?', const=0, help='GPU ID (negative value indicates CPU)') args = parser.parse_args() device = parse_device(args) print('Device: {}'.format(device)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Initialize the model to train model = archs[args.arch]() if args.initmodel: print('Load model from {}'.format(args.initmodel)) chainer.serializers.load_npz(args.initmodel, model) model.to_device(device) device.use() # Load the mean file mean = np.load(args.mean) if args.dali: if not dali_util._dali_available: raise RuntimeError('DALI seems not available on your system.') num_threads = args.loaderjob if num_threads is None or num_threads <= 0: num_threads = 1 ch_mean = list(np.average(mean, axis=(1, 2))) ch_std = [255.0, 255.0, 255.0] # Setup DALI pipelines train_pipe = dali_util.DaliPipelineTrain( args.train, args.root, model.insize, args.batchsize, num_threads, args.gpu, True, mean=ch_mean, std=ch_std) val_pipe = dali_util.DaliPipelineVal( args.val, args.root, model.insize, args.val_batchsize, num_threads, args.gpu, False, mean=ch_mean, std=ch_std) train_iter = chainer.iterators.DaliIterator(train_pipe) val_iter = chainer.iterators.DaliIterator(val_pipe, repeat=False) # converter = dali_converter converter = dali_util.DaliConverter(mean=mean, crop_size=model.insize) else: # Load the dataset files train = PreprocessedDataset(args.train, args.root, mean, model.insize) val = PreprocessedDataset(args.val, args.root, mean, model.insize, False) # These iterators load the images with subprocesses running in parallel # to the training/validation. train_iter = chainer.iterators.MultiprocessIterator( train, args.batchsize, n_processes=args.loaderjob) val_iter = chainer.iterators.MultiprocessIterator( val, args.val_batchsize, repeat=False, n_processes=args.loaderjob) converter = dataset.concat_examples # Set up an optimizer optimizer = chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) # Set up a trainer updater = training.updaters.StandardUpdater( train_iter, optimizer, converter=converter, device=device) trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out) val_interval = (1 if args.test else 100000), 'iteration' log_interval = (1 if args.test else 1000), 'iteration' trainer.extend(extensions.Evaluator(val_iter, model, converter=converter, device=device), trigger=val_interval) # TODO(sonots): Temporarily disabled for chainerx. Fix it. if not (chainerx.is_available() and isinstance(device, chainerx.Device)): trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=val_interval) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=val_interval) # Be careful to pass the interval directly to LogReport # (it determines when to emit log rather than when to read observations) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'lr' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def main(): archs = { 'alex': alex.Alex, 'alex_fp16': alex.AlexFp16, 'googlenet': googlenet.GoogLeNet, 'googlenetbn': googlenetbn.GoogLeNetBN, 'googlenetbn_fp16': googlenetbn.GoogLeNetBNFp16, 'nin': nin.NIN, 'resnet50': resnet50.ResNet50, 'resnext50': resnet50.ResNeXt50, } parser = argparse.ArgumentParser( description='Learning convnet from ILSVRC2012 dataset') parser.add_argument('train', help='Path to training image-label list file') parser.add_argument('val', help='Path to validation image-label list file') parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin', help='Convnet architecture') parser.add_argument('--batchsize', '-B', type=int, default=32, help='Learning minibatch size') parser.add_argument('--epoch', '-E', type=int, default=10, help='Number of epochs to train') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU') parser.add_argument('--initmodel', help='Initialize the model from given file') parser.add_argument('--loaderjob', '-j', type=int, help='Number of parallel data loading processes') parser.add_argument('--mean', '-m', default='mean.npy', help='Mean file (computed by compute_mean.py)') parser.add_argument('--resume', '-r', default='', help='Initialize the trainer from given file') parser.add_argument('--out', '-o', default='result', help='Output directory') parser.add_argument('--root', '-R', default='.', help='Root directory path of image files') parser.add_argument('--val_batchsize', '-b', type=int, default=250, help='Validation minibatch size') parser.add_argument('--test', action='store_true') parser.set_defaults(test=False) args = parser.parse_args() # Initialize the model to train model = archs[args.arch]() if args.initmodel: print('Load model from', args.initmodel) chainer.serializers.load_npz(args.initmodel, model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() # Make the GPU current model.to_gpu() # Load the datasets and mean file mean = np.load(args.mean) train = PreprocessedDataset(args.train, args.root, mean, model.insize) val = PreprocessedDataset(args.val, args.root, mean, model.insize, False) # These iterators load the images with subprocesses running in parallel to # the training/validation. train_iter = chainer.iterators.MultiprocessIterator( train, args.batchsize, n_processes=args.loaderjob) val_iter = chainer.iterators.MultiprocessIterator( val, args.val_batchsize, repeat=False, n_processes=args.loaderjob) # Set up an optimizer optimizer = chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) # Set up a trainer updater = training.updaters.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out) val_interval = (1 if args.test else 100000), 'iteration' log_interval = (1 if args.test else 1000), 'iteration' trainer.extend(extensions.Evaluator(val_iter, model, device=args.gpu), trigger=val_interval) trainer.extend(extensions.dump_graph('main/loss')) trainer.extend(extensions.snapshot(), trigger=val_interval) trainer.extend(extensions.snapshot_object( model, 'model_iter_{.updater.iteration}'), trigger=val_interval) # Be careful to pass the interval directly to LogReport # (it determines when to emit log rather than when to read observations) trainer.extend(extensions.LogReport(trigger=log_interval)) trainer.extend(extensions.observe_lr(), trigger=log_interval) trainer.extend(extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'lr' ]), trigger=log_interval) trainer.extend(extensions.ProgressBar(update_interval=10)) if args.resume: chainer.serializers.load_npz(args.resume, trainer) trainer.run()
def main(): args = parse_arguments() # Set up some useful variables that will be used later on. dataset_name = args.dataset method = args.method num_data = args.num_data n_unit = args.unit_num conv_layers = args.conv_layers task_type = molnet_default_config[dataset_name]['task_type'] model_filename = {'classification': 'classifier.pkl', 'regression': 'regressor.pkl'} print('Using dataset: {}...'.format(dataset_name)) # Set up some useful variables that will be used later on. if args.label: labels = args.label cache_dir = os.path.join('input', '{}_{}_{}'.format(dataset_name, method, labels)) class_num = len(labels) if isinstance(labels, list) else 1 else: labels = None cache_dir = os.path.join('input', '{}_{}_all'.format(dataset_name, method)) class_num = len(molnet_default_config[args.dataset]['tasks']) # Load the train and validation parts of the dataset. filenames = [dataset_part_filename(p, num_data) for p in ['train', 'valid']] paths = [os.path.join(cache_dir, f) for f in filenames] if all([os.path.exists(path) for path in paths]): dataset_parts = [] for path in paths: print('Loading cached dataset from {}.'.format(path)) dataset_parts.append(NumpyTupleDataset.load(path)) else: dataset_parts = download_entire_dataset(dataset_name, num_data, labels, method, cache_dir) train, valid = dataset_parts[0], dataset_parts[1] # # Scale the label values, if necessary. # if args.scale == 'standardize': # if task_type == 'regression': # print('Applying standard scaling to the labels.') # datasets, scaler = standardize_dataset_labels(datasets) # else: # print('Label scaling is not available for classification tasks.') # else: # print('No label scaling was selected.') # scaler = None # Set up the predictor. predictor = set_up_predictor(method, n_unit, conv_layers, class_num) # Set up the iterators. train_iter = iterators.SerialIterator(train, args.batchsize) valid_iter = iterators.SerialIterator(valid, args.batchsize, repeat=False, shuffle=False) # Load metrics for the current dataset. metrics = molnet_default_config[dataset_name]['metrics'] metrics_fun = {k: v for k, v in metrics.items() if isinstance(v, types.FunctionType)} loss_fun = molnet_default_config[dataset_name]['loss'] if task_type == 'regression': model = Regressor(predictor, lossfun=loss_fun, metrics_fun=metrics_fun, device=args.gpu) # TODO: Use standard scaler for regression task elif task_type == 'classification': model = Classifier(predictor, lossfun=loss_fun, metrics_fun=metrics_fun, device=args.gpu) else: raise ValueError('Invalid task type ({}) encountered when processing ' 'dataset ({}).'.format(task_type, dataset_name)) # Set up the optimizer. optimizer = optimizers.Adam() optimizer.setup(model) # Save model-related output to this directory. model_dir = os.path.join(args.out, os.path.basename(cache_dir)) if not os.path.exists(model_dir): os.makedirs(model_dir) # Set up the updater. updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu, converter=concat_mols) # Set up the trainer. trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=model_dir) trainer.extend(E.Evaluator(valid_iter, model, device=args.gpu, converter=concat_mols)) trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch')) trainer.extend(E.LogReport()) # Report various metrics. print_report_targets = ['epoch', 'main/loss', 'validation/main/loss'] for metric_name, metric_fun in metrics.items(): if isinstance(metric_fun, types.FunctionType): print_report_targets.append('main/' + metric_name) print_report_targets.append('validation/main/' + metric_name) elif issubclass(metric_fun, BatchEvaluator): trainer.extend(metric_fun(valid_iter, model, device=args.gpu, eval_func=predictor, converter=concat_mols, name='val', raise_value_error=False)) print_report_targets.append('val/main/' + metric_name) else: raise TypeError('{} is not a supported metrics function.' .format(type(metrics_fun))) print_report_targets.append('elapsed_time') # Augmented by Ishiguro # ToDo: consider go/no-go of the following block # (i) more reporting for val/evalutaion # (ii) best validation score snapshot if task_type == 'regression': if 'RMSE' in metric_name: trainer.extend(E.snapshot_object(model, "best_val_" + model_filename[task_type]), trigger=training.triggers.MinValueTrigger('validation/main/RMSE')) elif 'MAE' in metric_name: trainer.extend(E.snapshot_object(model, "best_val_" + model_filename[task_type]), trigger=training.triggers.MinValueTrigger('validation/main/MAE')) else: print("No validation metric defined?") assert(False) elif task_type == 'classification': train_eval_iter = iterators.SerialIterator(train, args.batchsize,repeat=False, shuffle=False) trainer.extend(ROCAUCEvaluator( train_eval_iter, predictor, eval_func=predictor, device=args.gpu, converter=concat_mols, name='train', pos_labels=1, ignore_labels=-1, raise_value_error=False)) # extension name='validation' is already used by `Evaluator`, # instead extension name `val` is used. trainer.extend(ROCAUCEvaluator( valid_iter, predictor, eval_func=predictor, device=args.gpu, converter=concat_mols, name='val', pos_labels=1, ignore_labels=-1)) print_report_targets.append('train/main/roc_auc') print_report_targets.append('validation/main/loss') print_report_targets.append('val/main/roc_auc') trainer.extend(E.snapshot_object(model, "best_val_" + model_filename[task_type]), trigger=training.triggers.MaxValueTrigger('val/main/roc_auc')) else: raise NotImplementedError( 'Not implemented task_type = {}'.format(task_type)) trainer.extend(E.PrintReport(print_report_targets)) trainer.extend(E.ProgressBar()) trainer.run() # Save the model's parameters. model_path = os.path.join(model_dir, model_filename[task_type]) print('Saving the trained model to {}...'.format(model_path)) model.save_pickle(model_path, protocol=args.protocol)
model = VGG16.VGG16() if gpuId >= 0: model.to_gpu(gpuId) optimizer = chainer.optimizers.MomentumSGD() optimizer.setup(model) model.base.disable_update() model = L.Classifier(model) updater = training.StandardUpdater(trainIter, optimizer, device=gpuId) trainer = training.Trainer( updater, (maxEpoch, 'epoch'), out='/home/yusuke/dataset/style-color-images/result') trainer.extend(extensions.LogReport()) trainer.extend(extensions.snapshot(filename='snapshot_epoch-{.updater.epoch}')) trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'main/accuracy', 'val/main/loss', 'val/main/accuracy', 'l1/W/data/std', 'elapsed_time' ])) trainer.extend( extensions.PlotReport(['l1/W/data/std'], x_key='epoch', file_name='std.png')) trainer.extend( extensions.PlotReport(['main/loss', 'val/main/loss'], x_key='epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport(['main/accuracy', 'val/main/accuracy'],