f_train = K.function(inputs, [ce, acc, cm], updates=updates, training=True) print('Building testing functions ...') f_score = K.function(inputs, [ce, acc, cm], training=False) print('Building predicting functions ...') f_pred_proba = K.function(X, y_proba, training=False) # Latent spaces f_z1 = K.function(inputs=X, outputs=z1, training=False) f_z2 = K.function(inputs=X, outputs=z2, training=False) f_z3 = K.function(inputs=X, outputs=z3, training=False) # =========================================================================== # Training # =========================================================================== print('Start training ...') task = training.MainLoop(batch_size=args.batch, seed=1234, shuffle_level=2, allow_rollback=True, labels=labels) task.set_checkpoint(MODEL_PATH, f) task.set_callbacks([ training.NaNDetector(), training.EarlyStopGeneralizationLoss('valid', ce, threshold=5, patience=5) ]) task.set_train_task(func=f_train, data=train, epoch=args.epoch, name='train') task.set_valid_task(func=f_score, data=valid, freq=training.Timer(percentage=0.8), name='valid') task.run() # =========================================================================== # Prediction
# Build trainer # =========================================================================== # ====== spliting the data ====== # idx = np.arange(len(X_train), dtype='int32') idx_train, idx_valid = train_valid_test_split(idx, train=0.8, inc_test=False, seed=1234) X_valid = X_train[idx_valid] y_valid = y_train[idx_valid] X_train = X_train[idx_train] y_train = y_train[idx_train] print("#Train:", X_train.shape, y_train.shape) print("#Valid:", X_valid.shape, y_valid.shape) print("#Test:", X_test.shape, y_test.shape) # ====== trainign ====== # print('Start training ...') task = training.MainLoop(batch_size=128, seed=1234, shuffle_level=2, allow_rollback=True) task.set_checkpoint(MODEL_PATH, model) task.set_callbacks([ training.NaNDetector(), training.EarlyStopGeneralizationLoss('valid', ce, threshold=5, patience=3) ]) task.set_train_task(func=f_train, data=(X_train, y_train), epoch=NB_EPOCH, name='train') task.set_valid_task(func=f_test, data=(X_valid, y_valid), freq=training.Timer(percentage=0.6), name='valid') task.set_eval_task(func=f_test, data=(X_test, y_test),
verbose=True) K.initialize_all_variables() # # ====== Functions ====== # print('Building training functions ...') f_train = K.function(inputs, [ce, acc], updates=updates, training=True) print('Building testing functions ...') f_score = K.function(inputs, [ce, acc], training=False) # Latent spaces f_z = K.function(inputs=X, outputs=z, training=False) # =========================================================================== # Create trainer # =========================================================================== if TRAIN_MODEL: print('Start training ...') task = training.MainLoop(batch_size=args.batch, seed=120825, shuffle_level=2, allow_rollback=True) task.set_checkpoint(MODEL_PATH, x_vec) task.set_callbacks([ training.NaNDetector(), training.EarlyStopGeneralizationLoss('valid', ce, threshold=5, patience=5) ]) task.set_train_task(func=f_train, data=train, epoch=args.epoch, name='train') task.set_valid_task(func=f_score, data=valid,
parameters = f.parameters print('Parameters:', [p.name for p in parameters]) optz = K.optimizers.RMSProp() updates = optz.get_updates(cost_train, parameters) print("Build training function ...") f_train = K.function([X, y_true], cost_train, updates=updates) print("Build scoring function ...") f_score = K.function([X, y_true], [cost_pred, cost_eval]) # =========================================================================== # Create trainer # =========================================================================== print("Create trainer ...") trainer = training.MainLoop(batch_size=32, seed=12082518, shuffle_level=2) trainer.set_save(utils.get_modelpath('cifar10.ai', override=True), f) trainer.set_task(f_train, [X_learn, y_learn], epoch=25, p=1, name='Train') trainer.set_subtask(f_score, [X_test, y_test], freq=1, name='Valid') trainer.set_callback([ training.ProgressMonitor(name='Train', format='Results: {:.4f}'), training.ProgressMonitor(name='Valid', format='Results: {:.4f},{:.4f}'), # early stop based on crossentropy on test (not a right procedure, # but only for testing) training.EarlyStopGeneralizationLoss( name='Valid', threshold=5, patience=3, get_value=lambda x: np.mean([j for i, j in x])), training.History() ])
parameters = ops.parameters optimizer = K.optimizers.SGD(lr=arg['lr']) updates = optimizer(cost_train, parameters) print('Building training functions ...') f_train = K.function([X, y], [cost_train, optimizer.norm], updates=updates) print('Building testing functions ...') f_test = K.function([X, y], [cost_test_1, cost_test_2, cost_test_3]) print('Building predicting functions ...') f_pred = K.function(X, y_pred_score) # =========================================================================== # Build trainer # =========================================================================== print('Start training ...') task = training.MainLoop(batch_size=64, seed=12, shuffle_level=2) task.set_save(get_modelpath(name='mnist.ai', override=True), ops) task.set_task(f_train, (ds['X_train'], ds['y_train']), epoch=arg['epoch'], name='train') task.set_subtask(f_test, (ds['X_test'], ds['y_test']), freq=0.6, name='valid') task.set_subtask(f_test, (ds['X_test'], ds['y_test']), when=-1, name='test') task.set_callback([ training.ProgressMonitor(name='train', format='Results: {:.4f}-{:.4f}'), training.ProgressMonitor(name='valid', format='Results: {:.4f}-{:.4f}', tracking={2: lambda x: sum(x)}), training.ProgressMonitor(name='test', format='Results: {:.4f}-{:.4f}'), training.History(), training.EarlyStopGeneralizationLoss('valid', threshold=5, patience=3), training.NaNDetector(('train', 'valid'), patience=3, rollback=True)
total_stdev=stdev_total, fontsize=8, title="Test Sample #%d" % i if j == 0 else None, **kws) curr_grid_index += 3 V.plot_save(os.path.join(FIGURE_PATH, 'latent_%d.png' % curr_epoch), dpi=200, log=True) exit() # ====== training ====== # runner = T.MainLoop(batch_size=args.batch, seed=1234, shuffle_level=2, allow_rollback=False, verbose=2) runner.set_callbacks([ T.NaNDetector(task_name=None, patience=-1, detect_inf=True), None if args.no_monitor else T.EpochSummary( task_name=('train', 'valid'), output_name=(loss, iw_loss, KL_mean, NLLK_mean), print_plot=False, save_path=os.path.join(FIGURE_PATH, 'summary.png')), T.LambdaCallback(fn=plot_epoch, task_name='train') ]) runner.set_train_task(func=f_train, data=[X_train, X_train], epoch=args.epoch, name='train')
print('Building training functions ...') f_train = K.function(inputs, [ce, acc, optimizer.norm], updates=updates, training=True) print('Building testing functions ...') f_score = K.function(inputs, [ce, acc], training=False) # Latent spaces f_z = K.function(inputs=X, outputs=z, training=False) # =========================================================================== # Create trainer # =========================================================================== print('Start training ...') task = training.MainLoop(batch_size=BATCH_SIZE, seed=120825, shuffle_level=2, allow_rollback=False, verbose=4) task.set_checkpoint(path=MODEL_PATH, obj=x_vec, increasing=True, max_checkpoint=-1) task.set_callbacks([ training.NaNDetector(task_name='train', patience=-1), training.CheckpointEpoch(task_name='train', epoch_percent=0.5), # training.EarlyStopGeneralizationLoss('valid', ce, # threshold=5, patience=3) ]) task.set_train_task(func=f_train, data=train, epoch=EPOCH, name='train') task.set_valid_task(func=f_score, data=valid, freq=training.Timer(percentage=0.5), name='valid') task.run()
training=True) print('Building testing functions ...') f_test = K.function(inputs=inputs, outputs=[ce, acc, cm], training=False) print('Building predicting functions ...') f_pred = K.function(inputs=inputs, outputs=y_prob, training=False) print("Building other functions ...") f_e = K.function(inputs=inputs, outputs=E, training=False) f_z = K.function(inputs=inputs, outputs=Z_infer, training=False) f_d = K.function(inputs=inputs, outputs=D_infer, training=False) # =========================================================================== # Training # =========================================================================== print('Start training ...') task = training.MainLoop(batch_size=BATCH_SIZE, seed=120825, shuffle_level=2, allow_rollback=True, labels=digits) task.set_checkpoint(MODEL_PATH, [f_encoder, f_decoder]) task.set_callbacks([ training.NaNDetector(), training.EarlyStopGeneralizationLoss('valid', ce, threshold=5, patience=5) ]) task.set_train_task(f_train, train, epoch=25, name='train') task.set_valid_task(f_test, valid, freq=training.Timer(percentage=0.8), name='valid') task.run()
N.Dense(num_units=nb_labels, activation=K.softmax) ], debug=True) y_pred = f(X) params = [p for p in f.parameters if not has_roles(p, EmbeddingWeight)] print('Params:', [p.name for p in params]) cost_train = K.mean(K.categorical_crossentropy(y_pred, y)) cost_score = K.mean(K.categorical_accuracy(y_pred, y)) opt = K.optimizers.RMSProp() updates = opt.get_updates(cost_train, params) print('Build training function ...') f_train = K.function([X, y], cost_train, updates) print('Build scoring function ...') f_score = K.function([X, y], cost_score) trainer = training.MainLoop(batch_size=128, seed=1234, shuffle_level=2) trainer.set_task(f_train, (X_train, y_train), epoch=args['epoch'], name='train') trainer.set_subtask(f_score, (X_valid, y_valid), freq=1., name='valid') trainer.set_callback([ training.ProgressMonitor('train', format='Train:{:.4f}'), training.ProgressMonitor('valid', format='Test:{:.4f}'), training.History() ]) trainer.run()
# =========================================================================== updates = optimizer.get_updates(cost_train, parameters) # ====== create function ====== # print('Building training functions ...') f_train = K.function([X, y], [cost_train, optimizer.norm], updates=updates) print('Building testing functions ...') f_test = K.function([X, y], [cost_test_1, cost_test_2, cost_test_3]) print('Building predicting functions ...') f_pred = K.function(X, y_score) # =========================================================================== # Build trainer # =========================================================================== print('Start training ...') task = training.MainLoop(batch_size=args['bs'], seed=1208, shuffle_level=2) task.set_save(get_modelpath(name='digit_audio.ai', override=True), f) task.set_task(f_train, train_feeder, epoch=args['epoch'], name='train') task.set_subtask(f_test, valid_feeder, freq=0.6, name='valid') task.set_subtask(f_test, test_feeder, when=-1, name='test') task.set_callback([ training.ProgressMonitor(name='train', format='Results: {:.4f}-{:.4f}'), training.ProgressMonitor(name='valid', format='Results: {:.4f}-{:.4f}', tracking={2: lambda x: sum(x)}), training.ProgressMonitor(name='test', format='Results: {:.4f}-{:.4f}'), training.History(), training.EarlyStopGeneralizationLoss('valid', threshold=5, patience=3), training.NaNDetector(('train', 'valid'), patience=3, rollback=True) ]) task.run()
# =========================================================================== print('Start training ...') # ====== some configurations ====== # model_save_path = '/tmp/EXP_MNIST' if os.path.exists(model_save_path): shutil.rmtree(model_save_path) os.mkdir(model_save_path) print("Save path:", model_save_path) N_EPOCH = 120 BATCH_SIZE = 512 # ====== run the training ====== # task = training.MainLoop(batch_size=BATCH_SIZE, seed=12, shuffle_level=2, allow_rollback=True, verbose=1) task.set_checkpoint(os.path.join(model_save_path, 'checkpoint'), ops, max_checkpoint=-1) task.set_callbacks([ training.NaNDetector(), training.CheckpointEpoch('train', epoch_percent=1.), training.EarlyStopGeneralizationLoss('valid', loss, threshold=5, patience=3), training.LambdaCallback(fn=lambda t: print(str(t)), task_name='train', signal=training.TaskSignal.EpochEnd),