def test_ComputeMetrics(): setka.base.environment_setup() ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=10, shuffle=False), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.ComputeMetrics([loss, acc, const], divide_first=[True, False, True], steps_to_compute=2, reduce=True) ]) trainer.run_train(2)
def test_Logger_tensor(): loss = tensor_loss acc = tensor_acc ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() def view_result(one_input, one_output): img = one_input[0] img = (img - img.min()) / (img.max() - img.min()) truth = one_input[1] label = one_output[0] fig = plt.figure() plt.imshow(img.permute(2, 1, 0)) plt.close() return {'figures': {'img.jpg': fig}} trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler( ds, batch_size=4, limits={ 'train': 3, 'valid': 3, 'test': 1 }), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.01, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.ComputeMetrics([loss, acc]), setka.pipes. Logger(f=view_result, full_snapshot_path=True, name='my_experiment') ]) trainer.run_train(2) trainer.run_epoch('test', 'test', n_iterations=2) assert (os.path.exists(os.path.join('runs', 'my_experiment'))) assert (len(os.listdir(os.path.join('runs', 'my_experiment'))) > 0) last_run = sorted(os.listdir(os.path.join('runs', 'my_experiment')))[-1] assert (os.path.exists( os.path.join('runs', 'my_experiment', last_run, '2_figures', 'test_2047', 'img.jpg'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_run, 'bash_command.txt'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_run, 'epoch_log.json'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_run, 'batch_log.json')))
def test_ComputeMetrics3(): setka.base.environment_setup() ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() cyclic_lr = lambda x: torch.optim.lr_scheduler.CyclicLR( x, base_lr=0.0, max_lr=0.0) reduce_lr = lambda x: torch.optim.lr_scheduler.ReduceLROnPlateau(x) trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=9, shuffle=False), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.ComputeMetrics([loss, acc, const], divide_first=True, steps_to_compute=2, reduce=[True, False, True]), setka.pipes.OneStepOptimizers([ setka.base.Optimizer( model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4, iter_schedulers=[ setka.base.Scheduler(torch.optim.lr_scheduler.CyclicLR, base_lr=1.0e-3, max_lr=1.0e-1) ], epoch_schedulers=[ setka.base.Scheduler( torch.optim.lr_scheduler.ReduceLROnPlateau, monitor=lambda: trainer._metrics['valid']['tensor_acc'] [0] / trainer._metrics['valid']['tensor_acc'][1], mode='max') ], # schedulers={ # 'batch': [cyclic_lr], # 'epoch': [(reduce_lr, 'valid', 'tensor_acc', 0) ) ]), ]) trainer.run_train(20)
def test_base(): setka.base.environment_setup() ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() # input, target, text, random_ones = ds['train', 0] trainer = setka.base.Trainer( pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=2, shuffle=True), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.Pipe() ], collection_op=setka.base.CollectionOperator(soft_collate_fn=True)) trainer.remove_pipe(setka.pipes.LossHandler) trainer.add_pipe(setka.pipes.LossHandler(loss)) trainer.run_train(n_epochs=2) trainer.run_epoch(mode='train', subset='train', n_iterations=100) trainer.run_epoch(mode='valid', subset='valid', n_iterations=10) trainer.run_epoch(mode='test', subset='valid', n_iterations=10) print("===== Training schedule =====") print(trainer.view_train()) print("===== Epoch schedule =====") print(trainer.view_epoch()) print("===== Batch schedule =====") print(trainer.view_batch()) print("===== Trianer =====") print(trainer.view_pipeline())
def test_WeightAveraging(): ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=5), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.WeightAveraging(epoch_start=0, interval=1) ]) trainer.run_train(3)
def test_Lambda(): ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() def print_message(): print("Message") trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=2), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.Lambda(on_batch_begin=print_message) ]) trainer.run_train(2)
def test_ProgressBar(): ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=2), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.ProgressBar() ]) trainer.status['Test'] = [ 'This is just a very-very long string that helps in testing the package. This string has no delimiters so it will be split in the middle of the string' ] trainer.status['Test2'] = [1.0e-12, 1.0e12, 1, 100000000000000000, 1000] trainer.run_train(1)
def test_SaveResult_tensor(): ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() def f(input, output): return input, output trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=2), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(tensor_loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.1, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.SaveResult(f=f) ]) trainer.run_train(1) trainer.run_epoch('test', 'test', n_iterations=2)
import torch import os import sys sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) import tiny_model import test_dataset import matplotlib.pyplot as plt from test_metrics import tensor_loss as loss from test_metrics import tensor_acc as acc ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() def view_result(one_input, one_output): # print("In view result") img = one_input[0] img = (img - img.min()) / (img.max() - img.min()) truth = one_input[1] label = one_output[0] # print(img.size()) fig = plt.figure() plt.imshow(img.permute(2, 1, 0)) plt.close() return {'figures': {'img': fig}}
def test_MakeCheckpoints(): ds = test_dataset.CIFAR10() model = tiny_model.TensorNet() trainer = setka.base.Trainer(pipes=[ setka.pipes.DatasetHandler(ds, batch_size=32, limits=2), setka.pipes.ModelHandler(model), setka.pipes.LossHandler(loss), setka.pipes.OneStepOptimizers([ setka.base.Optimizer(model, torch.optim.SGD, lr=0.01, momentum=0.9, weight_decay=5e-4) ]), setka.pipes.ComputeMetrics([loss, acc]), setka.pipes.Checkpointer('tensor_acc', max_mode=True, name='my_experiment', keep_best_only=False), setka.pipes.Checkpointer( 'tensor_loss', max_mode=False, name='my_experiment_best_only') ]) trainer.run_train(5) trainer.run_epoch('test', 'test', n_iterations=2) path = os.path.join('runs', 'my_experiment') last_exp = sorted(os.listdir(path))[-1] assert (len( os.listdir( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints'))) == 8 * 2) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_best.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_latest.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_weights_best.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_weights_latest.pth.tar'))) for index in range(6): assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_weights_' + str(index) + '.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment', last_exp, 'checkpoints', 'my_experiment_' + str(index) + '.pth.tar'))) # path = os.path.join('runs', 'my_experiment_best_only') # last_exp = sorted(os.listdir(path))[-1] assert (len( os.listdir( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints'))) == 4) assert (os.path.exists( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_best.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_latest.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_weights_best.pth.tar'))) assert (os.path.exists( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_weights_latest.pth.tar'))) latest_weights = torch.load( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_weights_latest.pth.tar')) latest_weights = latest_weights latest_trainer = torch.load( os.path.join('runs', 'my_experiment_best_only', last_exp, 'checkpoints', 'my_experiment_best_only_latest.pth.tar')) latest_trainer = latest_trainer['trainer'] assert (latest_trainer._model.state_dict().__str__() == trainer._model.state_dict().__str__()) assert (latest_weights.__str__() == trainer._model.state_dict().__str__())