def test_cifar(self):
from inferno.trainers.basic import Trainer
from inferno.io.box.cifar10 import get_cifar10_loaders
# Build cifar10 loaders
trainloader, testloader = get_cifar10_loaders(
root_directory=join(self.ROOT_DIR, 'data'),
download=self.DOWNLOAD_CIFAR)
# Make model
net = self._make_test_model()
tic = time.time()
# Make trainer
trainer = Trainer(model=net)\
.build_optimizer('Adam')\
.build_criterion('CrossEntropyLoss')\
.build_metric('CategoricalError')\
.validate_every((1, 'epochs'))\
.save_every((1, 'epochs'), to_directory=join(self.ROOT_DIR, 'saves'))\
.save_at_best_validation_score()\
.set_max_num_epochs(2)
# Bind trainer to datasets
trainer.bind_loader('train',
trainloader).bind_loader('validate', testloader)
# Check device and fit
if self.CUDA:
if self.HALF_PRECISION:
trainer.cuda().set_precision('half').fit()
else:
trainer.cuda().fit()
else:
trainer.fit()
toc = time.time()
print("[*] Elapsed time: {} seconds.".format(toc - tic))
def train(net, dataset, criterion, num_epochs, batch_size, learn_rate,
dir_name):
dir_name = os.path.join('net/', dir_name)
trainer = Trainer(net[0])
if (os.path.exists(os.path.join(dir_name, 'model.pytorch'))):
net_temp = trainer.load_model(dir_name).model
net[0].load_state_dict(net_temp.state_dict())
print("Loaded checkpoint directly")
else:
if (not os.path.exists(dir_name)):
os.makedirs(dir_name)
data_loader = torch.utils.data.DataLoader(dataset,
shuffle=True,
batch_size=batch_size)
net[0].train()
trainer \
.build_criterion(LossPrinter(criterion)) \
.bind_loader('train', data_loader) \
.build_optimizer('Adam', lr=learn_rate) \
.set_max_num_epochs(num_epochs)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
trainer.save_model(dir_name)
net[0].cpu()
net[0].eval()
def train_model(args):
model = MNISTCNNModel()
train_loader, validate_loader = mnist_data_loaders(args)
# Build trainer
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((2, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def test_multi_gpu(self):
import torch
if not torch.cuda.is_available():
return
from inferno.trainers.basic import Trainer
from inferno.io.box.cifar10 import get_cifar10_loaders
import os
# Make model
net = self._make_test_model()
# Make trainer
trainer = Trainer(model=net) \
.build_optimizer('Adam') \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs'), to_directory=os.path.join(self.ROOT_DIR, 'saves')) \
.save_at_best_validation_score() \
.set_max_num_epochs(2)\
.cuda(devices=[0, 1, 2, 3])
train_loader, validate_loader = get_cifar10_loaders(
root_directory=self.ROOT_DIR, download=True)
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', validate_loader)
trainer.fit()
def train_model(model, train_dataset, valid_dataset, args):
kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
**kw)
validate_loader = DataLoader(dataset=valid_dataset,
batch_size=args.batch_size,
**kw)
trainer = Trainer(model) \
.build_criterion(CrossEntropyLoss3D) \
.build_metric(CategoricalError3D) \
.build_optimizer('Adam', weight_decay=1e-6) \
.save_every((1, 'epochs')) \
.validate_every((100, 'iteration')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(
TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'),
log_directory='tb_log/'
) \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def test_serialization(self):
trainer = self.get_trainer(3)
# Serialize
trainer.save()
# Unserialize
trainer = Trainer().load(os.path.join(self.ROOT_DIR, 'saves'))
train_loader, test_loader = self.get_random_dataloaders(
input_channels=3)
trainer.bind_loader('train',
train_loader).bind_loader('validate', test_loader)
trainer.fit()
def task(env):
dataset_name = 'MNIST'
dataset_path = env.dataset(dataset_name)
batch_size = 128
transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
mnist_set = torchvision.datasets.MNIST(str(dataset_path), train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(mnist_set, batch_size=batch_size,
shuffle=True, num_workers=2)
mnist_test_set = torchvision.datasets.MNIST(str(dataset_path), train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(mnist_test_set, batch_size=512,
shuffle=True, num_workers=2)
model = Model(n_classes=10, in_channels=1, layers=4)
exp = TrainLog(env, dataset_name,model,log_time=True)
logging.info(' saving to %s', exp.save_directory)
logging.info(' logging to %s', exp.log_directory)
# Load loaders
# train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
# download=DOWNLOAD_CIFAR)
# Build trainer
iterations = 5000
trainer = Trainer(model) \
.build_criterion('NLLLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=0.001) \
.save_every((1, 'epochs')) \
.save_to_directory(str(exp.save_directory)) \
.validate_every((1, 'epochs'))\
.set_max_num_iterations(iterations) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=str(exp.log_directory))
# Bind loaders
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', test_loader)
trainer.cuda()
# Go!
logging.info('start training')
trainer.fit()
trainer.set_max_num_iterations(trainer.iteration_count + iterations)
trainer.build_optimizer('Adam',lr=0.0001)
logging.info('slower lr')
trainer.fit()
def task(env):
dataset_name = 'MNIST'
dataset_path = env.dataset(dataset_name)
dataset = MNIST(str(dataset_path), train=True, download=True, transform=mnist_to_tensor32)
testset = MNIST(str(dataset_path), train=False, download=True, transform=mnist_to_tensor32)
test_loader = DataLoader(testset, batch_size=512, shuffle=True, num_workers=2)
train_loader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=2)
triplets = dis.TripelDataset(dataset, dataset.train_labels, K=1)
train_loader_triple = DataLoader(triplets, batch_size=16,collate_fn=dis.stack_triplets)
model = Model(in_channels=1, n_classes=10, shortcut=True)
criterion = dis.TripletLoss()
exp = TrainLog(env, dataset_name,model.info,log_time=True)
logging.info(' saving to %s', exp.save_directory)
logging.info(' logging to %s', exp.log_directory)
# Load loaders
# train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
# download=DOWNLOAD_CIFAR)
iterations = 0
trainer = Trainer(model)
set_log_and_save(trainer, exp)
trainer.build_criterion(criterion) \
.build_optimizer('SGD', lr=0.001, weight_decay=0.0005) \
.set_max_num_iterations(iterations)\
.bind_loader('train', train_loader_triple)
trainer.cuda()
logging.info('start training')
trainer.fit()
print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda()))
model.use_shortcut = False
model.classify = True
print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda()))
trainer = Trainer(model)
set_log_and_save(trainer, exp)
trainer.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=0.001) \
.set_max_num_iterations(5000) \
.bind_loader('train', train_loader) \
.bind_loader('test', test_loader)
trainer.cuda()
logging.info('start training')
trainer.fit()
def train_model(args):
"""
Performs the training
"""
if os.path.exists(
os.path.join(args.save_directory,
Trainer()._checkpoint_filename)):
# Skip training if checkpoint exists
return
model = WsjModel(args)
kwargs = {
'num_workers': args.num_workers,
'pin_memory': True
} if args.cuda else {}
train_loader = DataLoader(WsjDataset('train', args),
shuffle=True,
batch_size=args.batch_size,
collate_fn=wsj_collate_fn,
**kwargs)
validate_loader = DataLoader(WsjDataset('dev', args),
shuffle=True,
batch_size=args.batch_size,
collate_fn=wsj_collate_fn,
**kwargs)
# Build trainer
trainer = Trainer(model) \
.build_criterion(ctc_loss()) \
.build_metric(ctc_loss()) \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
#.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
#log_images_every='never'),
#log_directory=args.save_directory)
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=3)
trainer.bind_loader('validate', validate_loader, num_inputs=3)
trainer.register_callback(EpochTimer())
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
def test_multi_io(self):
from torch.utils.data.dataset import Dataset
from torch.utils.data.dataloader import DataLoader
from inferno.trainers.basic import Trainer
import torch
class DummyDataset(Dataset):
def __len__(self):
return 42
def __getitem__(self, item):
# 2 inputs and 3 targets (say)
return torch.rand(3, 32, 32), \
torch.rand(3, 32, 32), \
torch.rand(1).uniform_(), \
torch.rand(1).uniform_(), \
torch.rand(1).uniform_()
class DummyNetwork(torch.nn.Module):
def __init__(self):
super(DummyNetwork, self).__init__()
self.conv = torch.nn.Conv2d(3, 1, 3, padding=1)
def forward(self, *inputs):
assert len(inputs) == 2
out = self.conv(inputs[0])
return out.view(inputs[0].size(0), -1).mean(1), \
out.view(inputs[0].size(0), -1).mean(1), \
out.view(inputs[0].size(0), -1).mean(1)
class DummyCriterion(torch.nn.Module):
def forward(self, predictions, targets):
assert len(predictions) == len(targets) == 3
return predictions[0].mean()
loader = DataLoader(DummyDataset())
net = DummyNetwork()
trainer = Trainer(net)\
.build_criterion(DummyCriterion)\
.build_optimizer('Adam')\
.set_max_num_iterations(50)\
.bind_loader('train', loader, num_inputs=2, num_targets=3)
trainer.fit()
def train_model(args):
"""
Perform training then call prediction
"""
model = all_cnn_module()
model.apply(initializer)
train_loader, validate_loader, test_loader = make_loaders(args)
# Build trainer
savepath = os.path.join(args.save_directory,
Trainer()._checkpoint_filename)
if os.path.exists(savepath):
trainer = Trainer().load(from_directory=args.save_directory)
if args.cuda:
trainer.cuda()
else:
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.save_every((1, 'epochs')) \
.validate_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# These are the params from the paper
trainer.build_optimizer('SGD',
lr=0.01,
momentum=0.9,
weight_decay=0.001)
# Also works with Adam and default settings
# trainer.build_optimizer('Adam')
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
write_predictions(args, trainer.model, test_loader)
def run(args):
save_args(args) # save command line to a file for reference
train_loader = mnist_data_loader(args) # get the data
model = GANModel(args,
discriminator=DiscriminatorNetwork(args),
generator=GeneratorNetwork(args))
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(
WGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model))
trainer.build_optimizer('Adam',
model.discriminator.parameters(),
lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(GenerateDataCallback(args))
trainer.register_callback(
GeneratorTrainingCallback(args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss()))
trainer.bind_loader('train', train_loader)
# Custom logging configuration so it knows to log our images
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(args.log_image_frequency,
'iteration'))
trainer.build_logger(logger, log_directory=args.save_directory)
logger.observe_state('generated_images')
logger.observe_state('real_images')
logger._trainer_states_being_observed_while_training.remove(
'training_inputs')
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
# Generate video from saved images
if not args.no_ffmpeg:
generate_video(args.save_directory)
def train_model(model, dataset, args):
kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=True,
**kw)
trainer = Trainer(model) \
.build_criterion(CrossEntropyLoss3D) \
.build_metric(CategoricalError3D) \
.build_optimizer('Adam', weight_decay=1e-6) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.register_callback(CustomLogger) \
.bind_loader('train', loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train(net, num_epochs, dataset):
data_loader = torch.utils.data.DataLoader(dataset,
shuffle=True,
batch_size=100)
net.train(mode=True)
for i in range(num_epochs):
lr = 0.002 / (1 + 0.2 * i)
trainer = Trainer(net) \
.build_criterion(SeqCrossEntropyLoss) \
.build_optimizer('Adam', lr=lr) \
.set_max_num_epochs(1) \
.save_every((10, 'iterations')) \
.save_to_directory('net/')
trainer.bind_loader('train', data_loader, num_inputs=3)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
max_num_iterations = 10000
trainer = Trainer(model) \
.build_criterion('NLLLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.save_every((1, 'epochs')) \
.save_to_directory(str(exp.save_directory))\
.validate_every((2, 'epochs'))\
.set_max_num_epochs(100) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iterations'),
log_images_every='never'),
log_directory=str(exp.log_directory))
# .save_every((2000, 'iterations'), to_directory=str(exp.save_directory), checkpoint_filename='latest') \
# Bind loaders
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', test_loader)
trainer.cuda()
# Go!
trainer.fit()
# TODO: train different max pool sizes
# TODO: train different kernel numbers
# TODO: train different conv layers
# TODO: train with different number of training images
# TODO: report run time
