def train_model(args):
model = model_fn()
train_loader, validate_loader = mnist_data_loaders(args)
# Build trainer
trainer = Trainer(model) \
.build_criterion('RegularizedCrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Record regularization losses
trainer.logger.observe_training_and_validation_states([
'main_loss', 'total_regularization_loss', 'activity_regularization',
'l1_weight_regularization'
])
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def setUp(self):
# Build model
net = self._make_test_model()
# Build trainer
self.trainer = Trainer(net)\
.build_logger(TensorboardLogger(send_image_at_batch_indices=0,
send_image_at_channel_indices='all',
log_images_every=(20, 'iterations')),
log_directory=os.path.join(self.ROOT_DIR, 'logs'))\
.build_criterion('CrossEntropyLoss')\
.build_metric('CategoricalError')\
.build_optimizer('Adam')\
.validate_every((1, 'epochs'))\
.save_every((2, 'epochs'), to_directory=os.path.join(self.ROOT_DIR, 'saves'))\
.save_at_best_validation_score()\
.set_max_num_epochs(2)\
.cuda().set_precision(self.PRECISION)
# Load CIFAR10 data
train_loader, test_loader = \
get_cifar10_loaders(root_directory=os.path.join(self.ROOT_DIR, 'data'),
download=self.DOWNLOAD_CIFAR)
# Bind loaders
self.trainer.bind_loader('train', train_loader).bind_loader('validate', test_loader)
def set_up_training(project_directory, config, data_config,
load_pretrained_model):
# Get model
if load_pretrained_model:
model = Trainer().load(from_directory=project_directory,
filename='Weights/checkpoint.pytorch').model
else:
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
criterion = SorensenDiceLoss()
loss_train = LossWrapper(criterion=criterion,
transforms=Compose(ApplyAndRemoveMask(),
InvertTarget()))
loss_val = LossWrapper(criterion=criterion,
transforms=Compose(RemoveSegmentationFromTarget(),
ApplyAndRemoveMask(),
InvertTarget()))
# Build trainer and validation metric
logger.info("Building trainer.")
smoothness = 0.95
offsets = data_config['volume_config']['segmentation']['affinity_config'][
'offsets']
metric = ArandErrorFromMulticut(average_slices=False,
use_2d_ws=True,
n_threads=8,
weight_edges=True,
offsets=offsets)
trainer = Trainer(model)\
.save_every((1000, 'iterations'),
to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss_train)\
.build_validation_criterion(loss_val)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=1)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.98,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))\
.register_callback(GarbageCollection())
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations'),
log_histograms_every='never').observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def train_model(model, train_dataset, valid_dataset, args):
kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
**kw)
validate_loader = DataLoader(dataset=valid_dataset,
batch_size=args.batch_size,
**kw)
trainer = Trainer(model) \
.build_criterion(CrossEntropyLoss3D) \
.build_metric(CategoricalError3D) \
.build_optimizer('Adam', weight_decay=1e-6) \
.save_every((1, 'epochs')) \
.validate_every((100, 'iteration')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(
TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'),
log_directory='tb_log/'
) \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train_model(args):
model = MNISTCNNModel()
train_loader, validate_loader = mnist_data_loaders(args)
# Build trainer
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((2, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def set_up_training(project_directory, config, data_config, criterion, balance,
load_pretrained_model):
# Get model
if load_pretrained_model:
model = Trainer().load(from_directory=project_directory,
filename='Weights/checkpoint.pytorch').model
else:
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
# TODO
logger.info("Using criterion: %s" % criterion)
# TODO this should go somewhere more prominent
affinity_offsets = data_config['volume_config']['segmentation'][
'affinity_offsets']
# TODO implement affinities on gpu again ?!
criterion = CRITERIA[criterion]
loss = LossWrapper(
criterion=criterion(),
transforms=Compose(MaskTransitionToIgnoreLabel(affinity_offsets),
RemoveSegmentationFromTarget(), InvertTarget()),
weight_function=BalanceAffinities(
ignore_label=0, offsets=affinity_offsets) if balance else None)
# Build trainer and validation metric
logger.info("Building trainer.")
smoothness = 0.95
# use multicut pipeline for validation
metric = ArandErrorFromSegmentationPipeline(
local_affinity_multicut_from_wsdt2d(n_threads=10, time_limit=120))
trainer = Trainer(model)\
.save_every((1000, 'iterations'), to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=1)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.98,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations')).observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def load_model(args):
trainer = Trainer()
trainer.load(from_directory=args.load_directory, best=False)
# trainer.load(from_directory=args.load_directory, best=False)
trainer.set_max_num_epochs(args.epochs + trainer.epoch_count)
model = trainer.model
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
trainer.save_to_directory(args.save_directory)
return (model, trainer)
def set_up_training(project_directory, config, data_config,
load_pretrained_model):
# Get model
if load_pretrained_model:
model = Trainer().load(from_directory=project_directory,
filename='Weights/checkpoint.pytorch').model
else:
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
affinity_offsets = data_config['volume_config']['segmentation'][
'affinity_offsets']
loss = LossWrapper(criterion=SorensenDiceLoss(),
transforms=Compose(
MaskTransitionToIgnoreLabel(affinity_offsets),
RemoveSegmentationFromTarget(), InvertTarget()))
# Build trainer and validation metric
logger.info("Building trainer.")
smoothness = 0.95
# use multicut pipeline for validation
# metric = ArandErrorFromSegmentationPipeline(local_affinity_multicut_from_wsdt2d(n_threads=10,
# time_limit=120))
# use damws for validation
stride = [2, 10, 10]
metric = ArandErrorFromSegmentationPipeline(
DamWatershed(affinity_offsets, stride, randomize_bounds=False))
trainer = Trainer(model)\
.save_every((1000, 'iterations'), to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=1)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.98,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations')).observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def set_up_training(project_directory, config, data_config):
# Get model
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
criterion = SorensenDiceLoss()
loss_train = LossWrapper(criterion=criterion, transforms=InvertTarget())
loss_val = LossWrapper(criterion=criterion,
transforms=Compose(RemoveSegmentationFromTarget(),
InvertTarget()))
# Build trainer and validation metric
logger.info("Building trainer.")
smoothness = 0.75
offsets = data_config['volume_config']['segmentation']['affinity_config'][
'offsets']
strides = [1, 10, 10]
metric = ArandErrorFromMWS(average_slices=False,
offsets=offsets,
strides=strides,
randomize_strides=False)
trainer = Trainer(model)\
.save_every((1000, 'iterations'),
to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss_train)\
.build_validation_criterion(loss_val)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=1)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness,
verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.99,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))\
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations'),
log_histograms_every='never').observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def main(args):
img_size = 224
train_trans = transforms.Compose([
transforms.RandomResizedCrop(img_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
test_trans = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(img_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
train_ds = IBMDataset(args.data_dir, transform=train_trans, name='train')
train_loader = data.DataLoader(train_ds,
batch_size = args.batch_size,
shuffle = True)
val_ds = IBMDataset(args.data_dir, transform=test_trans, name='val')
print(len(val_ds), len(train_ds))
val_loader = data.DataLoader(val_ds,
batch_size = args.batch_size,
shuffle = True)
LOG_DIRECTORY = '../logs/'
SAVE_DIRECTORY = '../models/resnet50/'
model = ResNet50()
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(SAVE_DIRECTORY) \
.set_max_num_epochs(args.num_epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=LOG_DIRECTORY)
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', val_loader)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
def set_up_training(project_directory, config, data_config,
load_pretrained_model, max_iters):
# Get model
if load_pretrained_model:
model = Trainer().load(from_directory=project_directory,
filename='Weights/checkpoint.pytorch').model
else:
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
loss = LossWrapper(criterion=SorensenDiceLoss(),
transforms=Compose(MaskIgnoreLabel(),
RemoveSegmentationFromTarget()))
# TODO loss transforms:
# - Invert Target ???
# Build trainer and validation metric
logger.info("Building trainer.")
# smoothness = 0.95
# TODO set up validation ?!
trainer = Trainer(model)\
.save_every((1000, 'iterations'), to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.register_callback(ManualLR(decay_specs=[((k * 100, 'iterations'), 0.99)
for k in range(1, max_iters // 100)]))
# .validate_every((100, 'iterations'), for_num_iterations=1)\
# .register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
# .build_metric(metric)\
# .register_callback(AutoLR(factor=0.98,
# patience='100 iterations',
# monitor_while='validating',
# monitor_momentum=smoothness,
# consider_improvement_with_respect_to='previous'))
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations')) # .observe_states(
# ['validation_input', 'validation_prediction, validation_target'],
# observe_while='validating'
# )
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def inferno_build_tensorboard(self):
if self.get('trainer/tensorboard') is not None:
if TensorboardLogger is None:
print("warning can not use TensorboardLogger")
return
tb_args = self.get('trainer/tensorboard')
tb_args['log_directory'] = f"{self.experiment_directory}/Logs"
print("logging to ", tb_args['log_directory'])
tb_logger = TensorboardLogger(**tb_args)
# register Tensorboard logger
self._trainer.build_logger(tb_logger)
# and set _logger to so it can be used by the Tensorboardmixin
self._logger = tb_logger
def set_up_training(project_directory, config, data_config,
load_pretrained_model):
# Get model
if load_pretrained_model:
model = Trainer().load(from_directory=project_directory,
filename='Weights/checkpoint.pytorch').model
else:
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
loss = dice_loss()
loss_val = dice_loss(is_val=True)
metric = mws_metric()
# metric = loss_val
# Build trainer and validation metric
logger.info("Building trainer.")
smoothness = 0.9
trainer = Trainer(model)\
.save_every((1000, 'iterations'),
to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss)\
.build_validation_criterion(loss_val)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=5)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.98,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))\
.register_callback(GarbageCollection())\
logger.info("Building logger.")
# Build logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations'),
log_histograms_every='never').observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
def my_train(load, folder):
click.echo('starting training')
os.makedirs(folder, exist_ok=True)
# joint_transform = Compose(
# RandomRotate(),
# RandomTranspose(),
# RandomFlip()
# )
# setup logger
os.makedirs('derived_data/log', exist_ok=True)
Logger.instance().setup('derived_data/log')
vae = Vae()
ds = HemoDataset(root_folder=root_folder, image_transform=None, training=True)
train_loader = torch.utils.data.DataLoader(ds, batch_size=1536, num_workers=8)
# Build trainer
trainer = Trainer(vae)
trainer.save_to_directory(folder)
if load:
trainer.load()
# trainer.cuda(devices=[0, 1])
trainer.cuda()
trainer.build_criterion(vae.loss_function())
trainer.build_optimizer('Adam', lr=0.001)
# trainer.validate_every((2, 'epochs'))
trainer.save_every((1, 'epochs'))
trainer.set_max_num_epochs(100)
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never',
# log_images_every=(1, 'iteration'),
log_directory=folder))
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=1, num_targets=1)
# bind callbacks
trainer.register_callback(GarbageCollection(), trigger='end_of_training_iteration')
trainer.register_callback(ShowMinimalConsoleInfo(), trigger='end_of_training_iteration')
# trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.fit()
pushover_notification.send('embeddings generated')
def run(args):
model = SpeechModel(args)
trainer = Trainer()
if os.path.exists(
os.path.join(args.save_directory, trainer._checkpoint_filename)):
trainer.load(from_directory=args.save_directory)
model.load_state_dict(trainer.model.state_dict())
if args.cuda:
model = model.cuda()
else:
train_loader = make_loader('train', args, batch_size=args.batch_size)
dev_loader = make_loader('dev', args, batch_size=args.batch_size)
# Build trainer
trainer = Trainer(model) \
.build_criterion(CTCCriterion, size_average=True) \
.build_optimizer('Adam', weight_decay=1e-7, lr=5e-5) \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.bind_loader('validate',
dev_loader,
num_inputs=3,
num_targets=1)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
test_loader = make_loader('test',
args=args,
shuffle=False,
batch_size=1,
test=True)
run_logits(loader=test_loader, model=model, args=args)
def train_model(args):
"""
Perform training then call prediction
"""
model = all_cnn_module()
model.apply(initializer)
train_loader, validate_loader, test_loader = make_loaders(args)
# Build trainer
savepath = os.path.join(args.save_directory,
Trainer()._checkpoint_filename)
if os.path.exists(savepath):
trainer = Trainer().load(from_directory=args.save_directory)
if args.cuda:
trainer.cuda()
else:
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.save_every((1, 'epochs')) \
.validate_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# These are the params from the paper
trainer.build_optimizer('SGD',
lr=0.01,
momentum=0.9,
weight_decay=0.001)
# Also works with Adam and default settings
# trainer.build_optimizer('Adam')
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
write_predictions(args, trainer.model, test_loader)
def main():
# Load dataset
train_loader = get_cremi_loaders(
config=join(PROJECT_DIRECTORY, 'Configurations', CONFIG_FILENAME))
# Build model
dense_unet = DUNet(1, 1)
# Build trainer
trainer = Trainer(model=dense_unet) \
.build_optimizer('Adam') \
.build_criterion('SorensenDiceLoss') \
.build_logger(TensorboardLogger(send_image_at_batch_indices=0,
send_image_at_channel_indices='all',
log_images_every=(20, 'iterations')),
log_directory=join(PROJECT_DIRECTORY, 'Logs')) \
.save_every((1000, 'iterations'), to_directory=join(PROJECT_DIRECTORY, 'Weights')) \
.set_max_num_iterations(1000000) \
.cuda()
# Bind loader to trainer
trainer.bind_loader('train', train_loader)
# Go!
trainer.fit()
def get_trainer(self, input_channels):
# Build model
net = self._make_test_model(input_channels)
# Build trainer
trainer = Trainer(net)\
.build_logger(TensorboardLogger(send_image_at_batch_indices=0,
send_image_at_channel_indices='all',
log_images_every=(20, 'iterations')),
log_directory=self.LOG_DIRECTORY)\
.build_criterion('CrossEntropyLoss')\
.build_metric('CategoricalError')\
.build_optimizer('Adam')\
.validate_every((1, 'epochs'))\
.save_every((2, 'epochs'), to_directory=self.SAVE_DIRECTORY)\
.save_at_best_validation_score()\
.set_max_num_epochs(2)\
.set_precision(self.PRECISION)
# Bind loaders
train_loader, test_loader = self.get_random_dataloaders(
input_channels=input_channels)
trainer.bind_loader('train',
train_loader).bind_loader('validate', test_loader)
return trainer
def run(args):
dataset = RorschachWrapper()
save_args(args) # save command line to a file for reference
train_loader = rorschach_cgan_data_loader(args,
dataset=dataset) # get the data
# todo
model = patchCWGANModel(args,
discriminator=patchCDiscriminatorNetwork(args),
generator=CGeneratorNetwork(args))
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(
CWGANDiscriminatorLoss(penalty_weight=args.penalty_weight,
model=model))
trainer.build_optimizer('Adam',
model.discriminator.parameters(),
lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(CGenerateDataCallback(args, dataset=dataset))
trainer.register_callback(
CGeneratorTrainingCallback(args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss(),
dataset=dataset))
trainer.bind_loader('train', train_loader, num_inputs=2)
# Custom logging configuration so it knows to log our images
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(args.log_image_frequency,
'iteration'))
trainer.build_logger(logger, log_directory=args.save_directory)
logger.observe_state('generated_images')
logger.observe_state('real_images')
logger._trainer_states_being_observed_while_training.remove(
'training_inputs')
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def main():
# Load dataset
train_loader = get_cremi_loaders(config=join(PROJECT_DIRECTORY, "Configurations", CONFIG_FILENAME))
# Build model
dense_unet = DUNet(1, 1)
# Build trainer
trainer = (
Trainer(model=dense_unet)
.build_optimizer("Adam")
.build_criterion("SorensenDiceLoss")
.build_logger(
TensorboardLogger(
send_image_at_batch_indices=0, send_image_at_channel_indices="all", log_images_every=(20, "iterations")
),
log_directory=join(PROJECT_DIRECTORY, "Logs"),
)
.save_every((1000, "iterations"), to_directory=join(PROJECT_DIRECTORY, "Weights"))
.set_max_num_iterations(1000000)
.cuda()
)
# Bind loader to trainer
trainer.bind_loader("train", train_loader)
# Go!
trainer.fit()
def run(args):
save_args(args) # save command line to a file for reference
train_loader = mnist_data_loader(args) # get the data
model = GANModel(args,
discriminator=DiscriminatorNetwork(args),
generator=GeneratorNetwork(args))
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(
WGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model))
trainer.build_optimizer('Adam',
model.discriminator.parameters(),
lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(GenerateDataCallback(args))
trainer.register_callback(
GeneratorTrainingCallback(args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss()))
trainer.bind_loader('train', train_loader)
# Custom logging configuration so it knows to log our images
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(args.log_image_frequency,
'iteration'))
trainer.build_logger(logger, log_directory=args.save_directory)
logger.observe_state('generated_images')
logger.observe_state('real_images')
logger._trainer_states_being_observed_while_training.remove(
'training_inputs')
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
# Generate video from saved images
if not args.no_ffmpeg:
generate_video(args.save_directory)
def set_up_training(project_directory, config):
# Load the model to train from the configuratuib file ('./config/train_config.yml')
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
# Initialize the loss: we use the SorensenDiceLoss, which has the nice property
# of being fairly robust for un-balanced targets
criterion = SorensenDiceLoss()
# Wrap the loss to apply additional transformations before the actual
# loss is applied. Here, we apply the mask to the target
# and invert the target (necessary for sorensen dice) during training.
# In addition, we need to remove the segmentation from the target
# during validation (we only keep the segmentation in the target during validation)
loss_train = LossWrapper(criterion=criterion,
transforms=Compose(ApplyAndRemoveMask(),
InvertTarget()))
loss_val = LossWrapper(criterion=criterion,
transforms=Compose(RemoveSegmentationFromTarget(),
ApplyAndRemoveMask(),
InvertTarget()))
# Build the validation metric: we validate by running connected components on
# the affinities for several thresholds
# metric = ArandErrorFromConnectedComponentsOnAffinities(thresholds=[.5, .6, .7, .8, .9],
# invert_affinities=True)
metric = ArandErrorFromConnectedComponents(thresholds=[.5, .6, .7, .8, .9],
invert_input=True,
average_input=True)
logger.info("Building trainer.")
smoothness = 0.95
# Build the trainer object
trainer = Trainer(model)\
.save_every((1000, 'iterations'), to_directory=os.path.join(project_directory, 'Weights'))\
.build_criterion(loss_train)\
.build_validation_criterion(loss_val)\
.build_optimizer(**config.get('training_optimizer_kwargs'))\
.evaluate_metric_every('never')\
.validate_every((100, 'iterations'), for_num_iterations=1)\
.register_callback(SaveAtBestValidationScore(smoothness=smoothness, verbose=True))\
.build_metric(metric)\
.register_callback(AutoLR(factor=0.98,
patience='100 iterations',
monitor_while='validating',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous'))
# .register_callback(DumpHDF5Every(frequency='99 iterations',
# to_directory=os.path.join(project_directory, 'debug')))
logger.info("Building logger.")
# Build tensorboard logger
tensorboard = TensorboardLogger(
log_scalars_every=(1, 'iteration'),
log_images_every=(100, 'iterations')).observe_states(
['validation_input', 'validation_prediction, validation_target'],
observe_while='validating')
trainer.build_logger(tensorboard,
log_directory=os.path.join(project_directory, 'Logs'))
return trainer
curent_model_dir = os.path.join(model_dir, args.id)
log_info('Model will be saved to %s' % (curent_model_dir))
log_info(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
weight_dir = os.path.join(curent_model_dir, 'weights')
log_info('Weights will be saved to %s' % (weight_dir))
if not os.path.exists(weight_dir):
os.mkdir(weight_dir)
logs_dir = os.path.join(curent_model_dir, 'logs')
if not os.path.exists(logs_dir):
os.mkdir(logs_dir)
log_info('Logs will be saved to %s' % (logs_dir))
# Build trainer
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(np.inf, 'epochs'))
def log_histogram(self, tag, values, bins=1000):
pass
logger.log_histogram = log_histogram
trainer = Trainer(model)\
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((2, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(weight_dir) \
.set_max_num_epochs(10000) \
.build_logger(logger, log_directory=logs_dir) \
##################################################
# data loaders
train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
download=DOWNLOAD_CIFAR)
##################################################
# Build trainer
trainer = Trainer(model)
trainer.build_criterion('CrossEntropyLoss')
trainer.build_metric('CategoricalError')
trainer.build_optimizer('Adam')
trainer.validate_every((2, 'epochs'))
trainer.save_every((5, 'epochs'))
trainer.save_to_directory(SAVE_DIRECTORY)
trainer.set_max_num_epochs(10)
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=LOG_DIRECTORY)
##################################################
# Bind loaders
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', validate_loader)
##################################################
# activate cuda
if USE_CUDA:
trainer.cuda()
##################################################
# fit
trainer.fit()
shuffle=True, num_workers=2)
net = torch.nn.Sequential(
ConvReLU2D(in_channels=1, out_channels=3, kernel_size=3),
UNet(in_channels=3, out_channels=N_DIRECTIONS, dim=2, final_activation='ReLU')
)
trainer = Trainer(net)
trainer.bind_loader('train', trainloader)
trainer.bind_loader('validate', valloader)
trainer.save_to_directory('./checkpoints')
trainer.save_every((200, 'iterations'))
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'), log_directory=LOG_DIRECTORY)
trainer.validate_every((200, 'iterations'), for_num_iterations=50)
#trainer.build_metric()
trainer.build_criterion(nn.L1Loss)
trainer.build_optimizer(optim.Adam, lr=1e-4, weight_decay=0.0005)
trainer.set_max_num_iterations(20000)
if torch.cuda.is_available():
trainer.cuda()
def train_model(args):
model = fm.model_fn(args)
print('Done initializing model')
# labels_all_dict = np.load('labels_all_dict.npy')
labels_all_dict = np.load('labels_all_dict_10s_subset_500.npy')
if args.no_val:
file_names_train = np.load(
'file_names_train_notest_hop_subset_500.npy')
# file_names_train = np.load('file_names_train_notest.npy')
train_dataloader = fd.create_dataloader(file_names_train,
args.data_directory,
labels_all_dict.item(), args)
else:
file_names_train = np.load('file_names_train.npy')
file_names_test = np.load('file_names_test.npy')
train_dataloader = fd.create_dataloader(file_names_train,
args.data_directory,
labels_all_dict.item(), args)
dev_dataloader = fd.create_dataloader(file_names_test,
args.data_directory,
labels_all_dict.item(), args)
# train_dataloader = fd.create_dataloader(train_feats, train_labels, args)
# dev_dataloader = fd.create_dataloader(dev_feats, dev_labels, args)
test_dataloader_flag = False
if test_dataloader_flag:
it = 0
for x, y, z in train_dataloader:
print('x.shape: {}'.format(x.shape))
print('y.shape: {}'.format(y.shape))
print('z.shape: {}'.format(z.shape))
print('x: {}'.format(x))
print('y: {}'.format(y))
print('z: {}'.format(z))
it = it + 1
print(it)
print('Done Creating Data Loaders')
if args.load_model:
(model, trainer) = load_model(args)
trainer \
.bind_loader('train', train_dataloader, num_inputs=2, num_targets=1)
if not args.no_val:
trainer.bind_loader('validate',
dev_dataloader,
num_inputs=2,
num_targets=1)
else:
# Build trainer
loss_fn = cust_CrossEntropyLoss()
trainer = Trainer(model) \
.build_criterion(loss_fn) \
.build_metric('CategoricalError') \
.build_optimizer(torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
if not args.no_val:
trainer \
.save_at_best_validation_score(True) \
.validate_every((1, 'epochs'))
trainer \
.bind_loader('train', train_dataloader, num_inputs=2, num_targets=1)
if not args.no_val:
trainer.bind_loader('validate',
dev_dataloader,
num_inputs=2,
num_targets=1)
# .build_optimizer('Adam') \
# .save_at_best_validation_score(True) \
# .build_metric(LevenshteinMetric) \
# .evaluate_metric_every('300 iterations') \
if args.cuda:
print('Switch trainer to CUDA')
trainer.cuda()
trainer.fit()
print('Done Fitting Trainer')
return model
model = nn.Sequential(
ConvELU2D(in_channels=3, out_channels=256, kernel_size=3),
nn.MaxPool2d(kernel_size=2, stride=2),
ConvELU2D(in_channels=256, out_channels=256, kernel_size=3),
nn.MaxPool2d(kernel_size=2, stride=2),
ConvELU2D(in_channels=256, out_channels=256, kernel_size=3),
nn.MaxPool2d(kernel_size=2, stride=2), Flatten(),
nn.Linear(in_features=(256 * 4 * 4), out_features=10), nn.Softmax())
##################################################
# data loaders
from inferno.io.box.cifar import get_cifar10_loaders
train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
download=DOWNLOAD_CIFAR)
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never')
##################################################
# Build trainer
from inferno.trainers.basic import Trainer
trainer = Trainer(model)
trainer.build_criterion('CrossEntropyLoss')
trainer.build_metric('CategoricalError')
trainer.build_optimizer('Adam')
trainer.validate_every((2, 'epochs'))
trainer.save_every((5, 'epochs'))
trainer.save_to_directory(SAVE_DIRECTORY)
trainer.set_max_num_epochs(10)
trainer.build_logger(logger, log_directory=LOG_DIRECTORY)
if not os.path.exists(logs_dir):
os.mkdir(logs_dir)
log_info('Logs will be saved to %s' % (logs_dir))
# Build trainer
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=args.lr, betas=(0.9, 0.999), eps=1e-08) \
.validate_every((1, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(model_dir) \
.set_max_num_epochs(10000) \
.register_callback(GradChecker()) \
.register_callback(AutoLR(0.96, (1, 'epochs'), monitor_momentum=0.9,
monitor_while='validating',
consider_improvement_with_respect_to='best'))\
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(1, 'epoch')),
log_directory=logs_dir)
# Bind loaders
trainer \
.bind_loader('train', train_dl) \
.bind_loader('validate', test_dl)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
log_info('Model will be saved to %s' % (curent_model_dir))
log_info(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
weight_dir = os.path.join(curent_model_dir, 'weights')
log_info('Weights will be saved to %s' % (weight_dir))
if not os.path.exists(weight_dir):
os.mkdir(weight_dir)
logs_dir = os.path.join(curent_model_dir, 'logs')
if not os.path.exists(logs_dir):
os.mkdir(logs_dir)
log_info('Logs will be saved to %s' % (logs_dir))
# Build trainer
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=None,
log_histograms_every=None)
def log_hist(self, tag, values=1, step=1, bins=1000):
"""Logs the histogram of a list/vector of values."""
pass
logger.log_histogram = log_hist
trainer = Trainer(model)\
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((400, 'iterations')) \
.save_every((400, 'iterations')) \
.save_to_directory(weight_dir) \
