def train(net, dataset, criterion, num_epochs, batch_size, learn_rate,
dir_name):
dir_name = os.path.join('net/', dir_name)
trainer = Trainer(net[0])
if (os.path.exists(os.path.join(dir_name, 'model.pytorch'))):
net_temp = trainer.load_model(dir_name).model
net[0].load_state_dict(net_temp.state_dict())
print("Loaded checkpoint directly")
else:
if (not os.path.exists(dir_name)):
os.makedirs(dir_name)
data_loader = torch.utils.data.DataLoader(dataset,
shuffle=True,
batch_size=batch_size)
net[0].train()
trainer \
.build_criterion(LossPrinter(criterion)) \
.bind_loader('train', data_loader) \
.build_optimizer('Adam', lr=learn_rate) \
.set_max_num_epochs(num_epochs)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
trainer.save_model(dir_name)
net[0].cpu()
net[0].eval()
def run(args):
dataset = AlderleyWrapper()
save_args(args) # save command line to a file for reference
train_loader = alderley_cgan_data_loader(args, dataset=dataset) # get the data
model = CGANModel(
args,
discriminators=[PixelDiscriminator(), patchCDiscriminatorNetwork(args)],
generator=UnetUpsample())
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(CWGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model))
trainer.build_optimizer('Adam', [parameter for discriminator in model.discriminators for parameter in discriminator.parameters()], lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(CGenerateDataCallback(args, dataset=dataset))
trainer.register_callback(CGeneratorTrainingCallback(
args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss(), dataset=dataset))
trainer.bind_loader('train', train_loader, num_inputs=2)
# Custom logging configuration so it knows to log our images
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def test_cifar(self):
from inferno.trainers.basic import Trainer
from inferno.io.box.cifar10 import get_cifar10_loaders
# Build cifar10 loaders
trainloader, testloader = get_cifar10_loaders(
root_directory=join(self.ROOT_DIR, 'data'),
download=self.DOWNLOAD_CIFAR)
# Make model
net = self._make_test_model()
tic = time.time()
# Make trainer
trainer = Trainer(model=net)\
.build_optimizer('Adam')\
.build_criterion('CrossEntropyLoss')\
.build_metric('CategoricalError')\
.validate_every((1, 'epochs'))\
.save_every((1, 'epochs'), to_directory=join(self.ROOT_DIR, 'saves'))\
.save_at_best_validation_score()\
.set_max_num_epochs(2)
# Bind trainer to datasets
trainer.bind_loader('train',
trainloader).bind_loader('validate', testloader)
# Check device and fit
if self.CUDA:
if self.HALF_PRECISION:
trainer.cuda().set_precision('half').fit()
else:
trainer.cuda().fit()
else:
trainer.fit()
toc = time.time()
print("[*] Elapsed time: {} seconds.".format(toc - tic))
def train_model(args):
model = model_fn()
train_loader, validate_loader = mnist_data_loaders(args)
# Build trainer
trainer = Trainer(model) \
.build_criterion('RegularizedCrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Record regularization losses
trainer.logger.observe_training_and_validation_states([
'main_loss', 'total_regularization_loss', 'activity_regularization',
'l1_weight_regularization'
])
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train_model(args):
model = MNISTCNNModel()
train_loader, validate_loader = mnist_data_loaders(args)
# Build trainer
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((2, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train_model(model, train_dataset, valid_dataset, args):
kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
**kw)
validate_loader = DataLoader(dataset=valid_dataset,
batch_size=args.batch_size,
**kw)
trainer = Trainer(model) \
.build_criterion(CrossEntropyLoss3D) \
.build_metric(CategoricalError3D) \
.build_optimizer('Adam', weight_decay=1e-6) \
.save_every((1, 'epochs')) \
.validate_every((100, 'iteration')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(
TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'),
log_directory='tb_log/'
) \
.bind_loader('train', train_loader) \
.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train_net_with_inferno(config_dict, net, criterion, optimizer, trainloader,
valloader):
"""
Trains the NeuralNet with inferno
:param config_dict: dict with configs
:param net: NeuralNet
:param criterion: criterion for NN
:param optimizer: optimizer for NN
:param trainloader: dataloader with traind ata
:param valloader: dataloader with validation data
"""
print("Start training with inferno!")
from inferno.trainers.basic import Trainer
from inferno.trainers.callbacks.essentials import SaveAtBestValidationScore
model_folder = os.path.join(config_dict["project_folder"], "model/")
if not os.path.exists(model_folder):
os.mkdir(model_folder)
trainer = Trainer(net) \
.save_every((1, 'epochs'), to_directory=model_folder) \
.build_criterion(criterion) \
.build_optimizer(optimizer) \
.build_metric(sorensen_dice_metric) \
.evaluate_metric_every('never') \
.validate_every((1, 'epochs'), for_num_iterations=50) \
.register_callback(SaveAtBestValidationScore(smoothness=.5))
trainer.set_max_num_epochs(config_dict['max_train_epochs'])
trainer.bind_loader('train',
trainloader).bind_loader('validate', valloader)
trainer.cuda()
trainer.fit()
def main(args):
img_size = 224
train_trans = transforms.Compose([
transforms.RandomResizedCrop(img_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
test_trans = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(img_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
train_ds = IBMDataset(args.data_dir, transform=train_trans, name='train')
train_loader = data.DataLoader(train_ds,
batch_size = args.batch_size,
shuffle = True)
val_ds = IBMDataset(args.data_dir, transform=test_trans, name='val')
print(len(val_ds), len(train_ds))
val_loader = data.DataLoader(val_ds,
batch_size = args.batch_size,
shuffle = True)
LOG_DIRECTORY = '../logs/'
SAVE_DIRECTORY = '../models/resnet50/'
model = ResNet50()
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(SAVE_DIRECTORY) \
.set_max_num_epochs(args.num_epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=LOG_DIRECTORY)
# Bind loaders
trainer \
.bind_loader('train', train_loader) \
.bind_loader('validate', val_loader)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
def task(env):
dataset_name = 'MNIST'
dataset_path = env.dataset(dataset_name)
batch_size = 128
transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
mnist_set = torchvision.datasets.MNIST(str(dataset_path), train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(mnist_set, batch_size=batch_size,
shuffle=True, num_workers=2)
mnist_test_set = torchvision.datasets.MNIST(str(dataset_path), train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(mnist_test_set, batch_size=512,
shuffle=True, num_workers=2)
model = Model(n_classes=10, in_channels=1, layers=4)
exp = TrainLog(env, dataset_name,model,log_time=True)
logging.info(' saving to %s', exp.save_directory)
logging.info(' logging to %s', exp.log_directory)
# Load loaders
# train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
# download=DOWNLOAD_CIFAR)
# Build trainer
iterations = 5000
trainer = Trainer(model) \
.build_criterion('NLLLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=0.001) \
.save_every((1, 'epochs')) \
.save_to_directory(str(exp.save_directory)) \
.validate_every((1, 'epochs'))\
.set_max_num_iterations(iterations) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=str(exp.log_directory))
# Bind loaders
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', test_loader)
trainer.cuda()
# Go!
logging.info('start training')
trainer.fit()
trainer.set_max_num_iterations(trainer.iteration_count + iterations)
trainer.build_optimizer('Adam',lr=0.0001)
logging.info('slower lr')
trainer.fit()
def task(env):
dataset_name = 'MNIST'
dataset_path = env.dataset(dataset_name)
dataset = MNIST(str(dataset_path), train=True, download=True, transform=mnist_to_tensor32)
testset = MNIST(str(dataset_path), train=False, download=True, transform=mnist_to_tensor32)
test_loader = DataLoader(testset, batch_size=512, shuffle=True, num_workers=2)
train_loader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=2)
triplets = dis.TripelDataset(dataset, dataset.train_labels, K=1)
train_loader_triple = DataLoader(triplets, batch_size=16,collate_fn=dis.stack_triplets)
model = Model(in_channels=1, n_classes=10, shortcut=True)
criterion = dis.TripletLoss()
exp = TrainLog(env, dataset_name,model.info,log_time=True)
logging.info(' saving to %s', exp.save_directory)
logging.info(' logging to %s', exp.log_directory)
# Load loaders
# train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY,
# download=DOWNLOAD_CIFAR)
iterations = 0
trainer = Trainer(model)
set_log_and_save(trainer, exp)
trainer.build_criterion(criterion) \
.build_optimizer('SGD', lr=0.001, weight_decay=0.0005) \
.set_max_num_iterations(iterations)\
.bind_loader('train', train_loader_triple)
trainer.cuda()
logging.info('start training')
trainer.fit()
print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda()))
model.use_shortcut = False
model.classify = True
print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda()))
trainer = Trainer(model)
set_log_and_save(trainer, exp)
trainer.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=0.001) \
.set_max_num_iterations(5000) \
.bind_loader('train', train_loader) \
.bind_loader('test', test_loader)
trainer.cuda()
logging.info('start training')
trainer.fit()
def train_model(args):
"""
Performs the training
"""
if os.path.exists(
os.path.join(args.save_directory,
Trainer()._checkpoint_filename)):
# Skip training if checkpoint exists
return
model = WsjModel(args)
kwargs = {
'num_workers': args.num_workers,
'pin_memory': True
} if args.cuda else {}
train_loader = DataLoader(WsjDataset('train', args),
shuffle=True,
batch_size=args.batch_size,
collate_fn=wsj_collate_fn,
**kwargs)
validate_loader = DataLoader(WsjDataset('dev', args),
shuffle=True,
batch_size=args.batch_size,
collate_fn=wsj_collate_fn,
**kwargs)
# Build trainer
trainer = Trainer(model) \
.build_criterion(ctc_loss()) \
.build_metric(ctc_loss()) \
.build_optimizer('Adam') \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
#.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
#log_images_every='never'),
#log_directory=args.save_directory)
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=3)
trainer.bind_loader('validate', validate_loader, num_inputs=3)
trainer.register_callback(EpochTimer())
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
def my_train(load, folder):
click.echo('starting training')
os.makedirs(folder, exist_ok=True)
# joint_transform = Compose(
# RandomRotate(),
# RandomTranspose(),
# RandomFlip()
# )
# setup logger
os.makedirs('derived_data/log', exist_ok=True)
Logger.instance().setup('derived_data/log')
vae = Vae()
ds = HemoDataset(root_folder=root_folder, image_transform=None, training=True)
train_loader = torch.utils.data.DataLoader(ds, batch_size=1536, num_workers=8)
# Build trainer
trainer = Trainer(vae)
trainer.save_to_directory(folder)
if load:
trainer.load()
# trainer.cuda(devices=[0, 1])
trainer.cuda()
trainer.build_criterion(vae.loss_function())
trainer.build_optimizer('Adam', lr=0.001)
# trainer.validate_every((2, 'epochs'))
trainer.save_every((1, 'epochs'))
trainer.set_max_num_epochs(100)
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never',
# log_images_every=(1, 'iteration'),
log_directory=folder))
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=1, num_targets=1)
# bind callbacks
trainer.register_callback(GarbageCollection(), trigger='end_of_training_iteration')
trainer.register_callback(ShowMinimalConsoleInfo(), trigger='end_of_training_iteration')
# trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.fit()
pushover_notification.send('embeddings generated')
def run(args):
dataset = RorschachWrapper()
save_args(args) # save command line to a file for reference
train_loader = rorschach_cgan_data_loader(args,
dataset=dataset) # get the data
# todo
model = patchCWGANModel(args,
discriminator=patchCDiscriminatorNetwork(args),
generator=CGeneratorNetwork(args))
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(
CWGANDiscriminatorLoss(penalty_weight=args.penalty_weight,
model=model))
trainer.build_optimizer('Adam',
model.discriminator.parameters(),
lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(CGenerateDataCallback(args, dataset=dataset))
trainer.register_callback(
CGeneratorTrainingCallback(args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss(),
dataset=dataset))
trainer.bind_loader('train', train_loader, num_inputs=2)
# Custom logging configuration so it knows to log our images
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(args.log_image_frequency,
'iteration'))
trainer.build_logger(logger, log_directory=args.save_directory)
logger.observe_state('generated_images')
logger.observe_state('real_images')
logger._trainer_states_being_observed_while_training.remove(
'training_inputs')
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def run(args):
model = SpeechModel(args)
trainer = Trainer()
if os.path.exists(
os.path.join(args.save_directory, trainer._checkpoint_filename)):
trainer.load(from_directory=args.save_directory)
model.load_state_dict(trainer.model.state_dict())
if args.cuda:
model = model.cuda()
else:
train_loader = make_loader('train', args, batch_size=args.batch_size)
dev_loader = make_loader('dev', args, batch_size=args.batch_size)
# Build trainer
trainer = Trainer(model) \
.build_criterion(CTCCriterion, size_average=True) \
.build_optimizer('Adam', weight_decay=1e-7, lr=5e-5) \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.bind_loader('validate',
dev_loader,
num_inputs=3,
num_targets=1)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
test_loader = make_loader('test',
args=args,
shuffle=False,
batch_size=1,
test=True)
run_logits(loader=test_loader, model=model, args=args)
def train_model(args):
"""
Perform training then call prediction
"""
model = all_cnn_module()
model.apply(initializer)
train_loader, validate_loader, test_loader = make_loaders(args)
# Build trainer
savepath = os.path.join(args.save_directory,
Trainer()._checkpoint_filename)
if os.path.exists(savepath):
trainer = Trainer().load(from_directory=args.save_directory)
if args.cuda:
trainer.cuda()
else:
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.save_every((1, 'epochs')) \
.validate_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# These are the params from the paper
trainer.build_optimizer('SGD',
lr=0.01,
momentum=0.9,
weight_decay=0.001)
# Also works with Adam and default settings
# trainer.build_optimizer('Adam')
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', validate_loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
write_predictions(args, trainer.model, test_loader)
def run(args):
save_args(args) # save command line to a file for reference
train_loader = mnist_data_loader(args) # get the data
model = GANModel(args,
discriminator=DiscriminatorNetwork(args),
generator=GeneratorNetwork(args))
# Build trainer
trainer = Trainer(model)
trainer.build_criterion(
WGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model))
trainer.build_optimizer('Adam',
model.discriminator.parameters(),
lr=args.discriminator_lr)
trainer.save_every((1, 'epochs'))
trainer.save_to_directory(args.save_directory)
trainer.set_max_num_epochs(args.epochs)
trainer.register_callback(GenerateDataCallback(args))
trainer.register_callback(
GeneratorTrainingCallback(args,
parameters=model.generator.parameters(),
criterion=WGANGeneratorLoss()))
trainer.bind_loader('train', train_loader)
# Custom logging configuration so it knows to log our images
logger = TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(args.log_image_frequency,
'iteration'))
trainer.build_logger(logger, log_directory=args.save_directory)
logger.observe_state('generated_images')
logger.observe_state('real_images')
logger._trainer_states_being_observed_while_training.remove(
'training_inputs')
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
# Generate video from saved images
if not args.no_ffmpeg:
generate_video(args.save_directory)
def train_model(model, dataset, args):
kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=True,
**kw)
trainer = Trainer(model) \
.build_criterion(CrossEntropyLoss3D) \
.build_metric(CategoricalError3D) \
.build_optimizer('Adam', weight_decay=1e-6) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.register_callback(CustomLogger) \
.bind_loader('train', loader)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
def train(net, num_epochs, dataset):
data_loader = torch.utils.data.DataLoader(dataset,
shuffle=True,
batch_size=100)
net.train(mode=True)
for i in range(num_epochs):
lr = 0.002 / (1 + 0.2 * i)
trainer = Trainer(net) \
.build_criterion(SeqCrossEntropyLoss) \
.build_optimizer('Adam', lr=lr) \
.set_max_num_epochs(1) \
.save_every((10, 'iterations')) \
.save_to_directory('net/')
trainer.bind_loader('train', data_loader, num_inputs=3)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
def train_model(args):
model = fm.model_fn(args)
print('Done initializing model')
# labels_all_dict = np.load('labels_all_dict.npy')
labels_all_dict = np.load('labels_all_dict_10s_subset_500.npy')
if args.no_val:
file_names_train = np.load(
'file_names_train_notest_hop_subset_500.npy')
# file_names_train = np.load('file_names_train_notest.npy')
train_dataloader = fd.create_dataloader(file_names_train,
args.data_directory,
labels_all_dict.item(), args)
else:
file_names_train = np.load('file_names_train.npy')
file_names_test = np.load('file_names_test.npy')
train_dataloader = fd.create_dataloader(file_names_train,
args.data_directory,
labels_all_dict.item(), args)
dev_dataloader = fd.create_dataloader(file_names_test,
args.data_directory,
labels_all_dict.item(), args)
# train_dataloader = fd.create_dataloader(train_feats, train_labels, args)
# dev_dataloader = fd.create_dataloader(dev_feats, dev_labels, args)
test_dataloader_flag = False
if test_dataloader_flag:
it = 0
for x, y, z in train_dataloader:
print('x.shape: {}'.format(x.shape))
print('y.shape: {}'.format(y.shape))
print('z.shape: {}'.format(z.shape))
print('x: {}'.format(x))
print('y: {}'.format(y))
print('z: {}'.format(z))
it = it + 1
print(it)
print('Done Creating Data Loaders')
if args.load_model:
(model, trainer) = load_model(args)
trainer \
.bind_loader('train', train_dataloader, num_inputs=2, num_targets=1)
if not args.no_val:
trainer.bind_loader('validate',
dev_dataloader,
num_inputs=2,
num_targets=1)
else:
# Build trainer
loss_fn = cust_CrossEntropyLoss()
trainer = Trainer(model) \
.build_criterion(loss_fn) \
.build_metric('CategoricalError') \
.build_optimizer(torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
if not args.no_val:
trainer \
.save_at_best_validation_score(True) \
.validate_every((1, 'epochs'))
trainer \
.bind_loader('train', train_dataloader, num_inputs=2, num_targets=1)
if not args.no_val:
trainer.bind_loader('validate',
dev_dataloader,
num_inputs=2,
num_targets=1)
# .build_optimizer('Adam') \
# .save_at_best_validation_score(True) \
# .build_metric(LevenshteinMetric) \
# .evaluate_metric_every('300 iterations') \
if args.cuda:
print('Switch trainer to CUDA')
trainer.cuda()
trainer.fit()
print('Done Fitting Trainer')
return model
def my_embed(folder, n):
click.echo('computing embeddings')
vae = Vae()
ds_train = HemoDataset(root_folder=root_folder, training=True)
ds_test = HemoDataset(root_folder=root_folder, training=False)
batch_size = 512
train_loader = data.DataLoader(ds_train, batch_size=batch_size, num_workers=8)
test_loader = data.DataLoader(ds_test, batch_size=batch_size, num_workers=8)
trainer = Trainer(vae)
trainer.save_to_directory(folder)
trainer.load()
trainer.cuda()
trainer.eval_mode()
i = 0
def compute_embedding_for_loader(loader, training_set: bool):
with torch.no_grad():
all_mu = []
all_filenames = []
with tqdm(total=len(loader)) as bar:
for k, x in enumerate(loader):
nonlocal i
i += 1
if n != 0 and i > n:
break
if training_set:
image, targets = x
else:
image = x
image = trainer.to_device(image)
_, x_reconstructed_batch, mu_batch, _ = trainer.apply_model(image)
image = image.detach().cpu().numpy()
x_reconstructed_batch = x_reconstructed_batch.detach().cpu().numpy()
mu_batch = mu_batch.detach().cpu().numpy()
all_mu.append(mu_batch)
start = k * loader.batch_size
end = start + loader.batch_size
if training_set:
paths = loader.dataset.training_file_paths
else:
paths = loader.dataset.testing_file_paths
filenames = [os.path.basename(paths[j]) for j in range(start, end)]
all_filenames.append(filenames)
bar.update(1)
all_mu = np.concatenate(all_mu, axis=0)
all_filenames = list(itertools.chain.from_iterable(all_filenames))
torch.cuda.empty_cache()
gc.collect()
return all_mu, all_filenames
h5_path = os.path.join(folder, 'embeddings.h5')
with h5py.File(h5_path, 'w') as f5:
embeddings, filenames = compute_embedding_for_loader(train_loader, training_set=True)
# breakpoint()
f5['training_data/embeddings'] = embeddings
pickle_path = os.path.join(folder, 'embeddings_training_data_row_names.pickle')
pickle.dump(filenames, open(pickle_path, 'wb'))
pushover_notification.send('training embeddings generated')
embeddings, filenames = compute_embedding_for_loader(test_loader, training_set=False)
f5['testing_data/embeddings'] = embeddings
pickle_path = os.path.join(folder, 'embeddings_testing_data_row_names.pickle')
pickle.dump(filenames, open(pickle_path, 'wb'))
pushover_notification.send('testing embeddings generated')
def main():
laptop = True
class LossWrapper(nn.Module):
def __init__(self):
super(LossWrapper, self).__init__()
#self.loss = torch.nn.CrossEntropyLoss(weight=torch.tensor([0.5, 2.0]), reduce=False)
self.loss = torch.nn.BCELoss(reduce=False)
def forward(self, input, target):
gt, sizes = target
input = torch.squeeze(input)
gt = torch.squeeze(gt)
w = (gt + 1)**2.75
sizes = torch.squeeze(sizes.float())
#print('it', input.size(), target.size())
l = self.loss(input, gt)
l = l * w
l = torch.sum(l * sizes) / torch.sum(sizes)
print("l", l.item())
return l
# Fill these in:
if laptop:
LOG_DIRECTORY = '/home/tbeier/src/extension-cpp/log/'
SAVE_DIRECTORY = '/home/tbeier/src/extension-cpp/savedir_new/'
RES_DIRECTORY = '/home/tbeier/src/extension-cpp/danu/'
pmap_root = "/home/tbeier/src/holy-edge/hed-data/out/"
bsd_root = "/home/tbeier/datasets/BSR/BSDS500/"
else:
LOG_DIRECTORY = '/export/home/tbeier/src/extension-cpp/log/'
SAVE_DIRECTORY = '/export/home/tbeier/src/extension-cpp/savedir_new/'
RES_DIRECTORY = '/export/home/tbeier/src/extension-cpp/danu/'
pmap_root = "/export/home/tbeier/bsd500_HED/"
bsd_root = "/home/tbeier/dataset/BSR/BSDS500/"
USE_CUDA = bool(1)
# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
# Define transforms (1)
#transforms = transforms.Compose([transforms.CenterCrop(100), transforms.ToTensor()])
#split = "train"
bsd_test = Bsd500Sp(bsd_root=bsd_root, pmap_root=pmap_root, split='test')
model = ConvNet() #.double()
#model.double()
# Build trainer
trainer = Trainer(model) \
.build_criterion(LossWrapper) \
.build_criterion(LossWrapper) \
.build_optimizer('Adam') \
.validate_every((4, 'epochs')) \
.save_every((4, 'epochs')) \
.save_to_directory(SAVE_DIRECTORY) \
.set_max_num_epochs(100) \
# .build_logger(TensorboardLogger(log_scalars_every='never',
# log_images_every='never'),
# log_directory=LOG_DIRECTORY)
# Bind loaders
#train_loader = torch.utils.data.DataLoader(dataset=bsd_train, num_workers=8)
#val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8)
#trainer \
# .bind_loader('train', train_loader, num_inputs=7, num_targets=2) \
# .bind_loader('validate', val_loader, num_inputs=7, num_targets=2)
#checkpoint = torch.load('savedir/checkpoint.pytorch')
# args.start_epoch = checkpoint['epoch']
# best_prec1 = checkpoint['best_prec1']
#model.load_state_dict(checkpoint['state_dict'])
trainer.cuda()
trainer.load(best=True,
from_directory=SAVE_DIRECTORY,
filename='best_checkpoint.pytorch')
meval = trainer.model.eval() #.cpu()
#model.load_state_dict(torch.load('savedir/checkpoint.pytorch'))
acc_vi_ds = 0.0
acc_ri_ds = 0.0
count = 0
for i in range(0, 200):
predictor = Predictor(model=meval,
ds=bsd_test,
output_folder=RES_DIRECTORY)
vi_img, ri_img = predictor.predict_mc(i)
acc_vi_ds += vi_img
acc_ri_ds += ri_img
count += 1
print("\n", i)
print("IMG ", vi_img, ri_img)
print("DS ", acc_vi_ds / count, acc_ri_ds / count)
if not os.path.exists(logs_dir):
os.mkdir(logs_dir)
log_info('Logs will be saved to %s' % (logs_dir))
# Build trainer
trainer = Trainer(model) \
.build_criterion('CrossEntropyLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam', lr=args.lr, betas=(0.9, 0.999), eps=1e-08) \
.validate_every((1, 'epochs')) \
.save_every((5, 'epochs')) \
.save_to_directory(model_dir) \
.set_max_num_epochs(10000) \
.register_callback(GradChecker()) \
.register_callback(AutoLR(0.96, (1, 'epochs'), monitor_momentum=0.9,
monitor_while='validating',
consider_improvement_with_respect_to='best'))\
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every=(1, 'epoch')),
log_directory=logs_dir)
# Bind loaders
trainer \
.bind_loader('train', train_dl) \
.bind_loader('validate', test_dl)
if torch.cuda.is_available():
trainer.cuda()
trainer.fit()
max_num_iterations = 10000
trainer = Trainer(model) \
.build_criterion('NLLLoss') \
.build_metric('CategoricalError') \
.build_optimizer('Adam') \
.save_every((1, 'epochs')) \
.save_to_directory(str(exp.save_directory))\
.validate_every((2, 'epochs'))\
.set_max_num_epochs(100) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iterations'),
log_images_every='never'),
log_directory=str(exp.log_directory))
# .save_every((2000, 'iterations'), to_directory=str(exp.save_directory), checkpoint_filename='latest') \
# Bind loaders
trainer.bind_loader('train', train_loader)
trainer.bind_loader('validate', test_loader)
trainer.cuda()
# Go!
trainer.fit()
# TODO: train different max pool sizes
# TODO: train different kernel numbers
# TODO: train different conv layers
# TODO: train with different number of training images
# TODO: report run time
