def train(net, dataset, criterion, num_epochs, batch_size, learn_rate, dir_name): dir_name = os.path.join('net/', dir_name) trainer = Trainer(net[0]) if (os.path.exists(os.path.join(dir_name, 'model.pytorch'))): net_temp = trainer.load_model(dir_name).model net[0].load_state_dict(net_temp.state_dict()) print("Loaded checkpoint directly") else: if (not os.path.exists(dir_name)): os.makedirs(dir_name) data_loader = torch.utils.data.DataLoader(dataset, shuffle=True, batch_size=batch_size) net[0].train() trainer \ .build_criterion(LossPrinter(criterion)) \ .bind_loader('train', data_loader) \ .build_optimizer('Adam', lr=learn_rate) \ .set_max_num_epochs(num_epochs) if torch.cuda.is_available(): trainer.cuda() trainer.fit() trainer.save_model(dir_name) net[0].cpu() net[0].eval()
def run(args): dataset = AlderleyWrapper() save_args(args) # save command line to a file for reference train_loader = alderley_cgan_data_loader(args, dataset=dataset) # get the data model = CGANModel( args, discriminators=[PixelDiscriminator(), patchCDiscriminatorNetwork(args)], generator=UnetUpsample()) # Build trainer trainer = Trainer(model) trainer.build_criterion(CWGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model)) trainer.build_optimizer('Adam', [parameter for discriminator in model.discriminators for parameter in discriminator.parameters()], lr=args.discriminator_lr) trainer.save_every((1, 'epochs')) trainer.save_to_directory(args.save_directory) trainer.set_max_num_epochs(args.epochs) trainer.register_callback(CGenerateDataCallback(args, dataset=dataset)) trainer.register_callback(CGeneratorTrainingCallback( args, parameters=model.generator.parameters(), criterion=WGANGeneratorLoss(), dataset=dataset)) trainer.bind_loader('train', train_loader, num_inputs=2) # Custom logging configuration so it knows to log our images if args.cuda: trainer.cuda() # Go! trainer.fit()
def test_cifar(self): from inferno.trainers.basic import Trainer from inferno.io.box.cifar10 import get_cifar10_loaders # Build cifar10 loaders trainloader, testloader = get_cifar10_loaders( root_directory=join(self.ROOT_DIR, 'data'), download=self.DOWNLOAD_CIFAR) # Make model net = self._make_test_model() tic = time.time() # Make trainer trainer = Trainer(model=net)\ .build_optimizer('Adam')\ .build_criterion('CrossEntropyLoss')\ .build_metric('CategoricalError')\ .validate_every((1, 'epochs'))\ .save_every((1, 'epochs'), to_directory=join(self.ROOT_DIR, 'saves'))\ .save_at_best_validation_score()\ .set_max_num_epochs(2) # Bind trainer to datasets trainer.bind_loader('train', trainloader).bind_loader('validate', testloader) # Check device and fit if self.CUDA: if self.HALF_PRECISION: trainer.cuda().set_precision('half').fit() else: trainer.cuda().fit() else: trainer.fit() toc = time.time() print("[*] Elapsed time: {} seconds.".format(toc - tic))
def train_model(args): model = model_fn() train_loader, validate_loader = mnist_data_loaders(args) # Build trainer trainer = Trainer(model) \ .build_criterion('RegularizedCrossEntropyLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam') \ .validate_every((1, 'epochs')) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) # Record regularization losses trainer.logger.observe_training_and_validation_states([ 'main_loss', 'total_regularization_loss', 'activity_regularization', 'l1_weight_regularization' ]) # Bind loaders trainer \ .bind_loader('train', train_loader) \ .bind_loader('validate', validate_loader) if args.cuda: trainer.cuda() # Go! trainer.fit()
def train_model(args): model = MNISTCNNModel() train_loader, validate_loader = mnist_data_loaders(args) # Build trainer trainer = Trainer(model) \ .build_criterion('CrossEntropyLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam') \ .validate_every((2, 'epochs')) \ .save_every((5, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) # Bind loaders trainer \ .bind_loader('train', train_loader) \ .bind_loader('validate', validate_loader) if args.cuda: trainer.cuda() # Go! trainer.fit()
def train_model(model, train_dataset, valid_dataset, args): kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {} train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size, shuffle=True, **kw) validate_loader = DataLoader(dataset=valid_dataset, batch_size=args.batch_size, **kw) trainer = Trainer(model) \ .build_criterion(CrossEntropyLoss3D) \ .build_metric(CategoricalError3D) \ .build_optimizer('Adam', weight_decay=1e-6) \ .save_every((1, 'epochs')) \ .validate_every((100, 'iteration')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger( TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory='tb_log/' ) \ .bind_loader('train', train_loader) \ .bind_loader('validate', validate_loader) if args.cuda: trainer.cuda() # Go! trainer.fit()
def train_net_with_inferno(config_dict, net, criterion, optimizer, trainloader, valloader): """ Trains the NeuralNet with inferno :param config_dict: dict with configs :param net: NeuralNet :param criterion: criterion for NN :param optimizer: optimizer for NN :param trainloader: dataloader with traind ata :param valloader: dataloader with validation data """ print("Start training with inferno!") from inferno.trainers.basic import Trainer from inferno.trainers.callbacks.essentials import SaveAtBestValidationScore model_folder = os.path.join(config_dict["project_folder"], "model/") if not os.path.exists(model_folder): os.mkdir(model_folder) trainer = Trainer(net) \ .save_every((1, 'epochs'), to_directory=model_folder) \ .build_criterion(criterion) \ .build_optimizer(optimizer) \ .build_metric(sorensen_dice_metric) \ .evaluate_metric_every('never') \ .validate_every((1, 'epochs'), for_num_iterations=50) \ .register_callback(SaveAtBestValidationScore(smoothness=.5)) trainer.set_max_num_epochs(config_dict['max_train_epochs']) trainer.bind_loader('train', trainloader).bind_loader('validate', valloader) trainer.cuda() trainer.fit()
def main(args): img_size = 224 train_trans = transforms.Compose([ transforms.RandomResizedCrop(img_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) test_trans = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(img_size), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) train_ds = IBMDataset(args.data_dir, transform=train_trans, name='train') train_loader = data.DataLoader(train_ds, batch_size = args.batch_size, shuffle = True) val_ds = IBMDataset(args.data_dir, transform=test_trans, name='val') print(len(val_ds), len(train_ds)) val_loader = data.DataLoader(val_ds, batch_size = args.batch_size, shuffle = True) LOG_DIRECTORY = '../logs/' SAVE_DIRECTORY = '../models/resnet50/' model = ResNet50() trainer = Trainer(model) \ .build_criterion('CrossEntropyLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam') \ .validate_every((1, 'epochs')) \ .save_every((1, 'epochs')) \ .save_to_directory(SAVE_DIRECTORY) \ .set_max_num_epochs(args.num_epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=LOG_DIRECTORY) # Bind loaders trainer \ .bind_loader('train', train_loader) \ .bind_loader('validate', val_loader) if torch.cuda.is_available(): trainer.cuda() trainer.fit()
def task(env): dataset_name = 'MNIST' dataset_path = env.dataset(dataset_name) batch_size = 128 transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) mnist_set = torchvision.datasets.MNIST(str(dataset_path), train=True, download=True, transform=transform) train_loader = torch.utils.data.DataLoader(mnist_set, batch_size=batch_size, shuffle=True, num_workers=2) mnist_test_set = torchvision.datasets.MNIST(str(dataset_path), train=False, download=True, transform=transform) test_loader = torch.utils.data.DataLoader(mnist_test_set, batch_size=512, shuffle=True, num_workers=2) model = Model(n_classes=10, in_channels=1, layers=4) exp = TrainLog(env, dataset_name,model,log_time=True) logging.info(' saving to %s', exp.save_directory) logging.info(' logging to %s', exp.log_directory) # Load loaders # train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY, # download=DOWNLOAD_CIFAR) # Build trainer iterations = 5000 trainer = Trainer(model) \ .build_criterion('NLLLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam', lr=0.001) \ .save_every((1, 'epochs')) \ .save_to_directory(str(exp.save_directory)) \ .validate_every((1, 'epochs'))\ .set_max_num_iterations(iterations) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=str(exp.log_directory)) # Bind loaders trainer.bind_loader('train', train_loader) trainer.bind_loader('validate', test_loader) trainer.cuda() # Go! logging.info('start training') trainer.fit() trainer.set_max_num_iterations(trainer.iteration_count + iterations) trainer.build_optimizer('Adam',lr=0.0001) logging.info('slower lr') trainer.fit()
def task(env): dataset_name = 'MNIST' dataset_path = env.dataset(dataset_name) dataset = MNIST(str(dataset_path), train=True, download=True, transform=mnist_to_tensor32) testset = MNIST(str(dataset_path), train=False, download=True, transform=mnist_to_tensor32) test_loader = DataLoader(testset, batch_size=512, shuffle=True, num_workers=2) train_loader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=2) triplets = dis.TripelDataset(dataset, dataset.train_labels, K=1) train_loader_triple = DataLoader(triplets, batch_size=16,collate_fn=dis.stack_triplets) model = Model(in_channels=1, n_classes=10, shortcut=True) criterion = dis.TripletLoss() exp = TrainLog(env, dataset_name,model.info,log_time=True) logging.info(' saving to %s', exp.save_directory) logging.info(' logging to %s', exp.log_directory) # Load loaders # train_loader, validate_loader = get_cifar10_loaders(DATASET_DIRECTORY, # download=DOWNLOAD_CIFAR) iterations = 0 trainer = Trainer(model) set_log_and_save(trainer, exp) trainer.build_criterion(criterion) \ .build_optimizer('SGD', lr=0.001, weight_decay=0.0005) \ .set_max_num_iterations(iterations)\ .bind_loader('train', train_loader_triple) trainer.cuda() logging.info('start training') trainer.fit() print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda())) model.use_shortcut = False model.classify = True print(model.forward(Variable(next(iter(train_loader_triple))[0]).cuda())) trainer = Trainer(model) set_log_and_save(trainer, exp) trainer.build_criterion('CrossEntropyLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam', lr=0.001) \ .set_max_num_iterations(5000) \ .bind_loader('train', train_loader) \ .bind_loader('test', test_loader) trainer.cuda() logging.info('start training') trainer.fit()
def train_model(args): """ Performs the training """ if os.path.exists( os.path.join(args.save_directory, Trainer()._checkpoint_filename)): # Skip training if checkpoint exists return model = WsjModel(args) kwargs = { 'num_workers': args.num_workers, 'pin_memory': True } if args.cuda else {} train_loader = DataLoader(WsjDataset('train', args), shuffle=True, batch_size=args.batch_size, collate_fn=wsj_collate_fn, **kwargs) validate_loader = DataLoader(WsjDataset('dev', args), shuffle=True, batch_size=args.batch_size, collate_fn=wsj_collate_fn, **kwargs) # Build trainer trainer = Trainer(model) \ .build_criterion(ctc_loss()) \ .build_metric(ctc_loss()) \ .build_optimizer('Adam') \ .validate_every((1, 'epochs')) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ #.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), #log_images_every='never'), #log_directory=args.save_directory) # Bind loaders trainer.bind_loader('train', train_loader, num_inputs=3) trainer.bind_loader('validate', validate_loader, num_inputs=3) trainer.register_callback(EpochTimer()) if args.cuda: trainer.cuda() # Go! trainer.fit() trainer.save()
def my_train(load, folder): click.echo('starting training') os.makedirs(folder, exist_ok=True) # joint_transform = Compose( # RandomRotate(), # RandomTranspose(), # RandomFlip() # ) # setup logger os.makedirs('derived_data/log', exist_ok=True) Logger.instance().setup('derived_data/log') vae = Vae() ds = HemoDataset(root_folder=root_folder, image_transform=None, training=True) train_loader = torch.utils.data.DataLoader(ds, batch_size=1536, num_workers=8) # Build trainer trainer = Trainer(vae) trainer.save_to_directory(folder) if load: trainer.load() # trainer.cuda(devices=[0, 1]) trainer.cuda() trainer.build_criterion(vae.loss_function()) trainer.build_optimizer('Adam', lr=0.001) # trainer.validate_every((2, 'epochs')) trainer.save_every((1, 'epochs')) trainer.set_max_num_epochs(100) trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never', # log_images_every=(1, 'iteration'), log_directory=folder)) # Bind loaders trainer.bind_loader('train', train_loader, num_inputs=1, num_targets=1) # bind callbacks trainer.register_callback(GarbageCollection(), trigger='end_of_training_iteration') trainer.register_callback(ShowMinimalConsoleInfo(), trigger='end_of_training_iteration') # trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1) trainer.fit() pushover_notification.send('embeddings generated')
def run(args): dataset = RorschachWrapper() save_args(args) # save command line to a file for reference train_loader = rorschach_cgan_data_loader(args, dataset=dataset) # get the data # todo model = patchCWGANModel(args, discriminator=patchCDiscriminatorNetwork(args), generator=CGeneratorNetwork(args)) # Build trainer trainer = Trainer(model) trainer.build_criterion( CWGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model)) trainer.build_optimizer('Adam', model.discriminator.parameters(), lr=args.discriminator_lr) trainer.save_every((1, 'epochs')) trainer.save_to_directory(args.save_directory) trainer.set_max_num_epochs(args.epochs) trainer.register_callback(CGenerateDataCallback(args, dataset=dataset)) trainer.register_callback( CGeneratorTrainingCallback(args, parameters=model.generator.parameters(), criterion=WGANGeneratorLoss(), dataset=dataset)) trainer.bind_loader('train', train_loader, num_inputs=2) # Custom logging configuration so it knows to log our images logger = TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every=(args.log_image_frequency, 'iteration')) trainer.build_logger(logger, log_directory=args.save_directory) logger.observe_state('generated_images') logger.observe_state('real_images') logger._trainer_states_being_observed_while_training.remove( 'training_inputs') if args.cuda: trainer.cuda() # Go! trainer.fit()
def run(args): model = SpeechModel(args) trainer = Trainer() if os.path.exists( os.path.join(args.save_directory, trainer._checkpoint_filename)): trainer.load(from_directory=args.save_directory) model.load_state_dict(trainer.model.state_dict()) if args.cuda: model = model.cuda() else: train_loader = make_loader('train', args, batch_size=args.batch_size) dev_loader = make_loader('dev', args, batch_size=args.batch_size) # Build trainer trainer = Trainer(model) \ .build_criterion(CTCCriterion, size_average=True) \ .build_optimizer('Adam', weight_decay=1e-7, lr=5e-5) \ .validate_every((1, 'epochs')) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) # Bind loaders trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1) trainer.bind_loader('validate', dev_loader, num_inputs=3, num_targets=1) if args.cuda: trainer.cuda() # Go! trainer.fit() trainer.save() test_loader = make_loader('test', args=args, shuffle=False, batch_size=1, test=True) run_logits(loader=test_loader, model=model, args=args)
def train_model(args): """ Perform training then call prediction """ model = all_cnn_module() model.apply(initializer) train_loader, validate_loader, test_loader = make_loaders(args) # Build trainer savepath = os.path.join(args.save_directory, Trainer()._checkpoint_filename) if os.path.exists(savepath): trainer = Trainer().load(from_directory=args.save_directory) if args.cuda: trainer.cuda() else: trainer = Trainer(model) \ .build_criterion('CrossEntropyLoss') \ .build_metric('CategoricalError') \ .save_every((1, 'epochs')) \ .validate_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) # These are the params from the paper trainer.build_optimizer('SGD', lr=0.01, momentum=0.9, weight_decay=0.001) # Also works with Adam and default settings # trainer.build_optimizer('Adam') trainer.bind_loader('train', train_loader) trainer.bind_loader('validate', validate_loader) if args.cuda: trainer.cuda() # Go! trainer.fit() trainer.save() write_predictions(args, trainer.model, test_loader)
def run(args): save_args(args) # save command line to a file for reference train_loader = mnist_data_loader(args) # get the data model = GANModel(args, discriminator=DiscriminatorNetwork(args), generator=GeneratorNetwork(args)) # Build trainer trainer = Trainer(model) trainer.build_criterion( WGANDiscriminatorLoss(penalty_weight=args.penalty_weight, model=model)) trainer.build_optimizer('Adam', model.discriminator.parameters(), lr=args.discriminator_lr) trainer.save_every((1, 'epochs')) trainer.save_to_directory(args.save_directory) trainer.set_max_num_epochs(args.epochs) trainer.register_callback(GenerateDataCallback(args)) trainer.register_callback( GeneratorTrainingCallback(args, parameters=model.generator.parameters(), criterion=WGANGeneratorLoss())) trainer.bind_loader('train', train_loader) # Custom logging configuration so it knows to log our images logger = TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every=(args.log_image_frequency, 'iteration')) trainer.build_logger(logger, log_directory=args.save_directory) logger.observe_state('generated_images') logger.observe_state('real_images') logger._trainer_states_being_observed_while_training.remove( 'training_inputs') if args.cuda: trainer.cuda() # Go! trainer.fit() # Generate video from saved images if not args.no_ffmpeg: generate_video(args.save_directory)
def train_model(model, dataset, args): kw = {'num_workers': 2, 'pin_memory': True} if args.cuda else {} loader = DataLoader(dataset=dataset, batch_size=args.batch_size, shuffle=True, **kw) trainer = Trainer(model) \ .build_criterion(CrossEntropyLoss3D) \ .build_metric(CategoricalError3D) \ .build_optimizer('Adam', weight_decay=1e-6) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .register_callback(CustomLogger) \ .bind_loader('train', loader) if args.cuda: trainer.cuda() # Go! trainer.fit()
def train(net, num_epochs, dataset): data_loader = torch.utils.data.DataLoader(dataset, shuffle=True, batch_size=100) net.train(mode=True) for i in range(num_epochs): lr = 0.002 / (1 + 0.2 * i) trainer = Trainer(net) \ .build_criterion(SeqCrossEntropyLoss) \ .build_optimizer('Adam', lr=lr) \ .set_max_num_epochs(1) \ .save_every((10, 'iterations')) \ .save_to_directory('net/') trainer.bind_loader('train', data_loader, num_inputs=3) if torch.cuda.is_available(): trainer.cuda() trainer.fit()
def train_model(args): model = fm.model_fn(args) print('Done initializing model') # labels_all_dict = np.load('labels_all_dict.npy') labels_all_dict = np.load('labels_all_dict_10s_subset_500.npy') if args.no_val: file_names_train = np.load( 'file_names_train_notest_hop_subset_500.npy') # file_names_train = np.load('file_names_train_notest.npy') train_dataloader = fd.create_dataloader(file_names_train, args.data_directory, labels_all_dict.item(), args) else: file_names_train = np.load('file_names_train.npy') file_names_test = np.load('file_names_test.npy') train_dataloader = fd.create_dataloader(file_names_train, args.data_directory, labels_all_dict.item(), args) dev_dataloader = fd.create_dataloader(file_names_test, args.data_directory, labels_all_dict.item(), args) # train_dataloader = fd.create_dataloader(train_feats, train_labels, args) # dev_dataloader = fd.create_dataloader(dev_feats, dev_labels, args) test_dataloader_flag = False if test_dataloader_flag: it = 0 for x, y, z in train_dataloader: print('x.shape: {}'.format(x.shape)) print('y.shape: {}'.format(y.shape)) print('z.shape: {}'.format(z.shape)) print('x: {}'.format(x)) print('y: {}'.format(y)) print('z: {}'.format(z)) it = it + 1 print(it) print('Done Creating Data Loaders') if args.load_model: (model, trainer) = load_model(args) trainer \ .bind_loader('train', train_dataloader, num_inputs=2, num_targets=1) if not args.no_val: trainer.bind_loader('validate', dev_dataloader, num_inputs=2, num_targets=1) else: # Build trainer loss_fn = cust_CrossEntropyLoss() trainer = Trainer(model) \ .build_criterion(loss_fn) \ .build_metric('CategoricalError') \ .build_optimizer(torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) if not args.no_val: trainer \ .save_at_best_validation_score(True) \ .validate_every((1, 'epochs')) trainer \ .bind_loader('train', train_dataloader, num_inputs=2, num_targets=1) if not args.no_val: trainer.bind_loader('validate', dev_dataloader, num_inputs=2, num_targets=1) # .build_optimizer('Adam') \ # .save_at_best_validation_score(True) \ # .build_metric(LevenshteinMetric) \ # .evaluate_metric_every('300 iterations') \ if args.cuda: print('Switch trainer to CUDA') trainer.cuda() trainer.fit() print('Done Fitting Trainer') return model
def my_embed(folder, n): click.echo('computing embeddings') vae = Vae() ds_train = HemoDataset(root_folder=root_folder, training=True) ds_test = HemoDataset(root_folder=root_folder, training=False) batch_size = 512 train_loader = data.DataLoader(ds_train, batch_size=batch_size, num_workers=8) test_loader = data.DataLoader(ds_test, batch_size=batch_size, num_workers=8) trainer = Trainer(vae) trainer.save_to_directory(folder) trainer.load() trainer.cuda() trainer.eval_mode() i = 0 def compute_embedding_for_loader(loader, training_set: bool): with torch.no_grad(): all_mu = [] all_filenames = [] with tqdm(total=len(loader)) as bar: for k, x in enumerate(loader): nonlocal i i += 1 if n != 0 and i > n: break if training_set: image, targets = x else: image = x image = trainer.to_device(image) _, x_reconstructed_batch, mu_batch, _ = trainer.apply_model(image) image = image.detach().cpu().numpy() x_reconstructed_batch = x_reconstructed_batch.detach().cpu().numpy() mu_batch = mu_batch.detach().cpu().numpy() all_mu.append(mu_batch) start = k * loader.batch_size end = start + loader.batch_size if training_set: paths = loader.dataset.training_file_paths else: paths = loader.dataset.testing_file_paths filenames = [os.path.basename(paths[j]) for j in range(start, end)] all_filenames.append(filenames) bar.update(1) all_mu = np.concatenate(all_mu, axis=0) all_filenames = list(itertools.chain.from_iterable(all_filenames)) torch.cuda.empty_cache() gc.collect() return all_mu, all_filenames h5_path = os.path.join(folder, 'embeddings.h5') with h5py.File(h5_path, 'w') as f5: embeddings, filenames = compute_embedding_for_loader(train_loader, training_set=True) # breakpoint() f5['training_data/embeddings'] = embeddings pickle_path = os.path.join(folder, 'embeddings_training_data_row_names.pickle') pickle.dump(filenames, open(pickle_path, 'wb')) pushover_notification.send('training embeddings generated') embeddings, filenames = compute_embedding_for_loader(test_loader, training_set=False) f5['testing_data/embeddings'] = embeddings pickle_path = os.path.join(folder, 'embeddings_testing_data_row_names.pickle') pickle.dump(filenames, open(pickle_path, 'wb')) pushover_notification.send('testing embeddings generated')
def main(): laptop = True class LossWrapper(nn.Module): def __init__(self): super(LossWrapper, self).__init__() #self.loss = torch.nn.CrossEntropyLoss(weight=torch.tensor([0.5, 2.0]), reduce=False) self.loss = torch.nn.BCELoss(reduce=False) def forward(self, input, target): gt, sizes = target input = torch.squeeze(input) gt = torch.squeeze(gt) w = (gt + 1)**2.75 sizes = torch.squeeze(sizes.float()) #print('it', input.size(), target.size()) l = self.loss(input, gt) l = l * w l = torch.sum(l * sizes) / torch.sum(sizes) print("l", l.item()) return l # Fill these in: if laptop: LOG_DIRECTORY = '/home/tbeier/src/extension-cpp/log/' SAVE_DIRECTORY = '/home/tbeier/src/extension-cpp/savedir_new/' RES_DIRECTORY = '/home/tbeier/src/extension-cpp/danu/' pmap_root = "/home/tbeier/src/holy-edge/hed-data/out/" bsd_root = "/home/tbeier/datasets/BSR/BSDS500/" else: LOG_DIRECTORY = '/export/home/tbeier/src/extension-cpp/log/' SAVE_DIRECTORY = '/export/home/tbeier/src/extension-cpp/savedir_new/' RES_DIRECTORY = '/export/home/tbeier/src/extension-cpp/danu/' pmap_root = "/export/home/tbeier/bsd500_HED/" bsd_root = "/home/tbeier/dataset/BSR/BSDS500/" USE_CUDA = bool(1) # Device configuration device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') print(device) # Define transforms (1) #transforms = transforms.Compose([transforms.CenterCrop(100), transforms.ToTensor()]) #split = "train" bsd_test = Bsd500Sp(bsd_root=bsd_root, pmap_root=pmap_root, split='test') model = ConvNet() #.double() #model.double() # Build trainer trainer = Trainer(model) \ .build_criterion(LossWrapper) \ .build_criterion(LossWrapper) \ .build_optimizer('Adam') \ .validate_every((4, 'epochs')) \ .save_every((4, 'epochs')) \ .save_to_directory(SAVE_DIRECTORY) \ .set_max_num_epochs(100) \ # .build_logger(TensorboardLogger(log_scalars_every='never', # log_images_every='never'), # log_directory=LOG_DIRECTORY) # Bind loaders #train_loader = torch.utils.data.DataLoader(dataset=bsd_train, num_workers=8) #val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8) #trainer \ # .bind_loader('train', train_loader, num_inputs=7, num_targets=2) \ # .bind_loader('validate', val_loader, num_inputs=7, num_targets=2) #checkpoint = torch.load('savedir/checkpoint.pytorch') # args.start_epoch = checkpoint['epoch'] # best_prec1 = checkpoint['best_prec1'] #model.load_state_dict(checkpoint['state_dict']) trainer.cuda() trainer.load(best=True, from_directory=SAVE_DIRECTORY, filename='best_checkpoint.pytorch') meval = trainer.model.eval() #.cpu() #model.load_state_dict(torch.load('savedir/checkpoint.pytorch')) acc_vi_ds = 0.0 acc_ri_ds = 0.0 count = 0 for i in range(0, 200): predictor = Predictor(model=meval, ds=bsd_test, output_folder=RES_DIRECTORY) vi_img, ri_img = predictor.predict_mc(i) acc_vi_ds += vi_img acc_ri_ds += ri_img count += 1 print("\n", i) print("IMG ", vi_img, ri_img) print("DS ", acc_vi_ds / count, acc_ri_ds / count)
if not os.path.exists(logs_dir): os.mkdir(logs_dir) log_info('Logs will be saved to %s' % (logs_dir)) # Build trainer trainer = Trainer(model) \ .build_criterion('CrossEntropyLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam', lr=args.lr, betas=(0.9, 0.999), eps=1e-08) \ .validate_every((1, 'epochs')) \ .save_every((5, 'epochs')) \ .save_to_directory(model_dir) \ .set_max_num_epochs(10000) \ .register_callback(GradChecker()) \ .register_callback(AutoLR(0.96, (1, 'epochs'), monitor_momentum=0.9, monitor_while='validating', consider_improvement_with_respect_to='best'))\ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every=(1, 'epoch')), log_directory=logs_dir) # Bind loaders trainer \ .bind_loader('train', train_dl) \ .bind_loader('validate', test_dl) if torch.cuda.is_available(): trainer.cuda() trainer.fit()
max_num_iterations = 10000 trainer = Trainer(model) \ .build_criterion('NLLLoss') \ .build_metric('CategoricalError') \ .build_optimizer('Adam') \ .save_every((1, 'epochs')) \ .save_to_directory(str(exp.save_directory))\ .validate_every((2, 'epochs'))\ .set_max_num_epochs(100) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iterations'), log_images_every='never'), log_directory=str(exp.log_directory)) # .save_every((2000, 'iterations'), to_directory=str(exp.save_directory), checkpoint_filename='latest') \ # Bind loaders trainer.bind_loader('train', train_loader) trainer.bind_loader('validate', test_loader) trainer.cuda() # Go! trainer.fit() # TODO: train different max pool sizes # TODO: train different kernel numbers # TODO: train different conv layers # TODO: train with different number of training images # TODO: report run time