def prepare(cls, model_name, **kwargs): """ Loads and prepares the data, the optimizer and the criterion Parameters ---------- model_name : str Name of the model. Used for loading the model. kwargs : dict Any additional arguments. Returns ------- model : DataParallel The model to train num_classes : int How many different classes there are in our problem. Used for loading the model. best_value : float Best value of the model so far. Non-zero only in case of --resume being used train_loader : torch.utils.data.dataloader.DataLoader Training dataloader val_loader : torch.utils.data.dataloader.DataLoader Validation dataloader test_loader : torch.utils.data.dataloader.DataLoader Test set dataloader optimizer : torch.optim Optimizer to use during training, e.g. SGD criterion : torch.nn.modules.loss Loss function to use, e.g. cross-entropy """ # Get the selected model input size model_expected_input_size = models.__dict__[model_name]( ).expected_input_size if type(model_expected_input_size ) is not tuple or len(model_expected_input_size) != 2: logging.error( 'Model {model_name} expected input size is not a tuple. ' 'Received: {model_expected_input_size}'.format( model_name=model_name, model_expected_input_size=model_expected_input_size)) sys.exit(-1) logging.info('Model {} expects input size of {}'.format( model_name, model_expected_input_size)) # Setting up the dataloaders train_loader, val_loader, test_loader, num_classes = set_up_dataloaders( model_expected_input_size, **kwargs) # Setting up model, optimizer, criterion model, criterion, optimizer, best_value = set_up_model( model_name=model_name, num_classes=num_classes, **kwargs) return model, num_classes, best_value, train_loader, val_loader, test_loader, optimizer, criterion
def single_run(writer, current_log_folder, model_name, epochs, lr, decay_lr, validation_interval, **kwargs): """ This is the main routine where train(), validate() and test() are called. Parameters ---------- writer : Tensorboard.SummaryWriter Responsible for writing logs in Tensorboard compatible format. current_log_folder : string Path to where logs/checkpoints are saved model_name : string Name of the model epochs : int Number of epochs to train lr : float Value for learning rate kwargs : dict Any additional arguments. decay_lr : boolean Decay the lr flag validation_interval: int Run evaluation on validation set every N epochs Returns ------- train_value : ndarray[floats] Accuracy values for train val_value : ndarray(1, `epochs`+1) test_value : float Precision values for train and validation splits. Single precision value for the test split. """ # Get the selected model input size model_expected_input_size = models.__dict__[model_name]( ).expected_input_size RandomLabel._validate_model_input_size(model_expected_input_size, model_name) logging.info('Model {} expects input size of {}'.format( model_name, model_expected_input_size)) # Setting up the dataloaders train_loader, val_loader, test_loader, num_classes = set_up_dataloaders( model_expected_input_size, **kwargs) # Remove the "shuffle=True" for the train loader train_loader.sampler = SequentialSampler(train_loader.dataset) train_loader.batch_sampler = BatchSampler(train_loader.sampler, train_loader.batch_size, train_loader.drop_last) # Setting up model, optimizer, criterion model, criterion, optimizer, best_value, start_epoch = set_up_model( num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) # Create an observer observer, observer_criterion, observer_optimizer, _, _ = set_up_model( num_classes=num_classes, model_name='Observer', lr=lr, train_loader=train_loader, **kwargs) # Core routine logging.info('Begin training') val_value = np.zeros((epochs + 1 - start_epoch)) train_value = np.zeros((epochs - start_epoch)) val_value[-1] = RandomLabel._validate(val_loader, model, criterion, observer, observer_criterion, writer, -1, **kwargs) for epoch in range(start_epoch, epochs): # Train train_value[epoch] = RandomLabel._train(train_loader, model, criterion, optimizer, observer, observer_criterion, observer_optimizer, writer, epoch, **kwargs) # Validate if epoch % validation_interval == 0: val_value[epoch] = RandomLabel._validate( val_loader, model, criterion, observer, observer_criterion, writer, epoch, **kwargs) if decay_lr is not None: adjust_learning_rate(lr=lr, optimizer=optimizer, epoch=epoch, decay_lr_epochs=decay_lr) best_value = checkpoint(epoch, val_value[epoch], best_value, model, optimizer, current_log_folder) # Test test_value = RandomLabel._test(test_loader, model, criterion, observer, observer_criterion, writer, epochs - 1, **kwargs) logging.info('Training completed') return train_value, val_value, test_value
def single_run(writer, current_log_folder, model_name, epochs, lr, decay_lr, validation_interval, **kwargs): """ This is the main routine where train(), validate() and test() are called. Parameters ---------- :param writer: Tensorboard SummaryWriter Responsible for writing logs in Tensorboard compatible format. :param current_log_folder: string Path to where logs/checkpoints are saved :param model_name: string Name of the model :param epochs: int Number of epochs to train :param lr: float Value for learning rate :param decay_lr: boolean Decay the lr flag :param validation_interval: int Run evaluation on validation set every N epochs :param kwargs: dict Any additional arguments. :return: train_value, val_value, test_value Precision values for train and validation splits. Single precision value for the test split. """ # Get the selected model model_expected_input_size = models.__dict__[model_name]().expected_input_size Bidimensional._validate_model_input_size(model_expected_input_size, model_name) logging.info('Model {} expects input size of {}'.format(model_name, model_expected_input_size)) # Setting up the dataloaders train_loader, val_loader, test_loader, num_classes = set_up_dataloaders(model_expected_input_size, **kwargs) # Setting up model, optimizer, criterion model, criterion, optimizer, best_value, start_epoch = set_up_model(num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) # Core routine logging.info('Begin training') val_value = np.zeros((epochs + 1 - start_epoch)) train_value = np.zeros((epochs - start_epoch)) # Make data for points grid_resolution = 100 mini_batches = np.array([input_mini_batch.numpy() for input_mini_batch, _ in val_loader]) val_coords = np.squeeze(np.array([sample for mini_batch in mini_batches for sample in mini_batch])) min_x, min_y = np.min(val_coords[:, 0]), np.min(val_coords[:, 1]) max_x, max_y = np.max(val_coords[:, 0]), np.max(val_coords[:, 1]) coords = np.array([[x, y] for x in np.linspace(min_x, max_x, grid_resolution) for y in np.linspace(min_y, max_y, grid_resolution) ]) coords = torch.autograd.Variable(torch.from_numpy(coords).type(torch.FloatTensor)) if not kwargs['no_cuda']: coords = coords.cuda(async=True) # PLOT: decision boundary routine Bidimensional._evaluate_and_plot_decision_boundary(model=model, val_coords=val_coords, coords=coords, grid_resolution=grid_resolution, val_loader=val_loader, num_classes=num_classes, writer=writer, epoch=-1, epochs=epochs, **kwargs) val_value[-1] = Bidimensional._validate(val_loader, model, criterion, writer, -1, **kwargs) # Add model parameters to Tensorboard for name, param in model.named_parameters(): writer.add_histogram(name + '_-1', param.clone().cpu().data.numpy(), -1, bins='auto') for epoch in range(start_epoch, epochs): # Train train_value[epoch] = Bidimensional._train(train_loader, model, criterion, optimizer, writer, epoch, **kwargs) # Validate if epoch % validation_interval == 0: val_value[epoch] = Bidimensional._validate(val_loader, model, criterion, writer, epoch, **kwargs) if decay_lr is not None: adjust_learning_rate(lr, optimizer, epoch, decay_lr) best_value = checkpoint(epoch, val_value[epoch], best_value, model, optimizer, current_log_folder) # PLOT: decision boundary routine Bidimensional._evaluate_and_plot_decision_boundary(model=model, val_coords=val_coords, coords=coords, grid_resolution=grid_resolution, val_loader=val_loader, num_classes=num_classes, writer=writer, epoch=epoch, epochs=epochs, **kwargs) # Add model parameters to Tensorboard for name, param in model.named_parameters(): writer.add_histogram(name + '_{}'.format(epoch), param.clone().cpu().data.numpy(), epoch, bins='auto') # Test test_value = Bidimensional._test(test_loader, model, criterion, writer, epochs, **kwargs) logging.info('Training completed') return train_value, val_value, test_value
def single_run(writer, current_log_folder, model_name, epochs, lr, decay_lr, validation_interval, checkpoint_all_epochs, **kwargs): """ This is the main routine where train(), validate() and test() are called. Parameters ---------- writer : Tensorboard.SummaryWriter Responsible for writing logs in Tensorboard compatible format. current_log_folder : string Path to where logs/checkpoints are saved model_name : string Name of the model epochs : int Number of epochs to train lr : float Value for learning rate kwargs : dict Any additional arguments. decay_lr : boolean Decay the lr flag validation_interval : int Run evaluation on validation set every N epochs checkpoint_all_epochs : bool If enabled, save checkpoint after every epoch. Returns ------- train_value : ndarray[floats] of size (1, `epochs`) Accuracy values for train split val_value : ndarray[floats] of size (1, `epochs`+1) Accuracy values for validation split test_value : float Accuracy value for test split """ # Get the selected model input size model_expected_input_size = models.__dict__[model_name]().expected_input_size ImageClassification._validate_model_input_size(model_expected_input_size, model_name) logging.info('Model {} expects input size of {}'.format(model_name, model_expected_input_size)) # Setting up the dataloaders train_loader, val_loader, test_loader, num_classes = set_up_dataloaders(model_expected_input_size, **kwargs) # Setting up model, optimizer, criterion model, criterion, optimizer, best_value, start_epoch = set_up_model(num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) pytorch_total_params = sum(p.numel() for p in model.parameters()) logging.info('Total parameters: ' + str(pytorch_total_params)) pytorch_total_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) logging.info('Total trainable parameters: ' + str(pytorch_total_params_trainable)) # Core routine logging.info('Begin training') val_value = np.zeros((epochs + 1 - start_epoch)) train_value = np.zeros((epochs - start_epoch)) val_value[-1] = ImageClassification._validate(val_loader, model, criterion, writer, -1, **kwargs) for epoch in range(start_epoch, epochs): # Train train_value[epoch] = ImageClassification._train(train_loader, model, criterion, optimizer, writer, epoch, **kwargs) # Validate if epoch % validation_interval == 0: val_value[epoch] = ImageClassification._validate(val_loader, model, criterion, writer, epoch, **kwargs) if decay_lr is not None: adjust_learning_rate(lr=lr, optimizer=optimizer, epoch=epoch, decay_lr_epochs=decay_lr) best_value = checkpoint(epoch=epoch, new_value=val_value[epoch], best_value=best_value, model=model, optimizer=optimizer, log_dir=current_log_folder, checkpoint_all_epochs=checkpoint_all_epochs) # Load the best model before evaluating on the test set. logging.info('Loading the best model before evaluating on the ' 'test set.') kwargs["load_model"] = os.path.join(current_log_folder, 'model_best.pth.tar') model, _, _, _, _ = set_up_model(num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) # Test test_value = ImageClassification._test(test_loader, model, criterion, writer, epochs - 1, **kwargs) logging.info('Training completed') return train_value, val_value, test_value
def single_run(writer, current_log_folder, model_name, epochs, lr, decay_lr, validation_interval, checkpoint_all_epochs, **kwargs): """ DESC Parameters ---------- Param Desc Returns ------- None """ if not kwargs['train'] and kwargs['load_model'] == None: logging.error( 'You have to provide load_model argument if model is not trained.' ) sys.exit(-1) # Get the selected model input size model_expected_input_size = models.__dict__[model_name]( ).expected_input_size ProcessActivation._validate_model_input_size(model_expected_input_size, model_name) logging.info('Model {} expects input size of {}'.format( model_name, model_expected_input_size)) # Setting up the dataloaders train_loader, val_loader, test_loader, num_classes = set_up_dataloaders( model_expected_input_size, **kwargs) # Freezing the dataset used for processing activation activation_dataset = [] for i, data in enumerate(train_loader): activation_dataset.append(data) if i >= kwargs['process_size']: break # Setting up model, optimizer, criterion model, criterion, optimizer, best_value, start_epoch = set_up_model( num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) # Setting up activation_worker activation_worker = Activation(current_log_folder, model_name, activation_dataset, kwargs['process_size'], kwargs['save_images'], kwargs['no_cuda']) activation_worker.init(model) activation_worker.resolve_items() # With training part if kwargs['train']: logging.info('Begin training') val_value = np.zeros((epochs + 1 - start_epoch)) train_value = np.zeros((epochs - start_epoch)) # Pretraining validation step val_value[-1] = ProcessActivation._validate( val_loader, model, criterion, writer, -1, **kwargs) # Training for epoch in range(start_epoch, epochs): train_value[epoch] = ProcessActivation._train( train_loader, model, criterion, optimizer, writer, epoch, **kwargs) # Validate if epoch % validation_interval == 0: val_value[epoch] = ProcessActivation._validate( val_loader, model, criterion, writer, epoch, **kwargs) # Activation if (epoch == start_epoch) or (epoch % kwargs['process_every'] == 0) or epoch == (epochs - 1): activation_worker.add_epoch(epoch, val_value[epoch], model) if decay_lr is not None: adjust_learning_rate(lr=lr, optimizer=optimizer, epoch=epoch, decay_lr_epochs=decay_lr) best_value = checkpoint( epoch=epoch, new_value=val_value[epoch], best_value=best_value, model=model, optimizer=optimizer, log_dir=current_log_folder, checkpoint_all_epochs=checkpoint_all_epochs) # Load the best model before evaluating on the test set. logging.info( 'Loading the best model before evaluating on the test set.') kwargs["load_model"] = os.path.join(current_log_folder, 'model_best.pth.tar') model, _, _, _, _ = set_up_model(num_classes=num_classes, model_name=model_name, lr=lr, train_loader=train_loader, **kwargs) # Test test_value = ProcessActivation._test(test_loader, model, criterion, writer, epochs - 1, **kwargs) logging.info('Training completed') # Without training part else: activation_worker.add_epoch(0, 0, model) sys.exit(-1)