def system_check(): import traceback try: import torch from olympus.models import Model from olympus.optimizers import Optimizer batch = torch.randn((32, 3, 64, 64)).cuda() model = Model('resnet18', input_size=(3, 64, 64), output_size=(10, )).cuda() model.init() optimizer = Optimizer('sgd', params=model.parameters()) optimizer.init(**optimizer.defaults) optimizer.zero_grad() loss = model(batch).sum() optimizer.backward(loss) optimizer.step() return True except: error(traceback.format_exc()) return False
def test_optimizer_factory_init(): # set init using its name model: Model = new_model() optim = Optimizer('sgd', params=model.parameters()) assert optim.get_space() == SGD.get_space() optim.init(**SGD.defaults())
def test_build_optimizer(optimizer): model = new_model() optimizer = Optimizer( optimizer, params=model.parameters() ) optimizer.init(**optimizer.defaults) optimizer_work(optimizer, model)
def make_detection_task(client=None): dataset = SplitDataset(Dataset('test_pennfudan', path=f'{base}/data'), split_method='original') loader = DataLoader(dataset, sampler_seed=0, batch_size=1) input_size, target_size = loader.get_shapes() model = Model('fasterrcnn_resnet18_fpn', input_size=input_size, output_size=dataset.dataset.dataset.num_classes, weight_init='glorot_uniform') optimizer = Optimizer('sgd', lr=0.01, momentum=0.99, weight_decay=1e-3) lr_schedule = LRSchedule('exponential', gamma=0.97) main_task = ObjectDetection( detector=model, optimizer=optimizer, lr_scheduler=lr_schedule, dataloader=loader.train(), device=device, criterion=reduce_loss, storage=StateStorage(folder=f'{base}/detection_short'), logger=client) return main_task
def make_task(): model = Model('logreg', input_size=(1, 28, 28), output_size=(10, )) optimizer = Optimizer('sgd') lr_schedule = LRSchedule('exponential') data = Dataset('test-mnist', path=f'{base}/data') splits = SplitDataset(data, split_method='original') loader = DataLoader(splits, sampler_seed=1, batch_size=32) main_task = Classification( classifier=model, optimizer=optimizer, lr_scheduler=lr_schedule, dataloader=loader.train(), device=device, storage=StateStorage(folder=f'{base}/hpo_simple')) main_task.metrics.append( Accuracy(name='validation', loader=loader.valid(batch_size=64))) return main_task
def detection_baseline(model, weight_init, optimizer, lr_scheduler, dataset, batch_size, device, split_method='original', sampler_seed=0, model_seed=0, storage=None, half=False, hpo_done=False, logger=None, **config): dataset = SplitDataset( Dataset(dataset, path=f'{base}/data'), split_method=split_method ) loader = DataLoader( dataset, sampler_seed=sampler_seed, batch_size=batch_size ) input_size, target_size = loader.get_shapes() init = Initializer( weight_init, seed=model_seed, gain=1.0 ) model = Model( model, input_size=input_size, output_size=dataset.dataset.dataset.num_classes, weight_init=init, half=half) optimizer = Optimizer(optimizer, half=half) lr_schedule = LRSchedule(lr_scheduler) train, valid, test = loader.get_loaders(hpo_done=hpo_done) main_task = ObjectDetection( detector=model, optimizer=optimizer, lr_scheduler=lr_schedule, dataloader=train, device=device, storage=storage, criterion=reduce_loss) name = 'validation' if hpo_done: name = 'test' main_task.metrics.append( Loss(name=name, loader=test) ) return main_task
def finance_baseline(tickers, start, end, optimizer, batch_size, device, window=70, sampler_seed=0, hpo_done=False): dataset = Dataset('stockmarket', path=f'{base}/data', tickers=tickers, start_date=start, end_date=end) dataset = WindowedDataset(dataset, window=window, transforms=lambda x: x.transpose(1, 0), overlaps=True) dataset = SplitDataset(dataset, split_method='original') loader = DataLoader(dataset, sampler_seed=sampler_seed, batch_size=batch_size) model = Model('MinVarianceReturnMomentEstimator', weight_init='noinit', input_size=(len(tickers), window), lags=2).to(device=device) optimizer = Optimizer(optimizer) train, valid, test = loader.get_loaders(hpo_done=hpo_done) main_task = Finance(model=model, optimizer=optimizer, oracle=oracle, dataset=train, device=device, criterion=SharpeRatioCriterion()) name = 'validation' if hpo_done: name = 'test' main_task.metrics.append(Loss(name=name, loader=test)) main_task.metrics.append(Loss(name='train', loader=train)) return main_task
def a2c_baseline(env_name, parallel_sim, weight_init, model, model_seed, optimizer, lr_scheduler, num_steps, half, device, storage, **config): def to_nchw(states): return states.permute(0, 3, 1, 2) env = Environment( env_name, parallel_env=parallel_sim, transforms=to_nchw ) init = Initializer( weight_init, seed=model_seed, gain=1.0 ) model = Model( model, input_size=env.input_size, output_size=env.target_size[0], weight_init=init, half=half) loader = RLDataLoader( env, replay=simple_replay_vector(num_steps=num_steps), actor=model.act, critic=model.critic ) optimizer = Optimizer(optimizer, half=half) lr_schedule = LRSchedule(lr_scheduler) task = A2C( model=model, optimizer=optimizer, dataloader=loader.train(), lr_scheduler=lr_schedule, device=device, storage=storage ) return task
from olympus.models import Model from olympus.observers import ObserverList, ProgressView, Speed from olympus.utils import fetch_device, option epochs = 2 device = fetch_device() base = option('base_path', '/tmp/olympus') # Model model = Model('resnet18', input_size=(1, 28, 28), output_size=(10, )) # Optimizer optimizer = Optimizer('sgd', params=model.parameters(), weight_decay=0.001, lr=1e-5, momentum=1e-5) # Schedule lr_schedule = LRSchedule('exponential', optimizer=optimizer, gamma=0.99) data = Dataset('fake_mnist', path=f'{base}/data') splits = SplitDataset(data, split_method='original') # Dataloader loader = DataLoader(splits, sampler_seed=1, batch_size=32) # event handler event_handler = ObserverList()
return [base_lr * self.gamma ** self.last_epoch for base_lr in self.base_lrs] @staticmethod def get_space(): return {'gamma': 'loguniform(0.97, 1)'} if __name__ == '__main__': model = Model( 'logreg', input_size=(290,), output_size=(10,) ) optimizer = Optimizer('sgd', params=model.parameters()) # If you use an hyper parameter optimizer, it will generate this for you optimizer.init(lr=1e-4, momentum=0.02, weight_decay=1e-3) schedule = LRSchedule(schedule=MyExponentialLR) schedule.init(optimizer=optimizer, gamma=0.97) optimizer.zero_grad() input = torch.randn((10, 290)) output = model(input) loss = output.sum() loss.backward() optimizer.step()
def setup(): model = Model('logreg', input_size=(28, ), output_size=(10, )) optimizer = Optimizer('sgd', params=model.parameters()) optimizer.init(**optimizer.defaults) return model, optimizer
def segmentation_baseline(model, initializer, optimizer, dataset, batch_size, device, split_method='original', sampler_seed=0, init_seed=0, global_seed=0, storage=None, half=False, hpo_done=False, data_path='/tmp/olympus', validate=True, hyper_parameters=None, uri_metric=None, valid_batch_size=None, **config): set_seeds(global_seed) # dataset size: 2913 dataset = SplitDataset( Dataset(dataset, path=option('data.path', data_path), cache=torch.device('cpu')), split_method=split_method, ) loader = DataLoader( dataset, sampler_seed=sampler_seed, batch_size=batch_size, valid_batch_size=valid_batch_size, pin_memory=True, num_workers=0, ) input_size, target_size = loader.get_shapes() init = Initializer(initializer, seed=init_seed, **get_parameters('initializer', hyper_parameters)) model = Model(model, input_size=input_size, output_size=target_size[0], weight_init=init, half=half) optimizer = Optimizer(optimizer, half=half, **get_parameters('optimizer', hyper_parameters)) lr_schedule = LRSchedule('none', **get_parameters('schedule', hyper_parameters)) train, valid, test = loader.get_loaders(hpo_done=hpo_done) additional_metrics = [] if validate and valid: additional_metrics.append(MeanIoU(name='validation', loader=valid)) if validate and test: additional_metrics.append(MeanIoU(name='test', loader=test)) def get_label_counts(dataloader): cumulative_counts = {} print('get_label_counts(): ', end='') for i, (_, labels) in enumerate(dataloader, 1): if labels.device.type == 'cuda': labels = labels.cpu() unique, counts = np.unique(labels.numpy(), return_counts=True) for u, c in zip(unique, counts): if u not in cumulative_counts: cumulative_counts[u] = 0 cumulative_counts[u] += c if i % (len(dataloader) // 10) == 0: print('{}%... '.format(100 * i // len(dataloader)), end='') print() return cumulative_counts def get_criterion_weight(counts, ignore_index=255): counts = counts.copy() if ignore_index in counts: del counts[ignore_index] total_count = sum([counts[unique] for unique in sorted(counts)]) weight = np.array( [total_count / counts[unique] for unique in sorted(counts)], dtype=np.float32) weight /= weight.size return weight nclasses = 21 counts = get_label_counts(train) weight = get_criterion_weight(counts) weight = torch.tensor(weight) if half: weight = weight.half() criterion = nn.CrossEntropyLoss(weight=weight, ignore_index=255) main_task = Segmentation(model, optimizer, lr_schedule, train, criterion, nclasses, device=device, storage=storage, metrics=additional_metrics) return main_task
if args.caching: args.caching = device dataset = SplitDataset(Dataset(args.dataset, cache=args.caching, transform=False), split_method='original') loaders = DataLoader(dataset, batch_size=args.batch_size, sampler_seed=0) input_size, target_size = loaders.get_shapes() model = Model(args.model, input_size=input_size, output_size=target_size[0]).init() optimizer = Optimizer('sgd', params=model.parameters(), lr=0.01, momentum=0.9, weight_decay=0.001) criterion = CrossEntropyLoss() model = model.to(device=device) train_loader = loaders.train() def epoch(): for batch in train_loader: x, *_, y = batch output = model(x[0].to(device=device)) loss = criterion(output, y.to(device=device))
def generate(self, image, min_confidence, lr=0.7, max_iter=500, round_trip=False): """Generate an adversarial example that should be misclassified as target_class Parameters ---------- image: Union[Tensor, List[Image]] list of images to tamper with min_confidence: float Confidence we need to reach before stopping lr: float learning rate for the optimizer max_iter: int Maximal number of iteration round_trip: bool when enabled the tensor is periodically converted to image and back to tensor """ self.model.eval() target_confidence = 0 target = self.target_class original_image, batch = self.to_batch(image) for i in range(max_iter): if target_confidence > min_confidence: break batch.requires_grad = True optimizer = Optimizer('sgd', params=[batch], lr=lr, momentum=0, weight_decay=0) probabilities = F.softmax(self.model(batch), dim=1) class_predicted = torch.argmax(probabilities) prediction_confidence = probabilities[0, class_predicted] target_confidence = probabilities[0, target] self.stats.update(class_predicted, prediction_confidence, target_confidence, probabilities) debug( f'{i:4d} Predicted {class_predicted} with {prediction_confidence:.4f},' f'our target: {target} has {target_confidence.item():.4f}') self.model.zero_grad() optimizer.backward(-1 * target_confidence) optimizer.step() if round_trip: batch = self.preprocessor(self.postprocessor(batch)) noises = self.get_noise(batch, original_image) return batch, noises
def classification_baseline(model, initializer, optimizer, schedule, dataset, batch_size, device, split_method='original', sampler_seed=0, init_seed=0, transform_seed=0, global_seed=0, transform=True, storage=None, half=False, hpo_done=False, data_path='/tmp/olympus', validate=True, hyper_parameters=None, uri_metric=None, valid_batch_size=None, cache=None, **config): set_seeds(global_seed) dataset = SplitDataset(Dataset(dataset, path=option('data.path', data_path), transform=transform, transform_seed=transform_seed, cache=cache), split_method=split_method) loader = DataLoader(dataset, sampler_seed=sampler_seed, batch_size=batch_size, valid_batch_size=valid_batch_size) input_size, target_size = loader.get_shapes() init = Initializer(initializer, seed=init_seed, **get_parameters('initializer', hyper_parameters)) model = Model(model, input_size=input_size, output_size=target_size[0], weight_init=init, half=half) optimizer = Optimizer(optimizer, half=half, **get_parameters('optimizer', hyper_parameters)) lr_schedule = LRSchedule(schedule, **get_parameters('schedule', hyper_parameters)) train, valid, test = loader.get_loaders(hpo_done=hpo_done) additional_metrics = [] if validate and valid: additional_metrics.append(Accuracy(name='validation', loader=valid)) if validate and test: additional_metrics.append(Accuracy(name='test', loader=test)) main_task = Classification(classifier=model, optimizer=optimizer, lr_scheduler=lr_schedule, dataloader=train, device=device, storage=storage, metrics=additional_metrics) return main_task
def test_optimizer_new_factory_init_fixed(): # set HP in the constructor model: Model = new_model() optim = Optimizer('sgd', params=model.parameters(), **SGD.defaults()) optimizer_work(optim, model)