import autotorch as at @at.obj( name=at.Choice('auto', 'torch'), ) class myobj: def __init__(self, name): self.name = name @at.func( framework=at.Choice('mxnet', 'pytorch'), ) def myfunc(framework): return framework @at.args( a=at.Real(1e-3, 1e-2, log=True), b=at.Real(1e-3, 1e-2), c=at.Int(1, 10), d=at.Choice('a', 'b', 'c', 'd'), e=at.Bool(), f=at.List( at.Int(1, 2), at.Choice(4, 5), ), g=at.Dict( a=at.Real(0, 10), obj=myobj(), ), h=at.Choice('test', myobj()),
############################################################### # AutoTorch HPO # ------------- # # In this section, we cover how to define a searchable network architecture, convert the training function to be searchable, create the scheduler, and then launch the experiment. # - Define a Searchable Network Achitecture # # Let's define a 'dynamic' network with searchable configurations by simply adding a decorator :func:`autotorch.obj`. In this example, we only search two arguments `hidden_conv` and `hidden_fc`, which represent the hidden channels in convolutional layer and fully connected layer. More info about searchable space is available at :meth:`autotorch.space`. import autotorch as at @at.obj( hidden_conv=at.Int(6, 12), hidden_fc=at.Choice(80, 120, 160), ) class Net(nn.Module): def __init__(self, hidden_conv, hidden_fc): super().__init__() self.conv1 = nn.Conv2d(1, hidden_conv, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(hidden_conv, 16, 5) self.fc1 = nn.Linear(16 * 4 * 4, hidden_fc) self.fc2 = nn.Linear(hidden_fc, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(F.relu(self.conv1(x))) x = self.pool(F.relu(self.conv2(x))) x = x.view(-1, 16 * 4 * 4)
import numpy as np import autotorch as at @at.args(lr=at.Real(1e-3, 1e-2, log=True), wd=at.Real(1e-3, 1e-2)) def train_fn(args, reporter): for e in range(10): dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2 * e)) reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd) @at.args(lr=at.Choice(1e-3, 1e-2), wd=at.Choice(1e-3, 1e-2)) def rl_train_fn(args, reporter): for e in range(10): dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2 * e)) reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd) def test_fifo_scheduler(): scheduler = at.scheduler.FIFOScheduler(train_fn, resource={ 'num_cpus': 2, 'num_gpus': 0 }, num_trials=10, reward_attr='accuracy', time_attr='epoch') scheduler.run() scheduler.join_jobs()
def train_network(args, gpu_manager, split_idx, return_dict): gpu = gpu_manager.request() print('gpu: {}, split_idx: {}'.format(gpu, split_idx)) # single gpu training only for evaluating the configurations model = encoding.models.get_model(args.model) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.wd) model.cuda(gpu) criterion.cuda(gpu) if args.amp: model, optimizer = amp.initialize(model, optimizer, opt_level='O2') # init dataloader base_size = args.base_size if args.base_size is not None else int( 1.0 * args.crop_size / 0.875) transform_train, _ = get_transform(args.dataset, args.base_size, args.crop_size) total_set = encoding.datasets.get_dataset('imagenet', root=args.data_dir, transform=transform_train, train=True, download=True) trainset, valset = subsample_dataset(total_set, args.nfolds, split_idx, args.reduced_size) train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, drop_last=True, pin_memory=True) # lr scheduler lr_scheduler = LR_Scheduler('cos', base_lr=args.lr, num_epochs=args.epochs, iters_per_epoch=len(train_loader), quiet=True) # write results into config file def train(epoch): model.train() top1 = AverageMeter() for batch_idx, (data, target) in enumerate(train_loader): lr_scheduler(optimizer, batch_idx, epoch, 0) data, target = data.cuda(gpu), target.cuda(gpu) optimizer.zero_grad() output = model(data) loss = criterion(output, target) if args.amp: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() optimizer.step() def validate(auto_policy): model.eval() top1 = AverageMeter() _, transform_val = get_transform(args.dataset, args.base_size, args.crop_size) if auto_policy is not None: transform_val.transforms.insert(0, Augmentation(auto_policy)) valset.transform = transform_val val_loader = torch.utils.data.DataLoader(valset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) for batch_idx, (data, target) in enumerate(val_loader): data, target = data.cuda(gpu), target.cuda(gpu) with torch.no_grad(): output = model(data) acc1 = accuracy(output, target, topk=(1, )) top1.update(acc1[0], data.size(0)) return top1.avg for epoch in tqdm(range(0, args.epochs)): train(epoch) #acc = validate(None) #print('baseline accuracy: {}'.format(acc)) ops = list(augment_dict.keys()) sub_policy = at.List( at.List(at.Choice(*ops), at.Real(0, 1), at.Real(0, 1)), at.List(at.Choice(*ops), at.Real(0, 1), at.Real(0, 1)), ) searcher = at.searcher.RandomSearcher(sub_policy.cs) # avoid same defaults config = searcher.get_config() for i in range(args.num_trials): config = searcher.get_config() auto_policy = sub_policy.sample(**config) acc = validate(auto_policy) searcher.update(config, acc.item(), done=True) gpu_manager.release(gpu) topK_cfgs = searcher.get_topK_configs(5) policy = [sub_policy.sample(**cfg) for cfg in topK_cfgs] return_dict[split_idx] = policy
import numpy as np import autotorch as at from nose.plugins.attrib import attr @at.args( lr=at.Real(1e-3, 1e-2, log=True), wd=at.Real(1e-3, 1e-2)) def train_fn(args, reporter): for e in range(10): dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2*e)) reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd) @at.args( lr=at.Choice(1e-3, 1e-2), wd=at.Choice(1e-3, 1e-2)) def rl_train_fn(args, reporter): for e in range(10): dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2*e)) reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd) def test_fifo_scheduler(): scheduler = at.scheduler.FIFOScheduler(train_fn, resource={'num_cpus': 2, 'num_gpus': 0}, num_trials=10, reward_attr='accuracy', time_attr='epoch') scheduler.run() scheduler.join_jobs() def test_hyperband_scheduler():
from mmdet.models import build_detector import torch import autotorch as at from mmcv import Config from mmdet.models import build_detector try: from mmcv.cnn import get_model_complexity_info except ImportError: raise ImportError('Please upgrade mmcv to >0.6.2') @at.obj( block=at.Choice('BasicBlock', 'Bottleneck'), base_channels=at.Int(8, 64), stage_blocks=at.List( at.Int(1, 10), at.Int(1, 10), at.Int(1, 10), at.Int(1, 10), ), stage_planes_ratio=at.List( at.Real(1.0, 4.0), at.Real(1.0, 4.0), at.Real(1.0, 4.0), ), ) class GenConfigBackbone: def __init__(self, **kwargs):
b = at.Real(lower=1e-4, upper=1e-2) print(b) ############################################################### # Real space in log scale: c = at.Real(lower=1e-4, upper=1e-2, log=True) print(c) ############################################################### # - Choice Space :class:`autotorch.space.Choice` # # Choice Space chooses one value from all the possible values during the searcher sampling. d = at.Choice('Monday', 'Tuesday', 'Wednesday') print(d) ############################################################### # Nested Search Space # ~~~~~~~~~~~~~~~~~~~ # - Choice Space :class:`autotorch.space.Choice` # # Choice Space can also be used as a nested search space. # For an example, see NestedExampleObj_. # # - List Space :class:`autotorch.space.List` # # List Space returns a list of sampled results. # In this example, the first element of the list is a Int Space sampled # from 0 to 3, and the second element is a Choice Space sampled from the choices of `'alpha'` and `'beta'`.