############################################################### # AutoTorch HPO # ------------- # # In this section, we cover how to define a searchable network architecture, convert the training function to be searchable, create the scheduler, and then launch the experiment. # - Define a Searchable Network Achitecture # # Let's define a 'dynamic' network with searchable configurations by simply adding a decorator :func:`autotorch.obj`. In this example, we only search two arguments `hidden_conv` and `hidden_fc`, which represent the hidden channels in convolutional layer and fully connected layer. More info about searchable space is available at :meth:`autotorch.space`. import autotorch as at @at.obj( hidden_conv=at.Int(6, 12), hidden_fc=at.Choice(80, 120, 160), ) class Net(nn.Module): def __init__(self, hidden_conv, hidden_fc): super().__init__() self.conv1 = nn.Conv2d(1, hidden_conv, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(hidden_conv, 16, 5) self.fc1 = nn.Linear(16 * 4 * 4, hidden_fc) self.fc2 = nn.Linear(hidden_fc, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(F.relu(self.conv1(x))) x = self.pool(F.relu(self.conv2(x)))
ax.plot_surface(X, Y, Z, cmap='plasma') ax.set_zlim(0, np.max(Z) + 2) plt.show() ############################################################### # Create Training Function # ~~~~~~~~~~~~~~~~~~~~~~~~ # # We can simply define an AutoTorch searchable function with a decorator `at.gargs`. # The `reporter` is used to communicate with AutoTorch search and scheduling algorithms. import autotorch as at @at.args( x=at.Int(0, 99), y=at.Int(0, 99), ) def toy_simulation(args, reporter): x, y = args.x, args.y reporter(accuracy=Z[y][x]) ############################################################### # Random Search # ~~~~~~~~~~~~~ # random_scheduler = at.scheduler.FIFOScheduler(toy_simulation, resource={ 'num_cpus': 1,
import torch import autotorch as at from mmcv import Config from mmcls.apis import multi_gpu_test, single_gpu_test from mmcls.datasets import build_dataloader, build_dataset from mmcls.models import build_classifier try: from mmcv.cnn import get_model_complexity_info except ImportError: raise ImportError('Please upgrade mmcv to >0.6.2') @at.obj( layer0_channels=at.Int(8,64), layer1_channels=at.Int(8,64), last_stride=at.Int(1,2), stage_blocks=at.List( at.Int(1,6), at.Int(1,6), at.Int(1,6), at.Int(1,6), ), stage_expands=at.List( at.Int(2,6), at.Int(2,6), at.Int(2,6), at.Int(2,6), ), stage_planes_ratio=at.List(
self.add_module("relu", nn.ReLU()) class GlobalAvgPool2d(nn.Module): def __init__(self): """Global average pooling over the input's spatial dimensions""" super(GlobalAvgPool2d, self).__init__() def forward(self, inputs): return nn.functional.adaptive_avg_pool2d(inputs, 1).view(inputs.size(0), -1) @at.obj( bottleneck_ratio=1, #at.Int(1, 2), initial_width=at.Int(16, 320), slope=at.Real(24, 128, log=True), quantized_param=at.Real(2.0, 3.2), network_depth=at.Int(12, 28), group_width=at.Int(8, 240), ) class GenConfg(BaseGen): def dump_config(self, config_file=None): config = configparser.ConfigParser() config['DEFAULT'] = {'bottleneck_ratio': '1'} config['net'] = {} self.group_width = self.group_width if self.group_width <= self.initial_width \ else self.initial_width self.group_width = int(self.group_width // 8 * 8) #self.initial_width = int(self.initial_width // self.group_width * self.group_width) for k, v in self.items():
name=at.Choice('auto', 'torch'), ) class myobj: def __init__(self, name): self.name = name @at.func( framework=at.Choice('mxnet', 'pytorch'), ) def myfunc(framework): return framework @at.args( a=at.Real(1e-3, 1e-2, log=True), b=at.Real(1e-3, 1e-2), c=at.Int(1, 10), d=at.Choice('a', 'b', 'c', 'd'), e=at.Bool(), f=at.List( at.Int(1, 2), at.Choice(4, 5), ), g=at.Dict( a=at.Real(0, 10), obj=myobj(), ), h=at.Choice('test', myobj()), i=myfunc(), ) def train_fn(args, reporter): a, b, c, d, e, f, g, h, i = args.a, args.b, args.c, args.d, args.e, \
import torch import autotorch as at from mmcv import Config from mmdet.models import build_detector try: from mmcv.cnn import get_model_complexity_info except ImportError: raise ImportError('Please upgrade mmcv to >0.6.2') @at.obj( block=at.Choice('BasicBlock', 'Bottleneck'), base_channels=at.Int(8, 64), stage_blocks=at.List( at.Int(1, 10), at.Int(1, 10), at.Int(1, 10), at.Int(1, 10), ), stage_planes_ratio=at.List( at.Real(1.0, 4.0), at.Real(1.0, 4.0), at.Real(1.0, 4.0), ), ) class GenConfigBackbone: def __init__(self, **kwargs): d = {}
(Choice, List, Dict). Each search space describes the set of possible values for a hyperparameter, from which the searcher will try particular values during hyperparameter optimization. AutoTorch also enables search spaces in user-defined objects using the decorator `at.obj` and user-defined functions using the decorator `at.func`. """ import autotorch as at ############################################################### # Search Space # ------------ # Simple Search Space # ~~~~~~~~~~~~~~~~~~~ # - Integer Space :class:`autotorch.space.Int` # # An integer is chosen between lower and upper value during the searcher sampling. a = at.Int(lower=0, upper=10) print(a) ############################################################### # Get default value: print(a.default) ############################################################### # Change default value, which is the first configuration that a random searcher # :class:`autotorch.searcher.RandomSearcher` will try: a = at.Int(lower=0, upper=10, default=2) print(a.default) ############################################################### # Pick a random value.