import autotorch as at
@at.obj(
name=at.Choice('auto', 'torch'), )
class myobj:
def __init__(self, name):
self.name = name
@at.func(
framework=at.Choice('mxnet', 'pytorch'), )
def myfunc(framework):
return framework
@at.args(
a=at.Real(1e-3, 1e-2, log=True),
b=at.Real(1e-3, 1e-2),
c=at.Int(1, 10),
d=at.Choice('a', 'b', 'c', 'd'),
e=at.Bool(),
f=at.List(
at.Int(1, 2),
at.Choice(4, 5),
),
g=at.Dict(
a=at.Real(0, 10),
obj=myobj(),
),
h=at.Choice('test', myobj()),
###############################################################
# AutoTorch HPO
# -------------
#
# In this section, we cover how to define a searchable network architecture, convert the training function to be searchable, create the scheduler, and then launch the experiment.
# - Define a Searchable Network Achitecture
#
# Let's define a 'dynamic' network with searchable configurations by simply adding a decorator :func:`autotorch.obj`. In this example, we only search two arguments `hidden_conv` and `hidden_fc`, which represent the hidden channels in convolutional layer and fully connected layer. More info about searchable space is available at :meth:`autotorch.space`.
import autotorch as at
@at.obj(
hidden_conv=at.Int(6, 12),
hidden_fc=at.Choice(80, 120, 160),
)
class Net(nn.Module):
def __init__(self, hidden_conv, hidden_fc):
super().__init__()
self.conv1 = nn.Conv2d(1, hidden_conv, 5)
self.pool = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(hidden_conv, 16, 5)
self.fc1 = nn.Linear(16 * 4 * 4, hidden_fc)
self.fc2 = nn.Linear(hidden_fc, 84)
self.fc3 = nn.Linear(84, 10)
def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
x = self.pool(F.relu(self.conv2(x)))
x = x.view(-1, 16 * 4 * 4)
import numpy as np
import autotorch as at
@at.args(lr=at.Real(1e-3, 1e-2, log=True), wd=at.Real(1e-3, 1e-2))
def train_fn(args, reporter):
for e in range(10):
dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2 * e))
reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd)
@at.args(lr=at.Choice(1e-3, 1e-2), wd=at.Choice(1e-3, 1e-2))
def rl_train_fn(args, reporter):
for e in range(10):
dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2 * e))
reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd)
def test_fifo_scheduler():
scheduler = at.scheduler.FIFOScheduler(train_fn,
resource={
'num_cpus': 2,
'num_gpus': 0
},
num_trials=10,
reward_attr='accuracy',
time_attr='epoch')
scheduler.run()
scheduler.join_jobs()
def train_network(args, gpu_manager, split_idx, return_dict):
gpu = gpu_manager.request()
print('gpu: {}, split_idx: {}'.format(gpu, split_idx))
# single gpu training only for evaluating the configurations
model = encoding.models.get_model(args.model)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.wd)
model.cuda(gpu)
criterion.cuda(gpu)
if args.amp:
model, optimizer = amp.initialize(model, optimizer, opt_level='O2')
# init dataloader
base_size = args.base_size if args.base_size is not None else int(
1.0 * args.crop_size / 0.875)
transform_train, _ = get_transform(args.dataset, args.base_size,
args.crop_size)
total_set = encoding.datasets.get_dataset('imagenet',
root=args.data_dir,
transform=transform_train,
train=True,
download=True)
trainset, valset = subsample_dataset(total_set, args.nfolds, split_idx,
args.reduced_size)
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
drop_last=True,
pin_memory=True)
# lr scheduler
lr_scheduler = LR_Scheduler('cos',
base_lr=args.lr,
num_epochs=args.epochs,
iters_per_epoch=len(train_loader),
quiet=True)
# write results into config file
def train(epoch):
model.train()
top1 = AverageMeter()
for batch_idx, (data, target) in enumerate(train_loader):
lr_scheduler(optimizer, batch_idx, epoch, 0)
data, target = data.cuda(gpu), target.cuda(gpu)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
if args.amp:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
def validate(auto_policy):
model.eval()
top1 = AverageMeter()
_, transform_val = get_transform(args.dataset, args.base_size,
args.crop_size)
if auto_policy is not None:
transform_val.transforms.insert(0, Augmentation(auto_policy))
valset.transform = transform_val
val_loader = torch.utils.data.DataLoader(valset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
for batch_idx, (data, target) in enumerate(val_loader):
data, target = data.cuda(gpu), target.cuda(gpu)
with torch.no_grad():
output = model(data)
acc1 = accuracy(output, target, topk=(1, ))
top1.update(acc1[0], data.size(0))
return top1.avg
for epoch in tqdm(range(0, args.epochs)):
train(epoch)
#acc = validate(None)
#print('baseline accuracy: {}'.format(acc))
ops = list(augment_dict.keys())
sub_policy = at.List(
at.List(at.Choice(*ops), at.Real(0, 1), at.Real(0, 1)),
at.List(at.Choice(*ops), at.Real(0, 1), at.Real(0, 1)),
)
searcher = at.searcher.RandomSearcher(sub_policy.cs)
# avoid same defaults
config = searcher.get_config()
for i in range(args.num_trials):
config = searcher.get_config()
auto_policy = sub_policy.sample(**config)
acc = validate(auto_policy)
searcher.update(config, acc.item(), done=True)
gpu_manager.release(gpu)
topK_cfgs = searcher.get_topK_configs(5)
policy = [sub_policy.sample(**cfg) for cfg in topK_cfgs]
return_dict[split_idx] = policy
import numpy as np
import autotorch as at
from nose.plugins.attrib import attr
@at.args(
lr=at.Real(1e-3, 1e-2, log=True),
wd=at.Real(1e-3, 1e-2))
def train_fn(args, reporter):
for e in range(10):
dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2*e))
reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd)
@at.args(
lr=at.Choice(1e-3, 1e-2),
wd=at.Choice(1e-3, 1e-2))
def rl_train_fn(args, reporter):
for e in range(10):
dummy_accuracy = 1 - np.power(1.8, -np.random.uniform(e, 2*e))
reporter(epoch=e, accuracy=dummy_accuracy, lr=args.lr, wd=args.wd)
def test_fifo_scheduler():
scheduler = at.scheduler.FIFOScheduler(train_fn,
resource={'num_cpus': 2, 'num_gpus': 0},
num_trials=10,
reward_attr='accuracy',
time_attr='epoch')
scheduler.run()
scheduler.join_jobs()
def test_hyperband_scheduler():
from mmdet.models import build_detector
import torch
import autotorch as at
from mmcv import Config
from mmdet.models import build_detector
try:
from mmcv.cnn import get_model_complexity_info
except ImportError:
raise ImportError('Please upgrade mmcv to >0.6.2')
@at.obj(
block=at.Choice('BasicBlock', 'Bottleneck'),
base_channels=at.Int(8, 64),
stage_blocks=at.List(
at.Int(1, 10),
at.Int(1, 10),
at.Int(1, 10),
at.Int(1, 10),
),
stage_planes_ratio=at.List(
at.Real(1.0, 4.0),
at.Real(1.0, 4.0),
at.Real(1.0, 4.0),
),
)
class GenConfigBackbone:
def __init__(self, **kwargs):
b = at.Real(lower=1e-4, upper=1e-2)
print(b)
###############################################################
# Real space in log scale:
c = at.Real(lower=1e-4, upper=1e-2, log=True)
print(c)
###############################################################
# - Choice Space :class:`autotorch.space.Choice`
#
# Choice Space chooses one value from all the possible values during the searcher sampling.
d = at.Choice('Monday', 'Tuesday', 'Wednesday')
print(d)
###############################################################
# Nested Search Space
# ~~~~~~~~~~~~~~~~~~~
# - Choice Space :class:`autotorch.space.Choice`
#
# Choice Space can also be used as a nested search space.
# For an example, see NestedExampleObj_.
#
# - List Space :class:`autotorch.space.List`
#
# List Space returns a list of sampled results.
# In this example, the first element of the list is a Int Space sampled
# from 0 to 3, and the second element is a Choice Space sampled from the choices of `'alpha'` and `'beta'`.