parser.add_argument('-c', '--config', help='config file')
parser.add_argument('-r', '--resume', help='resume from checkpoints')
args = parser.parse_args()
print(read_text(args.config))
cfg = get_config(args.config)
dataset = cfg.Dataset.type
assert dataset == 'MNIST'
fp16 = cfg.get("fp16", False)
if fp16:
assert cfg.device == 'gpu'
update_defaults(cfg.get("Global", {}))
manual_seed(cfg.seed)
data_home = cfg.Dataset.data_home
train_transform = get_transform(cfg.Dataset.Train.transforms)
val_transform = get_transform(cfg.Dataset.Val.transforms)
test_transform = get_transform(cfg.Dataset.Test.transforms)
num_classes = 10
if cfg.get("hpo"):
import nni
RCV_CONFIG = nni.get_next_parameter()
for k, v in RCV_CONFIG.items():
ks = k.split(".")
override(cfg, ks, v)
import torch.nn as nn
from torch.optim import SGD, Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor, Normalize, Compose, Pad, Lambda
from horch.config import cfg
from horch.datasets import train_test_split, CombineDataset
from horch.nas.darts.model_search_gdas import Network
from horch.nas.darts.trainer import DARTSTrainer
from horch.train import manual_seed
from horch.train.metrics import TrainLoss, Loss
from horch.train.metrics.classification import Accuracy
manual_seed(0)
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
Lambda(lambda x: x.expand(3, -1, -1))
])
root = '/Users/hrvvi/Code/study/pytorch/datasets'
ds_all = MNIST(root=root, train=True, download=True, transform=train_transform)
ds = train_test_split(ds_all, test_ratio=0.001, random=True)[1]
ds_train, ds_val = train_test_split(ds, test_ratio=0.5, random=True)
ds = CombineDataset(ds_train, ds_val)
def main():
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = False
manual_seed(args.seed)
train_transform = Compose([
RandomCrop(32, padding=4),
RandomHorizontalFlip(),
ToTensor(),
Normalize([0.491, 0.482, 0.447], [0.247, 0.243, 0.262]),
])
ds = CIFAR10(root=args.data, train=True, download=True)
ds_train, ds_search = train_test_split(
ds, test_ratio=0.5, shuffle=True, random_state=args.seed,
transform=train_transform, test_transform=train_transform)
train_queue = DataLoader(
ds_train, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2)
valid_queue = DataLoader(
ds_search, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2)
set_defaults({
'relu': {
'inplace': False,
},
'bn': {
'affine': False,
}
})
model = Network(args.init_channels, args.layers, num_classes=CIFAR_CLASSES)
criterion = nn.CrossEntropyLoss()
optimizer_arch = Adam(
model.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
optimizer_model = SGD(
model.model_parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = CosineLR(
optimizer_model, float(args.epochs), min_lr=args.learning_rate_min)
train_metrics = {
"loss": TrainLoss(),
"acc": Accuracy(),
}
eval_metrics = {
"loss": Loss(criterion),
"acc": Accuracy(),
}
learner = DARTSLearner(model, criterion, optimizer_arch, optimizer_model, scheduler,
train_metrics=train_metrics, eval_metrics=eval_metrics,
search_loader=valid_queue, grad_clip_norm=5.0, work_dir='models')
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
print(F.softmax(model.betas_normal[2:5], dim=-1))
# training
train_acc, train_obj = train(learner, train_queue, epoch)
logging.info('train_acc %f', train_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
