from horch.train.metrics import TrainLoss, Loss
from horch.train.metrics.classification import Accuracy
manual_seed(0)
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
Lambda(lambda x: x.expand(3, -1, -1))
])
root = '/Users/hrvvi/Code/study/pytorch/datasets'
ds_all = MNIST(root=root, train=True, download=True, transform=train_transform)
ds = train_test_split(ds_all, test_ratio=0.001, random=True)[1]
ds_train, ds_val = train_test_split(ds, test_ratio=0.5, random=True)
ds = CombineDataset(ds_train, ds_val)
train_loader = DataLoader(ds, batch_size=2, pin_memory=True, num_workers=2)
val_loader = DataLoader(ds_val, batch_size=2, pin_memory=True, num_workers=2)
cfg.relu.inplace = False
cfg.bn.affine = False
criterion = nn.CrossEntropyLoss()
tau_max, tau_min = 10, 0.1
model = Network(8, 8, steps=4, multiplier=4, stem_multiplier=1, tau=tau_max)
optimizer_arch = Adam(model.arch_parameters(),
lr=3e-4,
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
set_defaults({
'relu': {
'inplace': False,
},
'bn': {
'affine': False,
}
})
model = Network(args.init_channels, args.layers, num_classes=CIFAR_CLASSES)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.model_parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
ds_train, ds_search = train_test_split(train_data,
test_ratio=0.5,
shuffle=True,
random_state=args.seed)
train_queue = torch.utils.data.DataLoader(ds_train,
batch_size=args.batch_size,
pin_memory=True,
num_workers=2,
shuffle=True)
valid_queue = torch.utils.data.DataLoader(ds_search,
batch_size=args.batch_size,
pin_memory=True,
num_workers=2,
shuffle=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
optimizer_arch = torch.optim.Adam(model.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
print(F.softmax(model.betas_normal[2:5], dim=-1))
# model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
optimizer_arch, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
if args.epochs - epoch <= 1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
test_transform = Compose([
ToTensor(),
Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)),
])
data_home = "/Users/hrvvi/Code/study/pytorch/datasets/CIFAR10"
ds_train = CIFAR10(data_home,
train=True,
download=True,
transform=train_transform)
ds_test = CIFAR10(data_home,
train=False,
download=True,
transform=test_transform)
ds_train = train_test_split(ds_train, test_ratio=0.01)[1]
ds_test = train_test_split(ds_test, test_ratio=0.01)[1]
PC_DARTS_cifar = Genotype(normal=[('sep_conv_3x3', 1), ('skip_connect', 0),
('sep_conv_3x3', 0), ('dil_conv_3x3', 1),
('sep_conv_5x5', 0), ('sep_conv_3x3', 1),
('avg_pool_3x3', 0), ('dil_conv_3x3', 1)],
normal_concat=[2, 3, 4, 5],
reduce=[('sep_conv_5x5', 1), ('max_pool_3x3', 0),
('sep_conv_5x5', 1), ('sep_conv_5x5', 2),
('sep_conv_3x3', 0), ('sep_conv_3x3', 3),
('sep_conv_3x3', 1), ('sep_conv_3x3', 2)],
reduce_concat=[2, 3, 4, 5])
drop_path = 0.3
epochs = 600
test_transform = get_transform(cfg.Dataset.Test.transforms)
num_classes = 10
if cfg.get("hpo"):
import nni
RCV_CONFIG = nni.get_next_parameter()
for k, v in RCV_CONFIG.items():
ks = k.split(".")
override(cfg, ks, v)
ds_train = MNIST(data_home, train=True)
ds_train, ds_val = train_test_split(
ds_train,
test_ratio=cfg.Dataset.Split.val_ratio,
shuffle=cfg.Dataset.Split.get("shuffle", True),
random_state=cfg.Dataset.Split.get("seed", 42),
transform=train_transform,
test_transform=val_transform)
ds_test = MNIST(data_home, train=False, transform=test_transform)
if cfg.get("Debug") and cfg.Debug.get("subset"):
ratio = cfg.Debug.subset
ds_train = train_test_split(ds_train, ratio)[1]
ds_val = train_test_split(ds_val, ratio)[1]
ds_test = train_test_split(ds_test, ratio)[1]
use_mix = cfg.get("Mix") is not None
if use_mix:
cfg.Mix.num_classes = num_classes
ds_train = get_mix(cfg.Mix, ds_train)
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
test_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
data_home = "/Users/hrvvi/Code/study/pytorch/datasets"
ds = MNIST(data_home, train=True, download=False)
ds = train_test_split(ds, test_ratio=0.1, shuffle=True)[1]
ds_train, ds_val = train_test_split(
ds,
test_ratio=0.05,
shuffle=True,
transform=train_transform,
test_transform=test_transform,
)
ds_test = MNIST(data_home,
train=False,
download=False,
transform=test_transform)
ds_test = train_test_split(ds_test, test_ratio=0.1, shuffle=True)[1]
mul = 1
batch_size = 128
CIFAR10Policy(),
ToTensor(),
Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)),
Cutout(1, 16),
])
test_transform = Compose([
ToTensor(),
Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)),
])
# Dataset
data_home = "datasets/CIFAR10"
ds = CIFAR10(data_home, train=True, download=True)
ds = train_test_split(ds, test_ratio=0.5)[1]
ds_train, ds_val = train_test_split(
ds, test_ratio=0.02,
transform=train_transform,
test_transform=test_transform,
)
ds_test = CIFAR10(data_home, train=False, download=True, transform=test_transform)
net = LeNet5()
# net = efficientnet_b0(num_classes=10, dropout=0.3, drop_connect=0.2)
criterion = nn.CrossEntropyLoss()
optimizer = SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4, nesterov=True)
lr_scheduler = CosineAnnealingLR(optimizer, 100, eta_min=1e-3, warmup=5, warmup_eta_min=1e-3)
metrics = {
Conv2d(6, 16, kernel_size=5, norm_layer='default', activation='default'),
nn.AvgPool2d(kernel_size=2, stride=2),
Flatten(),
nn.Linear(8 * 8 * 16, 120),
nn.Linear(120, 84),
nn.Linear(84, num_classes),
)
train_transform = Compose(cfg.Dataset.Train.transforms)
val_transform = Compose(cfg.Dataset.Val.transforms)
test_transform = Compose(cfg.Dataset.Test.transforms)
data_home = cfg.Dataset.data_home
ds = MNIST(data_home, train=True, download=True)
ds = train_test_split(ds, test_ratio=0.1, random=True)[1]
ds_train, ds_val = train_test_split(
ds, test_ratio=cfg.Dataset.Split.test_ratio, random=cfg.Dataset.Split.random,
transform=train_transform,
test_transform=val_transform,
)
ds_test = MNIST(data_home, train=False, download=True)
ds_test = train_test_split(ds_test, test_ratio=0.1, random=True, test_transform=test_transform)[1]
net = eval(cfg.Model)(**cfg.get(cfg.Model))
criterion = nn.CrossEntropyLoss()
optimizer = get_optimizer(cfg.Optimizer, net)
lr_scheduler = get_lr_scheduler(cfg.LRScheduler, optimizer)
metrics = {
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
test_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
data_home = "datasets"
ds = MNIST(data_home, train=True, download=True)
ds = train_test_split(ds, test_ratio=0.1, random=True)[1]
ds_train, ds_val = train_test_split(
ds,
test_ratio=0.02,
transform=train_transform,
test_transform=test_transform,
)
ds_test = MNIST(data_home,
train=False,
download=True,
transform=test_transform)
train_loader = DataLoader(ds_train,
batch_size=128,
shuffle=True,
num_workers=0,
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
test_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
])
data_home = "/Users/hrvvi/Code/study/pytorch/datasets"
ds = MNIST(data_home, train=True, download=False)
ds = train_test_split(ds, test_ratio=0.02, shuffle=True)[1]
cv = KFold(5, True, train_transform, test_transform, random_state=0)
def fit_fn(ds_train, ds_val, verbose):
net = LeNet5()
criterion = nn.CrossEntropyLoss()
optimizer = SGD(net.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=1e-4,
nesterov=True)
# lr_scheduler = MultiStepLR(optimizer, [10, 20], gamma=0.1)
lr_scheduler = CosineAnnealingLR(optimizer,
else:
num_classes, CIFAR = 100, CIFAR100
if cfg.get("hpo"):
import nni
RCV_CONFIG = nni.get_next_parameter()
for k, v in RCV_CONFIG.items():
ks = k.split(".")
override(cfg, ks, v)
ds_train = CIFAR(data_home, train=True, transform=train_transform)
ds_test = CIFAR(data_home, train=False, transform=test_transform)
if cfg.get("Debug") and cfg.Debug.get("subset"):
ratio = cfg.Debug.subset
ds_train = train_test_split(ds_train, ratio)[1]
ds_test = train_test_split(ds_test, ratio)[1]
use_mix = cfg.get("Mix") is not None
if use_mix:
cfg.Mix.num_classes = num_classes
ds_train = get_mix(cfg.Mix, ds_train)
train_loader = get_data_loader(cfg.Dataset.Train, ds_train)
test_loader = get_data_loader(cfg.Dataset.Test, ds_test)
cfg.Model.num_classes = num_classes
model = get_model(cfg.Model, horch.models.cifar)
criterion = CrossEntropyLoss(non_sparse=use_mix,
label_smoothing=cfg.get("label_smooth"))
def main():
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = False
manual_seed(args.seed)
train_transform = Compose([
RandomCrop(32, padding=4),
RandomHorizontalFlip(),
ToTensor(),
Normalize([0.491, 0.482, 0.447], [0.247, 0.243, 0.262]),
])
ds = CIFAR10(root=args.data, train=True, download=True)
ds_train, ds_search = train_test_split(
ds, test_ratio=0.5, shuffle=True, random_state=args.seed,
transform=train_transform, test_transform=train_transform)
train_queue = DataLoader(
ds_train, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2)
valid_queue = DataLoader(
ds_search, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2)
set_defaults({
'relu': {
'inplace': False,
},
'bn': {
'affine': False,
}
})
model = Network(args.init_channels, args.layers, num_classes=CIFAR_CLASSES)
criterion = nn.CrossEntropyLoss()
optimizer_arch = Adam(
model.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
optimizer_model = SGD(
model.model_parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = CosineLR(
optimizer_model, float(args.epochs), min_lr=args.learning_rate_min)
train_metrics = {
"loss": TrainLoss(),
"acc": Accuracy(),
}
eval_metrics = {
"loss": Loss(criterion),
"acc": Accuracy(),
}
learner = DARTSLearner(model, criterion, optimizer_arch, optimizer_model, scheduler,
train_metrics=train_metrics, eval_metrics=eval_metrics,
search_loader=valid_queue, grad_clip_norm=5.0, work_dir='models')
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
print(F.softmax(model.betas_normal[2:5], dim=-1))
# training
train_acc, train_obj = train(learner, train_queue, epoch)
logging.info('train_acc %f', train_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
from horch.legacy.train.metrics import TrainLoss, Loss
from horch.legacy.train.metrics import Accuracy
manual_seed(0)
train_transform = Compose([
Pad(2),
ToTensor(),
Normalize((0.1307, ), (0.3081, )),
Lambda(lambda x: x.expand(3, -1, -1))
])
root = '/Users/hrvvi/Code/study/pytorch/datasets'
ds_all = MNIST(root=root, train=True, download=True, transform=train_transform)
ds = train_test_split(ds_all, test_ratio=0.001, shuffle=True)[1]
ds_train, ds_val = train_test_split(ds, test_ratio=0.5, shuffle=True)
ds = CombineDataset(ds_train, ds_val)
train_loader = DataLoader(ds, batch_size=2, pin_memory=True, num_workers=2)
val_loader = DataLoader(ds_val, batch_size=2, pin_memory=True, num_workers=2)
cfg.relu.inplace = False
cfg.bn.affine = False
criterion = nn.CrossEntropyLoss()
tau_max, tau_min = 10, 0.1
model = Network(8, 8, steps=4, multiplier=4, stem_multiplier=1, tau=tau_max)
optimizer_arch = Adam(model.arch_parameters(),
lr=3e-4,
train_transform = Compose(cfg.Dataset.Train.transforms)
test_transform = Compose(cfg.Dataset.Test.transforms)
data_home = cfg.Dataset.data_home
ds_train = CIFAR10(data_home,
train=True,
download=True,
transform=train_transform)
ds_test = CIFAR10(data_home,
train=False,
download=True,
transform=test_transform)
if cfg.get("Debug") and cfg.Debug.get("subset"):
ratio = cfg.Debug.subset
ds_train = train_test_split(ds_train, test_ratio=ratio, random=True)[1]
ds_test = train_test_split(ds_test, test_ratio=ratio, random=True)[1]
train_loader = get_dataloader(cfg.Dataset.Train, ds_train)
test_loader = get_dataloader(cfg.Dataset.Test, ds_test)
net = get_model(cfg, horch.models.cifar)
criterion = CrossEntropyLoss(label_smoothing=cfg.get("label_smooth"))
optimizer = get_optimizer(cfg.Optimizer, net)
lr_scheduler = get_lr_scheduler(cfg.LRScheduler, optimizer)
mix = get_mix(cfg.get("Mix"))
metrics = {
