from horch.train.metrics import TrainLoss, Loss from horch.train.metrics.classification import Accuracy manual_seed(0) train_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), Lambda(lambda x: x.expand(3, -1, -1)) ]) root = '/Users/hrvvi/Code/study/pytorch/datasets' ds_all = MNIST(root=root, train=True, download=True, transform=train_transform) ds = train_test_split(ds_all, test_ratio=0.001, random=True)[1] ds_train, ds_val = train_test_split(ds, test_ratio=0.5, random=True) ds = CombineDataset(ds_train, ds_val) train_loader = DataLoader(ds, batch_size=2, pin_memory=True, num_workers=2) val_loader = DataLoader(ds_val, batch_size=2, pin_memory=True, num_workers=2) cfg.relu.inplace = False cfg.bn.affine = False criterion = nn.CrossEntropyLoss() tau_max, tau_min = 10, 0.1 model = Network(8, 8, steps=4, multiplier=4, stem_multiplier=1, tau=tau_max) optimizer_arch = Adam(model.arch_parameters(), lr=3e-4,
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() set_defaults({ 'relu': { 'inplace': False, }, 'bn': { 'affine': False, } }) model = Network(args.init_channels, args.layers, num_classes=CIFAR_CLASSES) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD(model.model_parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.set == 'cifar100': train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) ds_train, ds_search = train_test_split(train_data, test_ratio=0.5, shuffle=True, random_state=args.seed) train_queue = torch.utils.data.DataLoader(ds_train, batch_size=args.batch_size, pin_memory=True, num_workers=2, shuffle=True) valid_queue = torch.utils.data.DataLoader(ds_search, batch_size=args.batch_size, pin_memory=True, num_workers=2, shuffle=True) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) optimizer_arch = torch.optim.Adam(model.arch_parameters(), lr=args.arch_learning_rate, betas=(0.5, 0.999), weight_decay=args.arch_weight_decay) for epoch in range(args.epochs): scheduler.step() lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) genotype = model.genotype() logging.info('genotype = %s', genotype) print(F.softmax(model.alphas_normal, dim=-1)) print(F.softmax(model.alphas_reduce, dim=-1)) print(F.softmax(model.betas_normal[2:5], dim=-1)) # model.drop_path_prob = args.drop_path_prob * epoch / args.epochs # training train_acc, train_obj = train(train_queue, valid_queue, model, optimizer_arch, criterion, optimizer, lr, epoch) logging.info('train_acc %f', train_acc) # validation if args.epochs - epoch <= 1: valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
test_transform = Compose([ ToTensor(), Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)), ]) data_home = "/Users/hrvvi/Code/study/pytorch/datasets/CIFAR10" ds_train = CIFAR10(data_home, train=True, download=True, transform=train_transform) ds_test = CIFAR10(data_home, train=False, download=True, transform=test_transform) ds_train = train_test_split(ds_train, test_ratio=0.01)[1] ds_test = train_test_split(ds_test, test_ratio=0.01)[1] PC_DARTS_cifar = Genotype(normal=[('sep_conv_3x3', 1), ('skip_connect', 0), ('sep_conv_3x3', 0), ('dil_conv_3x3', 1), ('sep_conv_5x5', 0), ('sep_conv_3x3', 1), ('avg_pool_3x3', 0), ('dil_conv_3x3', 1)], normal_concat=[2, 3, 4, 5], reduce=[('sep_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_5x5', 1), ('sep_conv_5x5', 2), ('sep_conv_3x3', 0), ('sep_conv_3x3', 3), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2)], reduce_concat=[2, 3, 4, 5]) drop_path = 0.3 epochs = 600
test_transform = get_transform(cfg.Dataset.Test.transforms) num_classes = 10 if cfg.get("hpo"): import nni RCV_CONFIG = nni.get_next_parameter() for k, v in RCV_CONFIG.items(): ks = k.split(".") override(cfg, ks, v) ds_train = MNIST(data_home, train=True) ds_train, ds_val = train_test_split( ds_train, test_ratio=cfg.Dataset.Split.val_ratio, shuffle=cfg.Dataset.Split.get("shuffle", True), random_state=cfg.Dataset.Split.get("seed", 42), transform=train_transform, test_transform=val_transform) ds_test = MNIST(data_home, train=False, transform=test_transform) if cfg.get("Debug") and cfg.Debug.get("subset"): ratio = cfg.Debug.subset ds_train = train_test_split(ds_train, ratio)[1] ds_val = train_test_split(ds_val, ratio)[1] ds_test = train_test_split(ds_test, ratio)[1] use_mix = cfg.get("Mix") is not None if use_mix: cfg.Mix.num_classes = num_classes ds_train = get_mix(cfg.Mix, ds_train)
train_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) test_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) data_home = "/Users/hrvvi/Code/study/pytorch/datasets" ds = MNIST(data_home, train=True, download=False) ds = train_test_split(ds, test_ratio=0.1, shuffle=True)[1] ds_train, ds_val = train_test_split( ds, test_ratio=0.05, shuffle=True, transform=train_transform, test_transform=test_transform, ) ds_test = MNIST(data_home, train=False, download=False, transform=test_transform) ds_test = train_test_split(ds_test, test_ratio=0.1, shuffle=True)[1] mul = 1 batch_size = 128
CIFAR10Policy(), ToTensor(), Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)), Cutout(1, 16), ]) test_transform = Compose([ ToTensor(), Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)), ]) # Dataset data_home = "datasets/CIFAR10" ds = CIFAR10(data_home, train=True, download=True) ds = train_test_split(ds, test_ratio=0.5)[1] ds_train, ds_val = train_test_split( ds, test_ratio=0.02, transform=train_transform, test_transform=test_transform, ) ds_test = CIFAR10(data_home, train=False, download=True, transform=test_transform) net = LeNet5() # net = efficientnet_b0(num_classes=10, dropout=0.3, drop_connect=0.2) criterion = nn.CrossEntropyLoss() optimizer = SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4, nesterov=True) lr_scheduler = CosineAnnealingLR(optimizer, 100, eta_min=1e-3, warmup=5, warmup_eta_min=1e-3) metrics = {
Conv2d(6, 16, kernel_size=5, norm_layer='default', activation='default'), nn.AvgPool2d(kernel_size=2, stride=2), Flatten(), nn.Linear(8 * 8 * 16, 120), nn.Linear(120, 84), nn.Linear(84, num_classes), ) train_transform = Compose(cfg.Dataset.Train.transforms) val_transform = Compose(cfg.Dataset.Val.transforms) test_transform = Compose(cfg.Dataset.Test.transforms) data_home = cfg.Dataset.data_home ds = MNIST(data_home, train=True, download=True) ds = train_test_split(ds, test_ratio=0.1, random=True)[1] ds_train, ds_val = train_test_split( ds, test_ratio=cfg.Dataset.Split.test_ratio, random=cfg.Dataset.Split.random, transform=train_transform, test_transform=val_transform, ) ds_test = MNIST(data_home, train=False, download=True) ds_test = train_test_split(ds_test, test_ratio=0.1, random=True, test_transform=test_transform)[1] net = eval(cfg.Model)(**cfg.get(cfg.Model)) criterion = nn.CrossEntropyLoss() optimizer = get_optimizer(cfg.Optimizer, net) lr_scheduler = get_lr_scheduler(cfg.LRScheduler, optimizer) metrics = {
train_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) test_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) data_home = "datasets" ds = MNIST(data_home, train=True, download=True) ds = train_test_split(ds, test_ratio=0.1, random=True)[1] ds_train, ds_val = train_test_split( ds, test_ratio=0.02, transform=train_transform, test_transform=test_transform, ) ds_test = MNIST(data_home, train=False, download=True, transform=test_transform) train_loader = DataLoader(ds_train, batch_size=128, shuffle=True, num_workers=0,
train_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) test_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), ]) data_home = "/Users/hrvvi/Code/study/pytorch/datasets" ds = MNIST(data_home, train=True, download=False) ds = train_test_split(ds, test_ratio=0.02, shuffle=True)[1] cv = KFold(5, True, train_transform, test_transform, random_state=0) def fit_fn(ds_train, ds_val, verbose): net = LeNet5() criterion = nn.CrossEntropyLoss() optimizer = SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4, nesterov=True) # lr_scheduler = MultiStepLR(optimizer, [10, 20], gamma=0.1) lr_scheduler = CosineAnnealingLR(optimizer,
else: num_classes, CIFAR = 100, CIFAR100 if cfg.get("hpo"): import nni RCV_CONFIG = nni.get_next_parameter() for k, v in RCV_CONFIG.items(): ks = k.split(".") override(cfg, ks, v) ds_train = CIFAR(data_home, train=True, transform=train_transform) ds_test = CIFAR(data_home, train=False, transform=test_transform) if cfg.get("Debug") and cfg.Debug.get("subset"): ratio = cfg.Debug.subset ds_train = train_test_split(ds_train, ratio)[1] ds_test = train_test_split(ds_test, ratio)[1] use_mix = cfg.get("Mix") is not None if use_mix: cfg.Mix.num_classes = num_classes ds_train = get_mix(cfg.Mix, ds_train) train_loader = get_data_loader(cfg.Dataset.Train, ds_train) test_loader = get_data_loader(cfg.Dataset.Test, ds_test) cfg.Model.num_classes = num_classes model = get_model(cfg.Model, horch.models.cifar) criterion = CrossEntropyLoss(non_sparse=use_mix, label_smoothing=cfg.get("label_smooth"))
def main(): torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True torch.backends.cudnn.deterministic = False manual_seed(args.seed) train_transform = Compose([ RandomCrop(32, padding=4), RandomHorizontalFlip(), ToTensor(), Normalize([0.491, 0.482, 0.447], [0.247, 0.243, 0.262]), ]) ds = CIFAR10(root=args.data, train=True, download=True) ds_train, ds_search = train_test_split( ds, test_ratio=0.5, shuffle=True, random_state=args.seed, transform=train_transform, test_transform=train_transform) train_queue = DataLoader( ds_train, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2) valid_queue = DataLoader( ds_search, batch_size=args.batch_size, pin_memory=True, shuffle=True, num_workers=2) set_defaults({ 'relu': { 'inplace': False, }, 'bn': { 'affine': False, } }) model = Network(args.init_channels, args.layers, num_classes=CIFAR_CLASSES) criterion = nn.CrossEntropyLoss() optimizer_arch = Adam( model.arch_parameters(), lr=args.arch_learning_rate, betas=(0.5, 0.999), weight_decay=args.arch_weight_decay) optimizer_model = SGD( model.model_parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = CosineLR( optimizer_model, float(args.epochs), min_lr=args.learning_rate_min) train_metrics = { "loss": TrainLoss(), "acc": Accuracy(), } eval_metrics = { "loss": Loss(criterion), "acc": Accuracy(), } learner = DARTSLearner(model, criterion, optimizer_arch, optimizer_model, scheduler, train_metrics=train_metrics, eval_metrics=eval_metrics, search_loader=valid_queue, grad_clip_norm=5.0, work_dir='models') for epoch in range(args.epochs): scheduler.step() lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) genotype = model.genotype() logging.info('genotype = %s', genotype) print(F.softmax(model.alphas_normal, dim=-1)) print(F.softmax(model.alphas_reduce, dim=-1)) print(F.softmax(model.betas_normal[2:5], dim=-1)) # training train_acc, train_obj = train(learner, train_queue, epoch) logging.info('train_acc %f', train_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
from horch.legacy.train.metrics import TrainLoss, Loss from horch.legacy.train.metrics import Accuracy manual_seed(0) train_transform = Compose([ Pad(2), ToTensor(), Normalize((0.1307, ), (0.3081, )), Lambda(lambda x: x.expand(3, -1, -1)) ]) root = '/Users/hrvvi/Code/study/pytorch/datasets' ds_all = MNIST(root=root, train=True, download=True, transform=train_transform) ds = train_test_split(ds_all, test_ratio=0.001, shuffle=True)[1] ds_train, ds_val = train_test_split(ds, test_ratio=0.5, shuffle=True) ds = CombineDataset(ds_train, ds_val) train_loader = DataLoader(ds, batch_size=2, pin_memory=True, num_workers=2) val_loader = DataLoader(ds_val, batch_size=2, pin_memory=True, num_workers=2) cfg.relu.inplace = False cfg.bn.affine = False criterion = nn.CrossEntropyLoss() tau_max, tau_min = 10, 0.1 model = Network(8, 8, steps=4, multiplier=4, stem_multiplier=1, tau=tau_max) optimizer_arch = Adam(model.arch_parameters(), lr=3e-4,
train_transform = Compose(cfg.Dataset.Train.transforms) test_transform = Compose(cfg.Dataset.Test.transforms) data_home = cfg.Dataset.data_home ds_train = CIFAR10(data_home, train=True, download=True, transform=train_transform) ds_test = CIFAR10(data_home, train=False, download=True, transform=test_transform) if cfg.get("Debug") and cfg.Debug.get("subset"): ratio = cfg.Debug.subset ds_train = train_test_split(ds_train, test_ratio=ratio, random=True)[1] ds_test = train_test_split(ds_test, test_ratio=ratio, random=True)[1] train_loader = get_dataloader(cfg.Dataset.Train, ds_train) test_loader = get_dataloader(cfg.Dataset.Test, ds_test) net = get_model(cfg, horch.models.cifar) criterion = CrossEntropyLoss(label_smoothing=cfg.get("label_smooth")) optimizer = get_optimizer(cfg.Optimizer, net) lr_scheduler = get_lr_scheduler(cfg.LRScheduler, optimizer) mix = get_mix(cfg.get("Mix")) metrics = {