def main(): np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True cudnn.enabled = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # equal to: genotype = genotypes.DARTS_v2 genotype = eval("genotypes.%s" % args.arch) print('Load genotype:', genotype) model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() utils.load(model, args.exp_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def _init_data_queue(self): train_transform, valid_transform = dutils._data_transforms_cifar10( self.args) train_data = dset.CIFAR10(root=self.args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=self.args.data, train=False, download=True, transform=valid_transform) self.num_classes = 10 self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, shuffle=True, pin_memory=True, num_workers=4) self.valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=self.args.batch_size, shuffle=False, pin_memory=True, num_workers=4)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) torch.cuda.set_device(args.gpu) cudnn.enabled = True logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=False, num_workers=2) model.drop_path_prob = 0.0 test_acc, test_obj = infer(test_queue, model, criterion) logging.info('Test_acc %f', test_acc)
def main(): np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True torch.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batchsz, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc) train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc: %f', train_acc) utils.save(model, os.path.join(args.save, 'trained.pt')) print('saved to: trained.pt')
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) args.seed = np.random.randint(0, 100) logging.info("----------seed = {}-----------".format(args.seed)) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) logging.info("Arc = {}".format(genotype)) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs),eta_min=args.learning_rate_min) Best_valid_acc = 0.0 for epoch in range(args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) scheduler.step() if valid_acc >= Best_valid_acc: Best_valid_acc = valid_acc logging.info('Best_valid_acc %f', Best_valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) cudnn.benchmark = True cudnn.enabled=True logging.info("args = %s", args) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, genotype) if args.parallel: model = MyDataParallel(model).cuda() else: model = model.cuda() bin_op = bin_utils.BinOp(model, args) _, valid_transform = utils._data_transforms_cifar10(args) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) utils.load(model, args.path_to_weights) if args.parallel: model.module.drop_path_prob = args.drop_path_prob * (args.epochs-1) / args.epochs else: model.drop_path_prob = args.drop_path_prob * (args.epochs-1) / args.epochs valid_acc, valid_obj = infer(valid_queue, model, criterion, bin_op) logging.info('valid_acc %f', valid_acc)
def main(): # logging.info('no gpu device available') # sys.exit(1) np.random.seed(args.seed) torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) utils.load(model, args.model_path, strict=False) model = model.to_device() # to(torch._default_device) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.to_device() # to(torch._default_device) _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def load_data_cifar(args): ## step 3 load test data if args.cifar100: train_transform, valid_transform = utils._data_transforms_cifar100( args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.cifar100: # train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data_dir, train=False, download=True, transform=valid_transform) else: # train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data_dir, train=False, download=True, transform=valid_transform) # train_queue = torch.utils.data.DataLoader( # train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) return valid_queue
def load_dataset(args=DEFAULT_ARGS): if args.dataset == 'cifar10': train_transform, valid_transform = utils._data_transforms_cifar10(args.cutout_length if args.cutout else None) train_data = CIFAR10(root=args.data, train=True, download=True, transform=train_transform) test_data = CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=2) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.evaluate_batch_size, shuffle=False, pin_memory=True, num_workers=2) else: raise NotImplementedError("Temporary not used.") return train_queue, valid_queue, test_queue
def _create_dataset(): train_transform, valid_transform = utils._data_transforms_cifar10( args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) self.loader_train = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[:num_train]), pin_memory=True, num_workers=args.num_workers) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) self.loader_test = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers)
def build_cifar100(model_state_dict=None, optimizer_state_dict=None, args=None, **kwargs): epoch = kwargs.pop('epoch') ratio = kwargs.pop('ratio') train_transform, valid_transform = utils._data_transforms_cifar10( args.cutout_size) train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=valid_transform) num_train = len(train_data) assert num_train == len(valid_data) indices = list(range(num_train)) split = int(np.floor(ratio * num_train)) np.random.shuffle(indices) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.child_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=16) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.child_eval_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=16) model = NASWSNetworkCIFAR(100, args.child_layers, args.child_nodes, args.child_channels, args.child_keep_prob, args.child_drop_path_keep_prob, args.child_use_aux_head, args.steps) model = model.cuda() train_criterion = nn.CrossEntropyLoss().cuda() eval_criterion = nn.CrossEntropyLoss().cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD( model.parameters(), args.child_lr_max, momentum=0.9, weight_decay=args.child_l2_reg, ) if model_state_dict is not None: model.load_state_dict(model_state_dict) if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, args.child_epochs, args.child_lr_min, epoch) return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def __init__(self, arm): args = {} args['data'] = '/home/liamli4465/darts/data/' args['batch_size'] = 96 args['learning_rate'] = 0.025 args['momentum'] = 0.9 args['weight_decay'] = 3e-4 args['report_freq'] = 50 args['gpu'] = 0 args['init_channels'] = 36 args['layers'] = 20 args['auxiliary'] = True args['auxiliary_weight'] = 0.4 args['cutout'] = True args['cutout_length'] = 16 args['drop_path_prob'] = 0.2 args['grad_clip'] = 5 args['epochs'] = 300 args['save'] = arm['dir'] args['seed'] = arm['seed'] args['arch'] = arm['genotype'] args['cuda'] = True args = AttrDict(args) self.args = args random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = False cudnn.enabled = True cudnn.deterministic = True torch.cuda.manual_seed_all(args.seed) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=False, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=False, transform=valid_transform) self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=0) self.valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=0) self.epoch = 0
def init_loaddata(self): train_transform, valid_transform = utils._data_transforms_cifar10( self.args) train_data = dset.CIFAR10(root=self.args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=self.args.data, train=False, download=True, transform=valid_transform) if self.args.seed: def worker_init_fn(): seed = self.seed np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) return else: worker_init_fn = None if self.args.distributed: train_sampler = DistributedSampler(train_data) valid_sampler = DistributedSampler(valid_data) self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size // self.world_size, shuffle=False, num_workers=0, pin_memory=False, sampler=train_sampler) self.valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=self.args.batch_size // self.world_size, shuffle=False, num_workers=0, pin_memory=False, sampler=valid_sampler) else: self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, shuffle=True, pin_memory=False, num_workers=2) self.valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=self.args.batch_size, shuffle=False, pin_memory=False, num_workers=2)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) args.seed += 1 logging.info("args = %s", args) arch = utils.decode_arch(args.arch) model = Network(arch, CIFAR_CLASSES) model = model.cuda() params = utils.count_parameters_in_MB(model) logging.info("param size = %fMB", params / 1e6) flops = utils.count_FLOPs(model) logging.info("FLOPs = %d", flops) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10() train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=True) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=False) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) best_acc = 0 for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc logging.info('valid_acc %f', valid_acc) return params, flops, best_acc
def evaluate(individual, num_classes, num_epochs, batch_size, learning_rate): train_transform, test_transform = utils._data_transforms_cifar10() train_dataset = torchvision.datasets.CIFAR10(root='../../data', train=True, transform=train_transform) test_dataset = torchvision.datasets.CIFAR10(root='../../data', train=False, transform=test_transform) train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=batch_size, shuffle=True, # pin_memory=True, # num_workers=2 ) test_loader = torch.utils.data.DataLoader( dataset=test_dataset, batch_size=batch_size, shuffle=False, # pin_memory=True, # num_workers=2 ) structure = Network(individual.structure, [(3, 32), (32, 128), (128, 128)], num_classes, (32, 32)).to(device) individual.size = utils.count_parameters_in_MB(structure) parameters = filter(lambda p: p.requires_grad, structure.parameters()) cudnn.enabled = True cudnn.benchmark = True criterion = torch.nn.CrossEntropyLoss().to(device) optimizer = torch.optim.SGD(parameters, lr=learning_rate, momentum=0.9, weight_decay=3e-4) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs, eta_min=0.0) best_acc = 0 for epoch in range(num_epochs): print('epoch[{}/{}]:'.format(epoch + 1, num_epochs)) train(train_loader, structure, criterion, optimizer) scheduler.step() valid_acc = test(test_loader, structure, criterion) print() if valid_acc > best_acc: best_acc = valid_acc individual.accuracy = best_acc
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) in_channels = 3 num_classes = 10 stride_for_aux = 3 genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, in_channels, stride_for_aux, num_classes, args.layers, args.auxiliary, genotype) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) #test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) #version = 'v6' #images, labels = utils_test.load_new_test_data(version) #num_images = images.shape[0] data_10 = np.load( "/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_data.npy") labels_10 = np.load( '/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_labels.npy') # tensor_x = torch.stack([torch.Tensor(i) for i in data_10]) # transform to torch tensors tensor_y = torch.stack([torch.Tensor(i) for i in labels_10]) test_data = utils_d.TensorDataset(tensor_x, tensor_y) # create your datset test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # 得到train_search里学好的normal cell 和reduction cell,genotypes.DARTS就是选的学好的DARTS_V2 genotype = eval("genotypes.%s" % args.arch)#DARTS_V2 = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 2)], normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 2), ('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('skip_connect', 2), ('max_pool_3x3', 1)], reduce_concat=[2, 3, 4, 5]) # 这里的Network用的是model.py的NetworkCIFAR model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def get_data_transforms(dataset_name, args): train_transform, valid_transform = None, None if dataset_name == 'cifar10' or dataset_name == 'cifar100': train_transform, valid_transform = utils._data_transforms_cifar10(args) elif dataset_name == 'mnist': train_transform, valid_transform = utils._data_transforms_mnist(args) elif dataset_name == 'mnistm': train_transform, valid_transform = utils._data_transforms_mnistm(args) else: assert False and f'Unrecognized dataset: {dataset_name}' return train_transform, valid_transform
def build_cifar100(model_state_dict, optimizer_state_dict, **kwargs): epoch = kwargs.pop('epoch') train_transform, valid_transform = utils._data_transforms_cifar10( args.cutout_size) train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=16) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.eval_batch_size, shuffle=False, pin_memory=True, num_workers=16) model = NASNetworkCIFAR(args, 100, args.layers, args.nodes, args.channels, args.keep_prob, args.drop_path_keep_prob, args.use_aux_head, args.steps, args.arch) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) logging.info("multi adds = %fM", model.multi_adds / 1000000) if model_state_dict is not None: model.load_state_dict(model_state_dict) if torch.cuda.device_count() > 1: logging.info("Use %d %s", torch.cuda.device_count(), "GPUs !") model = nn.DataParallel(model) model = model.cuda() train_criterion = nn.CrossEntropyLoss().cuda() eval_criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( model.parameters(), args.lr_max, momentum=0.9, weight_decay=args.l2_reg, ) if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), args.lr_min, epoch) return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main(): args = get_args() if torch.cuda.is_available(): print('Train on GPU!') device = torch.device("cuda") else: device = torch.device("cpu") train_transform, valid_transform = _data_transforms_cifar10(args) trainset = torchvision.datasets.CIFAR10(root=args.data_dir, train=True, download=True, transform=train_transform) train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) valset = torchvision.datasets.CIFAR10(root=args.data_dir, train=False, download=True, transform=valid_transform) val_loader = torch.utils.data.DataLoader(valset, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4) model = Network(classes=10, gap_size=1).to(device) criterion = nn.CrossEntropyLoss().to(device) optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lambda epoch: 1 - (epoch / args.epochs)) summary(model, (3, 32, 32)) print('Start training!') for epoch in range(args.epochs): print('epoch:%d, lr:%f' % (epoch, scheduler.get_lr()[0])) train(args, epoch, train_loader, device, model, criterion, optimizer, scheduler) scheduler.step() if (epoch + 1) % args.val_interval == 0: validate(args, epoch, val_loader, device, model) utils.save_checkpoint({ 'state_dict': model.state_dict(), }, epoch + 1, tag=args.exp_name)
def __init__( self, name, args, ): self.name = name self.args = args self.criterion = nn.CrossEntropyLoss() self.criterion = self.criterion.cuda() CIFAR_CLASSES = 10 self.model = Network(args.init_channels, CIFAR_CLASSES, args.layers, self.criterion) self.model = self.model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(self.model)) self.optimizer = torch.optim.SGD(self.model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) self.num_train = num_train indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.train_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[:split]), pin_memory=True, num_workers=2) self.valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.valid_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=2) self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( self.optimizer, float(args.epochs), eta_min=args.learning_rate_min)
def main(): if not torch.cuda.is_available(): logging.info('No GPU device available') sys.exit(1) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) num_gpus = torch.cuda.device_count() logging.info('Testing with %d GPU(s)', num_gpus) model = eval("se_resnet%s(num_classes=CIFAR_CLASSES)" % args.resnet_type) if num_gpus > 1: model = torch.nn.DataParallel(model) model = model.cuda() try: utils.load(model, args.model_path) except: model = model.module utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() if args.cifar100: _, test_transform = utils._data_transforms_cifar100(args) else: _, test_transform = utils._data_transforms_cifar10(args) if args.cifar100: test_data = dset.CIFAR100(root=args.tmp_data_dir, train=False, download=True, transform=test_transform) else: test_data = dset.CIFAR10(root=args.tmp_data_dir, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) test_acc, test_obj = infer(test_queue, model, criterion) logging.info('TEST ACCURACY: --- %f% ---', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) graph = [AGraph(g) for g in glob.glob('comp_graphs/new/*.dot')] _, comp_graph = cell_graph.generate_comp_graph(graph) conf = arch_config(comp_graph=comp_graph, channels=args.init_channels, repeat_list=args.layers, classes=CIFAR_CLASSES) model = get_net(conf) model = model.cuda() state_dict = torch.load(args.model_path)['state_dict'] if isinstance(state_dict, torch.nn.DataParallel): state_dict = state_dict.module model.load_state_dict(state_dict) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): np.random.seed(args.seed) gpus = [int(i) for i in args.gpu.split(',')] if len(gpus) == 1: torch.cuda.set_device(int(args.gpu)) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) logging.info('gpu device = %s' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model.cuda() if len(gpus) > 1: print("True") model = nn.parallel.DataParallel(model, device_ids=gpus, output_device=gpus[0]) model = model.module utils.load(model, args.model_path) print("If the model is running on GPU:", next(model.parameters()).is_cuda) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() checkpoint = torch.load(args.model_path) model.load_state_dict(checkpoint['model_state_dict']) print("param size = {:.1f}MB".format( floor(utils.count_parameters_in_MB(model), 1))) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def init_loaddata(self): train_transform, valid_transform = utils._data_transforms_cifar10( self.args) train_data = dset.CIFAR10(root=self.args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=self.args.data, train=False, download=True, transform=valid_transform) if self.args.seed: def worker_init_fn(): seed = self.seed np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) return else: worker_init_fn = None num_train = len(train_data) indices = list(range(num_train)) self.train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, shuffle=True, pin_memory=False, num_workers=2) self.valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=self.args.batch_size, shuffle=False, pin_memory=False, num_workers=2)
def build_validation_data_loader(self) -> DataLoader: train_transform, valid_transform = utils._data_transforms_cifar10( AttrDict(self.hparams)) valid_data = dset.CIFAR10( root="{}/data-rank{}".format( self.data_config["data_download_dir"], self.context.distributed.get_rank(), ), train=False, download=True, transform=valid_transform, ) valid_queue = DataLoader( valid_data, batch_size=self.context.get_per_slot_batch_size(), shuffle=False, pin_memory=True, num_workers=2, ) return valid_queue
def _create_dataset(): train_transform, valid_transform = utils._data_transforms_cifar10( args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) ratio = 1.0 if args.mode == 0 else args.train_portion split = int(np.floor(ratio * num_train)) self.loader_train = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[:split]), pin_memory=True, num_workers=args.num_workers) if args.mode != 0: self.loader_eval = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=args.num_workers) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) self.loader_test = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers)
def _create_dataset(): train_transform, valid_transform = utils._data_transforms_cifar10( args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) self.loader_train = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) self.loader_test = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers)
def main(): np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info( "device = %s" % 'cuda:{}'.format(args.gpu) if torch.cuda.is_available() else 'cpu') logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype).to(device) utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().to(device) _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2, ) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info("test_acc %f", test_acc)